@superblocksteam/vite-plugin-file-sync 2.0.156-next.3 → 2.0.156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-service/agent/prompts/build-base-system-prompt.d.ts +2 -9
- package/dist/ai-service/agent/prompts/build-base-system-prompt.d.ts.map +1 -1
- package/dist/ai-service/agent/prompts/build-base-system-prompt.js +6 -34
- package/dist/ai-service/agent/prompts/build-base-system-prompt.js.map +1 -1
- package/dist/ai-service/agent/prompts/build-security-scan-prompt.d.ts +1 -2
- package/dist/ai-service/agent/prompts/build-security-scan-prompt.d.ts.map +1 -1
- package/dist/ai-service/agent/prompts/build-security-scan-prompt.js +9 -52
- package/dist/ai-service/agent/prompts/build-security-scan-prompt.js.map +1 -1
- package/dist/ai-service/agent/tool-permission-delivery.d.ts +16 -25
- package/dist/ai-service/agent/tool-permission-delivery.d.ts.map +1 -1
- package/dist/ai-service/agent/tool-permission-delivery.js +38 -384
- package/dist/ai-service/agent/tool-permission-delivery.js.map +1 -1
- package/dist/ai-service/agent/tools/apis/get-sdk-api-docs.d.ts +35 -9
- package/dist/ai-service/agent/tools/apis/get-sdk-api-docs.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/apis/get-sdk-api-docs.js +98 -71
- package/dist/ai-service/agent/tools/apis/get-sdk-api-docs.js.map +1 -1
- package/dist/ai-service/agent/tools/apis/test-api.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/apis/test-api.js +2 -3
- package/dist/ai-service/agent/tools/apis/test-api.js.map +1 -1
- package/dist/ai-service/agent/tools/app-environment/update-app-environment.d.ts +1 -1
- package/dist/ai-service/agent/tools/app-environment/update-app-environment.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/app-environment/update-app-environment.js +1 -13
- package/dist/ai-service/agent/tools/app-environment/update-app-environment.js.map +1 -1
- package/dist/ai-service/agent/tools/build-capture-screenshot.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/build-capture-screenshot.js +3 -8
- package/dist/ai-service/agent/tools/build-capture-screenshot.js.map +1 -1
- package/dist/ai-service/agent/tools/build-navigate-preview.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/build-navigate-preview.js +2 -4
- package/dist/ai-service/agent/tools/build-navigate-preview.js.map +1 -1
- package/dist/ai-service/agent/tools/build-reload-file.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/build-reload-file.js +2 -4
- package/dist/ai-service/agent/tools/build-reload-file.js.map +1 -1
- package/dist/ai-service/agent/tools/get-logs.d.ts +2 -2
- package/dist/ai-service/agent/tools/integrations/delete-integration.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/integrations/delete-integration.js +3 -6
- package/dist/ai-service/agent/tools/integrations/delete-integration.js.map +1 -1
- package/dist/ai-service/agent/tools/integrations/execute-request.d.ts +13 -13
- package/dist/ai-service/agent/tools/integrations/integration-setup.d.ts +6 -18
- package/dist/ai-service/agent/tools/integrations/integration-setup.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/integrations/integration-setup.js +15 -41
- package/dist/ai-service/agent/tools/integrations/integration-setup.js.map +1 -1
- package/dist/ai-service/agent/tools/integrations/internal.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/integrations/internal.js +2 -4
- package/dist/ai-service/agent/tools/integrations/internal.js.map +1 -1
- package/dist/ai-service/agent/tools/report-security-findings.d.ts +23 -23
- package/dist/ai-service/agent/tools.d.ts +2 -4
- package/dist/ai-service/agent/tools.d.ts.map +1 -1
- package/dist/ai-service/agent/tools.js +14 -21
- package/dist/ai-service/agent/tools.js.map +1 -1
- package/dist/ai-service/agent/tools2/tools/git.d.ts +2 -2
- package/dist/ai-service/index.d.ts +25 -26
- package/dist/ai-service/index.d.ts.map +1 -1
- package/dist/ai-service/index.js +121 -340
- package/dist/ai-service/index.js.map +1 -1
- package/dist/ai-service/integrations/store.d.ts +1 -2
- package/dist/ai-service/integrations/store.d.ts.map +1 -1
- package/dist/ai-service/integrations/store.js +2 -3
- package/dist/ai-service/integrations/store.js.map +1 -1
- package/dist/ai-service/judge/tools/playwright-action.d.ts +1 -1
- package/dist/ai-service/judge/tools/submit-feedback.d.ts +1 -1
- package/dist/ai-service/llm/client.d.ts +3 -9
- package/dist/ai-service/llm/client.d.ts.map +1 -1
- package/dist/ai-service/llm/client.js +3 -10
- package/dist/ai-service/llm/client.js.map +1 -1
- package/dist/ai-service/llm/context-v2/context-metrics.d.ts.map +1 -1
- package/dist/ai-service/llm/context-v2/context-metrics.js +0 -3
- package/dist/ai-service/llm/context-v2/context-metrics.js.map +1 -1
- package/dist/ai-service/llm/stream/observers/llmobs.d.ts +6 -6
- package/dist/ai-service/llm/stream/observers/llmobs.d.ts.map +1 -1
- package/dist/ai-service/llm/stream/observers/llmobs.js +9 -10
- package/dist/ai-service/llm/stream/observers/llmobs.js.map +1 -1
- package/dist/ai-service/llm/stream/orchestrator.d.ts +0 -4
- package/dist/ai-service/llm/stream/orchestrator.d.ts.map +1 -1
- package/dist/ai-service/llm/stream/orchestrator.js +41 -19
- package/dist/ai-service/llm/stream/orchestrator.js.map +1 -1
- package/dist/ai-service/llm/stream/retry-engine.d.ts +1 -1
- package/dist/ai-service/llm/stream/retry-engine.d.ts.map +1 -1
- package/dist/ai-service/llm/stream/retry-engine.js +1 -26
- package/dist/ai-service/llm/stream/retry-engine.js.map +1 -1
- package/dist/ai-service/policy-agent-metrics.d.ts +0 -10
- package/dist/ai-service/policy-agent-metrics.d.ts.map +1 -1
- package/dist/ai-service/policy-agent-metrics.js +0 -33
- package/dist/ai-service/policy-agent-metrics.js.map +1 -1
- package/dist/ai-service/state-machine/clark-fsm.d.ts +1 -29
- package/dist/ai-service/state-machine/clark-fsm.d.ts.map +1 -1
- package/dist/ai-service/state-machine/clark-fsm.js +1 -3
- package/dist/ai-service/state-machine/clark-fsm.js.map +1 -1
- package/dist/ai-service/state-machine/handlers/agent-planning.d.ts.map +1 -1
- package/dist/ai-service/state-machine/handlers/agent-planning.js +6 -7
- package/dist/ai-service/state-machine/handlers/agent-planning.js.map +1 -1
- package/dist/ai-service/state-machine/handlers/llm-generating.d.ts.map +1 -1
- package/dist/ai-service/state-machine/handlers/llm-generating.js +5 -18
- package/dist/ai-service/state-machine/handlers/llm-generating.js.map +1 -1
- package/dist/ai-service/state-machine/helpers/fetch-with-reconnect-retry.d.ts.map +1 -1
- package/dist/ai-service/state-machine/helpers/fetch-with-reconnect-retry.js +6 -8
- package/dist/ai-service/state-machine/helpers/fetch-with-reconnect-retry.js.map +1 -1
- package/dist/ai-service/state-machine/helpers/policy-gate-usage-metrics.d.ts +14 -0
- package/dist/ai-service/state-machine/helpers/policy-gate-usage-metrics.d.ts.map +1 -1
- package/dist/ai-service/state-machine/helpers/policy-gate-usage-metrics.js +36 -0
- package/dist/ai-service/state-machine/helpers/policy-gate-usage-metrics.js.map +1 -1
- package/dist/ai-service/state-machine/helpers/pre-session-metrics.d.ts +0 -2
- package/dist/ai-service/state-machine/helpers/pre-session-metrics.d.ts.map +1 -1
- package/dist/ai-service/state-machine/helpers/pre-session-metrics.js +0 -3
- package/dist/ai-service/state-machine/helpers/pre-session-metrics.js.map +1 -1
- package/dist/ai-service/state-machine/helpers/prompt-request-metrics.d.ts +0 -1
- package/dist/ai-service/state-machine/helpers/prompt-request-metrics.d.ts.map +1 -1
- package/dist/ai-service/state-machine/helpers/prompt-request-metrics.js +27 -60
- package/dist/ai-service/state-machine/helpers/prompt-request-metrics.js.map +1 -1
- package/dist/ai-service/state-machine/helpers/stable-peer.d.ts +1 -12
- package/dist/ai-service/state-machine/helpers/stable-peer.d.ts.map +1 -1
- package/dist/ai-service/state-machine/helpers/stable-peer.js +9 -42
- package/dist/ai-service/state-machine/helpers/stable-peer.js.map +1 -1
- package/dist/ai-service/state-machine/mocks.d.ts.map +1 -1
- package/dist/ai-service/state-machine/mocks.js +0 -6
- package/dist/ai-service/state-machine/mocks.js.map +1 -1
- package/dist/ai-service/util/rpc-timeout.d.ts +5 -12
- package/dist/ai-service/util/rpc-timeout.d.ts.map +1 -1
- package/dist/ai-service/util/rpc-timeout.js +5 -12
- package/dist/ai-service/util/rpc-timeout.js.map +1 -1
- package/dist/file-sync-vite-plugin.d.ts.map +1 -1
- package/dist/file-sync-vite-plugin.js +21 -8
- package/dist/file-sync-vite-plugin.js.map +1 -1
- package/dist/lock-service/activity-tracker.d.ts +5 -0
- package/dist/lock-service/activity-tracker.d.ts.map +1 -1
- package/dist/lock-service/activity-tracker.js +13 -0
- package/dist/lock-service/activity-tracker.js.map +1 -1
- package/dist/lock-service/index.d.ts +27 -308
- package/dist/lock-service/index.d.ts.map +1 -1
- package/dist/lock-service/index.js +156 -680
- package/dist/lock-service/index.js.map +1 -1
- package/dist/policy-gate-callback-mapper.d.ts +2 -3
- package/dist/policy-gate-callback-mapper.d.ts.map +1 -1
- package/dist/policy-gate-callback-mapper.js.map +1 -1
- package/dist/policy-gate-runner.d.ts +2 -4
- package/dist/policy-gate-runner.d.ts.map +1 -1
- package/dist/policy-gate-runner.js +8 -31
- package/dist/policy-gate-runner.js.map +1 -1
- package/dist/socket-manager.d.ts +6 -47
- package/dist/socket-manager.d.ts.map +1 -1
- package/dist/socket-manager.js +66 -283
- package/dist/socket-manager.js.map +1 -1
- package/dist/sync-service/index.d.ts.map +1 -1
- package/dist/sync-service/index.js +3 -3
- package/dist/sync-service/index.js.map +1 -1
- package/package.json +8 -8
- package/dist/ai-service/agent/tool-permission-delivery-metrics.d.ts +0 -3
- package/dist/ai-service/agent/tool-permission-delivery-metrics.d.ts.map +0 -1
- package/dist/ai-service/agent/tool-permission-delivery-metrics.js +0 -23
- package/dist/ai-service/agent/tool-permission-delivery-metrics.js.map +0 -1
- package/dist/ai-service/llm/stream/observers/step-token-usage.d.ts +0 -15
- package/dist/ai-service/llm/stream/observers/step-token-usage.d.ts.map +0 -1
- package/dist/ai-service/llm/stream/observers/step-token-usage.js +0 -88
- package/dist/ai-service/llm/stream/observers/step-token-usage.js.map +0 -1
- package/dist/ai-service/llm/stream/terminal-reason.d.ts +0 -6
- package/dist/ai-service/llm/stream/terminal-reason.d.ts.map +0 -1
- package/dist/ai-service/llm/stream/terminal-reason.js +0 -73
- package/dist/ai-service/llm/stream/terminal-reason.js.map +0 -1
- package/dist/ai-service/state-machine/helpers/prompt-degradation-metrics.d.ts +0 -28
- package/dist/ai-service/state-machine/helpers/prompt-degradation-metrics.d.ts.map +0 -1
- package/dist/ai-service/state-machine/helpers/prompt-degradation-metrics.js +0 -254
- package/dist/ai-service/state-machine/helpers/prompt-degradation-metrics.js.map +0 -1
- package/dist/ai-service/util/call-peer.d.ts +0 -11
- package/dist/ai-service/util/call-peer.d.ts.map +0 -1
- package/dist/ai-service/util/call-peer.js +0 -47
- package/dist/ai-service/util/call-peer.js.map +0 -1
- package/dist/ai-service/util/peer-liveness.d.ts +0 -23
- package/dist/ai-service/util/peer-liveness.d.ts.map +0 -1
- package/dist/ai-service/util/peer-liveness.js +0 -42
- package/dist/ai-service/util/peer-liveness.js.map +0 -1
- package/dist/scan-scope-merge.d.ts +0 -16
- package/dist/scan-scope-merge.d.ts.map +0 -1
- package/dist/scan-scope-merge.js +0 -90
- package/dist/scan-scope-merge.js.map +0 -1
- package/dist/test-setup.d.ts +0 -2
- package/dist/test-setup.d.ts.map +0 -1
- package/dist/test-setup.js +0 -21
- package/dist/test-setup.js.map +0 -1
|
@@ -2,10 +2,18 @@ import { addTracingToMethods, ConflictError, NotFoundError, TracedEventEmitter,
|
|
|
2
2
|
import { unwrapResponseDto } from "@superblocksteam/shared";
|
|
3
3
|
import { applyErrorHandling, } from "../errors/error-handler.js";
|
|
4
4
|
import { getErrorMeta } from "../util/logger.js";
|
|
5
|
-
import { OperationQueue } from "../util/operation-queue.js";
|
|
6
5
|
import { ActivityTracker } from "./activity-tracker.js";
|
|
7
6
|
const LOCK_HEARTBEAT_INTERVAL_MS = 30 * 1000; // 30 seconds
|
|
8
7
|
const LOCK_BUFFER_MS = 30 * 1000; // 30 seconds
|
|
8
|
+
// Startup no-socket watchdog: if a freshly-acquired lock never sees an editor
|
|
9
|
+
// socket attach, the pod is reclaimed so an abandoned startup can't hold the
|
|
10
|
+
// exclusive per-app-branch lock forever. The window must cover the entire
|
|
11
|
+
// startup — DBFS download + package install, then socket attach — so it sits
|
|
12
|
+
// well above worst-case hydration (~90s observed) instead of racing app size
|
|
13
|
+
// the way the old 90s window did and bricking slow-hydrating apps. Trade-off:
|
|
14
|
+
// an abandoned-but-hydrated pod is reclaimed up to this long rather than
|
|
15
|
+
// sooner; that case is bounded and force-takeover is available.
|
|
16
|
+
const STARTUP_CONNECTION_TIMEOUT_MS = 4 * 60 * 1000; // 4 minutes
|
|
9
17
|
/**
|
|
10
18
|
* How long a gateway peer's liveness signal counts for before it has to be
|
|
11
19
|
* renewed. A gateway refreshes it on every authorized call and, when it has
|
|
@@ -41,25 +49,6 @@ export var LockType;
|
|
|
41
49
|
LockType["LOCAL"] = "local";
|
|
42
50
|
LockType["CSB"] = "csb";
|
|
43
51
|
})(LockType || (LockType = {}));
|
|
44
|
-
/**
|
|
45
|
-
* The methods {@link LockService} reports as spans.
|
|
46
|
-
*
|
|
47
|
-
* `addTracingToMethods` matches by string, so an entry naming a method that no
|
|
48
|
-
* longer exists is dropped at runtime rather than failing to compile — tracing
|
|
49
|
-
* disappears and nothing says so. Exported so a test can assert every name here
|
|
50
|
-
* still resolves to a method.
|
|
51
|
-
*/
|
|
52
|
-
export const TRACED_LOCK_SERVICE_METHODS = [
|
|
53
|
-
"acquireLock",
|
|
54
|
-
"releaseLock",
|
|
55
|
-
"relinquishLock",
|
|
56
|
-
"notifyClosedByServer",
|
|
57
|
-
"checkAndUpdateLockIfPossible",
|
|
58
|
-
"ensureLockForWrite",
|
|
59
|
-
"shutdownAndExit",
|
|
60
|
-
"restartInactivityTimer",
|
|
61
|
-
"switchBranch",
|
|
62
|
-
];
|
|
63
52
|
export class LockService extends TracedEventEmitter {
|
|
64
53
|
applicationId;
|
|
65
54
|
branchName;
|
|
@@ -68,7 +57,6 @@ export class LockService extends TracedEventEmitter {
|
|
|
68
57
|
syncCallback;
|
|
69
58
|
waitForGenerationCompleteCallback;
|
|
70
59
|
onGatewayLivenessLapsed;
|
|
71
|
-
onLockConflict;
|
|
72
60
|
_status = LockServiceStatus.IDLE;
|
|
73
61
|
_lockId;
|
|
74
62
|
_heartbeatTimeout;
|
|
@@ -80,22 +68,6 @@ export class LockService extends TracedEventEmitter {
|
|
|
80
68
|
// (ready for takeover). Distinguishes that case from operational stops
|
|
81
69
|
// (lock release, shutdown, branch switch) so only idle-stops auto-resume.
|
|
82
70
|
_heartbeatStoppedForIdle = false;
|
|
83
|
-
// True from the moment the lock is deliberately given up until one is held
|
|
84
|
-
// again. The durable answer to "did we choose to let this go?", which the
|
|
85
|
-
// status cannot give: `checkAndUpdateLockIfPossible`, `switchBranch` and a
|
|
86
|
-
// failed release all reset it to IDLE, and IDLE is what tells the claim path
|
|
87
|
-
// a lock has never been held. Also the only thing that still remembers by the
|
|
88
|
-
// time a grab left over from before the release fails.
|
|
89
|
-
_lockGivenUp = false;
|
|
90
|
-
_closedByServer = false;
|
|
91
|
-
/**
|
|
92
|
-
* True only while the last-chance sync of a session that is ending is in
|
|
93
|
-
* flight. That write is not gated on still holding the lock — see
|
|
94
|
-
* {@link ensureLockForWrite}.
|
|
95
|
-
*/
|
|
96
|
-
_flushingFinalWork = false;
|
|
97
|
-
/** The evaluation overlapping callers join rather than starting their own. */
|
|
98
|
-
_inFlightLockEvaluation;
|
|
99
71
|
_inactivityTimeout;
|
|
100
72
|
_bufferPeriodTimeout;
|
|
101
73
|
_isRestarting = false;
|
|
@@ -104,9 +76,9 @@ export class LockService extends TracedEventEmitter {
|
|
|
104
76
|
_rpcClient;
|
|
105
77
|
_inactivityThresholdMs;
|
|
106
78
|
_lockBufferMs;
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
79
|
+
_startupConnectionTimeout;
|
|
80
|
+
_startupConnectionTimeoutStartedAt;
|
|
81
|
+
_startupConnectionTimeoutExpired = false;
|
|
110
82
|
_activityTracker;
|
|
111
83
|
/** Gateway peer id -> when its liveness was last refreshed. */
|
|
112
84
|
_gatewayLivenessByPeerId = new Map();
|
|
@@ -135,7 +107,15 @@ export class LockService extends TracedEventEmitter {
|
|
|
135
107
|
acquireLock: { operation: "acquiring lock" },
|
|
136
108
|
releaseLock: { operation: "releasing lock" },
|
|
137
109
|
});
|
|
138
|
-
|
|
110
|
+
const methods = [
|
|
111
|
+
"acquireLock",
|
|
112
|
+
"releaseLock",
|
|
113
|
+
"checkAndUpdateLockIfPossible",
|
|
114
|
+
"shutdown",
|
|
115
|
+
"shutdownAndExit",
|
|
116
|
+
"restartInactivityTimer",
|
|
117
|
+
];
|
|
118
|
+
addTracingToMethods(this, methods, this._tracer);
|
|
139
119
|
void this.restartInactivityTimer();
|
|
140
120
|
}
|
|
141
121
|
getLogger() {
|
|
@@ -158,28 +138,12 @@ export class LockService extends TracedEventEmitter {
|
|
|
158
138
|
setOnGatewayLivenessLapsed(callback) {
|
|
159
139
|
this.onGatewayLivenessLapsed = callback;
|
|
160
140
|
}
|
|
161
|
-
/**
|
|
162
|
-
* Called when a client attached and somebody else already holds the lock.
|
|
163
|
-
*
|
|
164
|
-
* What a conflict means is the host's to decide, not this service's: a CLI
|
|
165
|
-
* with a person at it can describe the holder and offer a takeover, a
|
|
166
|
-
* sandbox can only write it down. This is the hook that used to be the
|
|
167
|
-
* `ConflictError` catch around the boot-time acquire.
|
|
168
|
-
*/
|
|
169
|
-
setOnLockConflict(callback) {
|
|
170
|
-
this.onLockConflict = callback;
|
|
171
|
-
}
|
|
172
141
|
get status() {
|
|
173
142
|
return this._status;
|
|
174
143
|
}
|
|
175
144
|
set status(status) {
|
|
176
145
|
if (this._status !== status) {
|
|
177
146
|
this._status = status;
|
|
178
|
-
if (status === LockServiceStatus.LOCKED) {
|
|
179
|
-
// Derived from the transition rather than set by each of the three
|
|
180
|
-
// places that can obtain a lock, so it cannot be forgotten in one.
|
|
181
|
-
this._lockGivenUp = false;
|
|
182
|
-
}
|
|
183
147
|
this.emit("statusChange", status);
|
|
184
148
|
}
|
|
185
149
|
}
|
|
@@ -204,185 +168,25 @@ export class LockService extends TracedEventEmitter {
|
|
|
204
168
|
get timeSinceLastActivity() {
|
|
205
169
|
return this._activityTracker.timeSinceLastActivity;
|
|
206
170
|
}
|
|
207
|
-
/** Whether any client — a browser editor or a headless gateway peer — is here. */
|
|
208
|
-
get hasAttachedClient() {
|
|
209
|
-
return (this._activityTracker.connectedUsers.length > 0 ||
|
|
210
|
-
this.hasAttachedGatewayPeer);
|
|
211
|
-
}
|
|
212
|
-
/**
|
|
213
|
-
* A client attached and this pod has never held the lock: take it for them.
|
|
214
|
-
*
|
|
215
|
-
* The lock is a mutual-exclusion marker between clients, not a capability the
|
|
216
|
-
* pod needs in order to serve — no write path on either side gates on holding
|
|
217
|
-
* it. So it belongs to whoever is attached, and a pod that grabs it at boot
|
|
218
|
-
* only invents a state nobody wants: a lock held on behalf of nobody, which
|
|
219
|
-
* then needs a watchdog to clean up. Claiming it on attach makes that state
|
|
220
|
-
* unreachable instead.
|
|
221
|
-
*
|
|
222
|
-
* Deliberately narrower than {@link reclaimLockForActiveClient}: neither a
|
|
223
|
-
* lock this pod handed back nor one it was refused is taken here. A tab left
|
|
224
|
-
* open reconnects its socket on its own, so an attach is no evidence its user
|
|
225
|
-
* came back, and treating it as evidence would hand the app to somebody who
|
|
226
|
-
* walked away (APPS-4840).
|
|
227
|
-
*/
|
|
228
|
-
claimLockForFirstClient() {
|
|
229
|
-
// `IDLE` on its own does not mean "never held". Several paths reset the
|
|
230
|
-
// status to it after a lock we did hold went away — a heartbeat grab that
|
|
231
|
-
// misses, a branch switch, a failed release — so trusting the status alone
|
|
232
|
-
// would turn a lock we handed back into a claimable one, and a tab that
|
|
233
|
-
// reconnects by itself would take the app back from whoever is using it.
|
|
234
|
-
if (this._lockGivenUp) {
|
|
235
|
-
return;
|
|
236
|
-
}
|
|
237
|
-
this.acquireLockForAttachedClient([LockServiceStatus.IDLE]);
|
|
238
|
-
}
|
|
239
|
-
/**
|
|
240
|
-
* The client proved it is still there: take the lock if we do not have it.
|
|
241
|
-
*
|
|
242
|
-
* Driven by editor activity pings and by a gateway peer starting a turn —
|
|
243
|
-
* both are a client doing something, which is the evidence a lock we are not
|
|
244
|
-
* holding needs before it is worth asking for again.
|
|
245
|
-
*
|
|
246
|
-
* That includes a lock we were refused. A refusal is what happened to one
|
|
247
|
-
* attempt, not a mode this pod is stuck in: the other session ends, and the
|
|
248
|
-
* user sitting here still wants to edit. Without this a pod that lost one
|
|
249
|
-
* race could never take the lock again for the rest of its life — and since
|
|
250
|
-
* SABS hands running pods to the next caller, it would go on failing for
|
|
251
|
-
* people who had nothing to do with the race.
|
|
252
|
-
*/
|
|
253
|
-
reclaimLockForActiveClient() {
|
|
254
|
-
// Editors only. A gateway's route to the lock is {@link acquireLockForTurn}
|
|
255
|
-
// — an attached gateway peer is a socket, not a turn, and taking the lock
|
|
256
|
-
// for one would hold write access across the gap between prompts.
|
|
257
|
-
if (this._closedByServer ||
|
|
258
|
-
this._activityTracker.connectedUsers.length === 0) {
|
|
259
|
-
return;
|
|
260
|
-
}
|
|
261
|
-
this.acquireLockForAttachedClient([
|
|
262
|
-
LockServiceStatus.IDLE,
|
|
263
|
-
LockServiceStatus.LOCK_RELEASED,
|
|
264
|
-
LockServiceStatus.FAILED_TO_ACQUIRE_LOCK,
|
|
265
|
-
]);
|
|
266
|
-
}
|
|
267
|
-
/**
|
|
268
|
-
* Fire-and-forget by design: the sockets, pings and branch switches that ask
|
|
269
|
-
* for this have nothing to do with the answer.
|
|
270
|
-
*
|
|
271
|
-
* What keeps a retry from becoming a stream of 409s is the trigger, not the
|
|
272
|
-
* status: the only thing that asks repeatedly is the editor's activity ping,
|
|
273
|
-
* which the pod acts on solely while the user is genuinely active. A blocked
|
|
274
|
-
* session asks about as often as a held lock heartbeats; one nobody is
|
|
275
|
-
* sitting at goes quiet on its own.
|
|
276
|
-
*/
|
|
277
|
-
acquireLockForAttachedClient(from) {
|
|
278
|
-
if (this._clientLockAcquisition || !this.hasAttachedClient) {
|
|
279
|
-
return;
|
|
280
|
-
}
|
|
281
|
-
if (!from.includes(this._status)) {
|
|
282
|
-
return;
|
|
283
|
-
}
|
|
284
|
-
// Only the first refusal is news. The host's handler can prompt on stdin,
|
|
285
|
-
// so reporting each retry would be a prompt every ping.
|
|
286
|
-
const alreadyRefused = this._status === LockServiceStatus.FAILED_TO_ACQUIRE_LOCK;
|
|
287
|
-
if (!alreadyRefused) {
|
|
288
|
-
this.getLogger().info(`[lock-service] client attached with no lock held, acquiring (appId=${this.applicationId})`);
|
|
289
|
-
}
|
|
290
|
-
// Track the queued operation immediately so a lock-state evaluation cannot
|
|
291
|
-
// slip in before the queue starts it. `acquireLock` tracks the RPC itself as
|
|
292
|
-
// well, which covers callers that do not use this attach path.
|
|
293
|
-
void this.trackAcquisition(this._lockOperationQueue.enqueue(async () => {
|
|
294
|
-
if (!this.hasAttachedClient || !from.includes(this._status)) {
|
|
295
|
-
return;
|
|
296
|
-
}
|
|
297
|
-
await this.acquireLock();
|
|
298
|
-
})).catch((error) => alreadyRefused ? undefined : this.reportLockConflict(error));
|
|
299
|
-
}
|
|
300
|
-
/**
|
|
301
|
-
* Record an acquisition so anything that settles can see it.
|
|
302
|
-
*
|
|
303
|
-
* Every source goes through here — a client attaching, an activity ping, a
|
|
304
|
-
* force takeover, the reacquire after a heartbeat grab misses, a branch
|
|
305
|
-
* switch. Tracking only the attach path left the others invisible:
|
|
306
|
-
* `isLockedWhenSettled` would answer "no lock" for one that was an rpc away
|
|
307
|
-
* from landing, and `relinquishLock` would find nothing to release and let
|
|
308
|
-
* the lock land on a pod that had already gone.
|
|
309
|
-
*
|
|
310
|
-
* What is stored is a swallowed copy. Callers of
|
|
311
|
-
* {@link settleClientLockAcquisition} only want the state to have stopped
|
|
312
|
-
* moving; a rejection surfacing as theirs would turn a refused lock into a
|
|
313
|
-
* failed disconnect. The promise handed back keeps its rejection for whoever
|
|
314
|
-
* asked for the lock.
|
|
315
|
-
*/
|
|
316
|
-
trackAcquisition(acquisition) {
|
|
317
|
-
const tracked = acquisition
|
|
318
|
-
.catch(() => undefined)
|
|
319
|
-
.finally(() => {
|
|
320
|
-
if (this._clientLockAcquisition === tracked) {
|
|
321
|
-
this._clientLockAcquisition = undefined;
|
|
322
|
-
}
|
|
323
|
-
});
|
|
324
|
-
this._clientLockAcquisition = tracked;
|
|
325
|
-
return acquisition;
|
|
326
|
-
}
|
|
327
|
-
/**
|
|
328
|
-
* Wait for an acquisition a client attach kicked off, if one is in flight.
|
|
329
|
-
*
|
|
330
|
-
* Anything that decides what to do about the lock has to do this first, or it
|
|
331
|
-
* decides on a half-known state. While the RPC is out `isLocked` is false and
|
|
332
|
-
* `_lockId` is unset, so a disconnect would "release" nothing and the lock
|
|
333
|
-
* would land moments later with nobody left to hold it for — and a branch
|
|
334
|
-
* switch would move on and leave it attached to the branch it came from.
|
|
335
|
-
*
|
|
336
|
-
* Never rejects: the acquisition's own catch has already handled failure.
|
|
337
|
-
*/
|
|
338
|
-
async settleClientLockAcquisition() {
|
|
339
|
-
await this._clientLockAcquisition;
|
|
340
|
-
}
|
|
341
|
-
/**
|
|
342
|
-
* Whether this pod holds the lock after attach and queued turn operations
|
|
343
|
-
* already in flight have settled. The synchronous getter remains the current
|
|
344
|
-
* state for status reporting.
|
|
345
|
-
*/
|
|
346
|
-
async isLockedWhenSettled() {
|
|
347
|
-
await this.settleClientLockAcquisition();
|
|
348
|
-
return await this._lockOperationQueue.enqueue(async () => this.isLocked);
|
|
349
|
-
}
|
|
350
171
|
/**
|
|
351
|
-
*
|
|
352
|
-
*
|
|
353
|
-
*
|
|
354
|
-
* the status to `FAILED_TO_ACQUIRE_LOCK`; the host is told separately because
|
|
355
|
-
* only it knows whether there is anyone to tell.
|
|
172
|
+
* Check if the lock service is in a state where it can reacquire a lock.
|
|
173
|
+
* This happens when the lock was released, the dev server is still alive,
|
|
174
|
+
* and an editor is attached to reacquire it for.
|
|
356
175
|
*/
|
|
357
|
-
|
|
358
|
-
if (
|
|
359
|
-
return;
|
|
360
|
-
}
|
|
361
|
-
try {
|
|
362
|
-
await this.onLockConflict(error);
|
|
363
|
-
}
|
|
364
|
-
catch (handlerError) {
|
|
365
|
-
this.getLogger().warn(`[lock-service] onLockConflict threw; continuing (appId=${this.applicationId})`, getErrorMeta(handlerError));
|
|
176
|
+
get canReacquireLock() {
|
|
177
|
+
if (this._activityTracker.connectedUsers.length === 0) {
|
|
178
|
+
return false;
|
|
366
179
|
}
|
|
180
|
+
return (this._status === LockServiceStatus.LOCK_RELEASED ||
|
|
181
|
+
this._status === LockServiceStatus.IDLE);
|
|
367
182
|
}
|
|
368
|
-
/**
|
|
369
|
-
* Serializes branch changes with lock acquisition and handback. Omitting
|
|
370
|
-
* `reacquireLock` preserves the lock state observed inside that operation.
|
|
371
|
-
*/
|
|
372
183
|
async switchBranch(nextBranchName, options) {
|
|
373
|
-
// Git activation runs off the same socket connect that starts an
|
|
374
|
-
// acquisition, so the switch really can arrive while one is out.
|
|
375
|
-
await this.settleClientLockAcquisition();
|
|
376
|
-
await this._lockOperationQueue.enqueue(() => this.switchBranchNow(nextBranchName, options));
|
|
377
|
-
}
|
|
378
|
-
async switchBranchNow(nextBranchName, options) {
|
|
379
184
|
if (this.branchName === nextBranchName) {
|
|
380
185
|
return;
|
|
381
186
|
}
|
|
382
187
|
const previousBranchName = this.branchName;
|
|
383
188
|
const lockIdBeforeSwitch = this._lockId;
|
|
384
|
-
|
|
385
|
-
if (reacquireLock && this.isLocked && lockIdBeforeSwitch) {
|
|
189
|
+
if (options?.reacquireLock && this.isLocked && lockIdBeforeSwitch) {
|
|
386
190
|
this.stopHeartbeat();
|
|
387
191
|
try {
|
|
388
192
|
const lockResponse = await this._rpcClient.call(async (client) => {
|
|
@@ -400,6 +204,9 @@ export class LockService extends TracedEventEmitter {
|
|
|
400
204
|
this.branchName = nextBranchName;
|
|
401
205
|
this._lockId = transferredLock.lockId;
|
|
402
206
|
this.status = LockServiceStatus.LOCKED;
|
|
207
|
+
// A same-pod atomic transfer is not a fresh startup, so the no-socket
|
|
208
|
+
// watchdog is intentionally left untouched: it keeps ticking from the
|
|
209
|
+
// original acquisition rather than restarting on every branch switch.
|
|
403
210
|
this.getLogger().info(`[lock-service] atomically transferred lock branch context from '${previousBranchName}' to '${nextBranchName}'`);
|
|
404
211
|
void this.scheduleNextHeartbeat();
|
|
405
212
|
return;
|
|
@@ -420,27 +227,19 @@ export class LockService extends TracedEventEmitter {
|
|
|
420
227
|
else {
|
|
421
228
|
this.stopHeartbeat();
|
|
422
229
|
this._lockId = undefined;
|
|
423
|
-
|
|
424
|
-
this.status = LockServiceStatus.IDLE;
|
|
425
|
-
}
|
|
230
|
+
this.status = LockServiceStatus.IDLE;
|
|
426
231
|
}
|
|
427
232
|
this.branchName = nextBranchName;
|
|
428
233
|
this.getLogger().info(`[lock-service] switched branch context from '${previousBranchName}' to '${nextBranchName}'`);
|
|
429
|
-
if (reacquireLock) {
|
|
234
|
+
if (options?.reacquireLock) {
|
|
430
235
|
await this.acquireLock();
|
|
431
236
|
}
|
|
432
237
|
}
|
|
433
238
|
/**
|
|
434
|
-
* Acquire a lock on the specified application and branch
|
|
435
|
-
*
|
|
436
|
-
* Not `async`: the body has to start and be registered in the same tick, so
|
|
437
|
-
* a caller that immediately settles sees this acquisition rather than
|
|
438
|
-
* missing it by a microtask.
|
|
239
|
+
* Acquire a lock on the specified application and branch
|
|
439
240
|
*/
|
|
440
|
-
acquireLock() {
|
|
441
|
-
|
|
442
|
-
}
|
|
443
|
-
async acquireLockImpl() {
|
|
241
|
+
async acquireLock() {
|
|
242
|
+
this.startStartupConnectionTimeoutIfNeeded();
|
|
444
243
|
this.status = LockServiceStatus.ACQUIRING;
|
|
445
244
|
// Stop any existing heartbeat before acquiring a new lock to prevent duplicate heartbeat loops
|
|
446
245
|
this.stopHeartbeat();
|
|
@@ -460,27 +259,17 @@ export class LockService extends TracedEventEmitter {
|
|
|
460
259
|
this.status = LockServiceStatus.LOCKED;
|
|
461
260
|
this._heartbeatStoppedForIdle = false;
|
|
462
261
|
this.emit("lockAcquired", this._lockId);
|
|
262
|
+
this.evaluateStartupConnectionTimeoutIfNeeded();
|
|
463
263
|
void this.scheduleNextHeartbeat();
|
|
464
264
|
return;
|
|
465
265
|
}
|
|
466
|
-
// A response carrying no lock is not success. Falling through would leave
|
|
467
|
-
// the status in ACQUIRING, which is what the catch below exists to avoid.
|
|
468
|
-
throw new Error("lock acquire returned no lock");
|
|
469
266
|
}
|
|
470
267
|
catch (error) {
|
|
471
|
-
// Whatever went wrong, this attempt is over and the lock is not held.
|
|
472
|
-
// Leaving ACQUIRING behind would be a lie the retry paths cannot see past:
|
|
473
|
-
// it is in neither the claim nor the reclaim set, so a single transient
|
|
474
|
-
// failure would stop this pod ever taking the lock again.
|
|
475
|
-
this.status = LockServiceStatus.FAILED_TO_ACQUIRE_LOCK;
|
|
476
268
|
if (error instanceof ConflictError) {
|
|
477
269
|
this.getLogger().error(`[lock-service] ${error.message}`);
|
|
478
270
|
this.emit("statusChange", LockServiceStatus.FAILED_TO_ACQUIRE_LOCK);
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
// closed the rpc client — leaving a pod bound to its port with no way
|
|
482
|
-
// to reach the server, which is the zombie state. The error goes to the
|
|
483
|
-
// caller, which owns what happens to the process.
|
|
271
|
+
this.status = LockServiceStatus.FAILED_TO_ACQUIRE_LOCK;
|
|
272
|
+
await this.shutdown({ serverInitiated: false });
|
|
484
273
|
throw error;
|
|
485
274
|
}
|
|
486
275
|
else {
|
|
@@ -488,67 +277,6 @@ export class LockService extends TracedEventEmitter {
|
|
|
488
277
|
}
|
|
489
278
|
}
|
|
490
279
|
}
|
|
491
|
-
/**
|
|
492
|
-
* Take write access for a turn that is about to start.
|
|
493
|
-
*
|
|
494
|
-
* The lock is permission to write for one turn, not a lease on the session: a
|
|
495
|
-
* headless turn takes it on the way in and hands it back when it ends, so the
|
|
496
|
-
* app sits unlocked between prompts and a person can open it in the editor.
|
|
497
|
-
*
|
|
498
|
-
* Deliberately not routed through {@link claimLockForFirstClient} or
|
|
499
|
-
* {@link reclaimLockForActiveClient}. Those weigh up whether a signal is
|
|
500
|
-
* evidence of a client that wants the lock; a turn starting is not evidence
|
|
501
|
-
* of that, it *is* that, so it asks directly and a lock we handed back
|
|
502
|
-
* earlier is taken again without ceremony.
|
|
503
|
-
*
|
|
504
|
-
* Unlike an attach, this is awaited and its failure is the caller's: a turn
|
|
505
|
-
* that cannot write must fail before routing is bound, carrying the server's
|
|
506
|
-
* own reason. Failing does not take the pod down — the lock is per app+branch
|
|
507
|
-
* while a pod is per live edit, so somebody editing in the browser says
|
|
508
|
-
* nothing about whether this pod can serve the turn after them.
|
|
509
|
-
*/
|
|
510
|
-
async acquireLockForTurn() {
|
|
511
|
-
await this.settleClientLockAcquisition();
|
|
512
|
-
await this._lockOperationQueue.enqueue(async () => {
|
|
513
|
-
if (!this.isLocked) {
|
|
514
|
-
await this.acquireLock();
|
|
515
|
-
}
|
|
516
|
-
});
|
|
517
|
-
}
|
|
518
|
-
/**
|
|
519
|
-
* Hand back the write access a turn took, once that turn has finished.
|
|
520
|
-
*
|
|
521
|
-
* The wait belongs here rather than in the callers. A turn ends in more than
|
|
522
|
-
* one way — the gateway releases it, its socket drops, its liveness lapses —
|
|
523
|
-
* and only the first of those arrives with the generation already complete.
|
|
524
|
-
* The other two reach this while Clark may still be writing, and handing the
|
|
525
|
-
* lock back there gives write access away mid-generation.
|
|
526
|
-
*
|
|
527
|
-
* Before the queue, never inside it, for the reason spelled out on
|
|
528
|
-
* {@link handBackTurnLockWhenGenerationSettles}: that queue serialises the
|
|
529
|
-
* handback, branch switches and the next turn's acquisition, so a wait
|
|
530
|
-
* inside it lets one hung turn stall all of them.
|
|
531
|
-
*
|
|
532
|
-
* `isStillNeeded` is re-read after that wait, because waiting is exactly
|
|
533
|
-
* what makes the caller's decision stale: a replacement gateway can attach
|
|
534
|
-
* and take a turn while the old one's generation is finishing, and releasing
|
|
535
|
-
* then would pull write access out from under it. The caller owns the
|
|
536
|
-
* question — this cannot ask "is a gateway attached?" itself, since the
|
|
537
|
-
* explicit `editor.releaseTurnLock` path runs with one attached by
|
|
538
|
-
* definition.
|
|
539
|
-
*/
|
|
540
|
-
async relinquishLockForTurn(isStillNeeded) {
|
|
541
|
-
await this.waitForGenerationCompleteCallback?.();
|
|
542
|
-
await this._lockOperationQueue.enqueue(async () => {
|
|
543
|
-
if (!this.isLocked ||
|
|
544
|
-
this._activityTracker.connectedUsers.length > 0 ||
|
|
545
|
-
isStillNeeded?.()) {
|
|
546
|
-
return;
|
|
547
|
-
}
|
|
548
|
-
await this.relinquishLockNow(() => this._activityTracker.connectedUsers.length > 0 ||
|
|
549
|
-
(isStillNeeded?.() ?? false));
|
|
550
|
-
});
|
|
551
|
-
}
|
|
552
280
|
/**
|
|
553
281
|
* Release the currently held lock
|
|
554
282
|
*/
|
|
@@ -556,7 +284,6 @@ export class LockService extends TracedEventEmitter {
|
|
|
556
284
|
if (this._status !== LockServiceStatus.LOCKED) {
|
|
557
285
|
return false;
|
|
558
286
|
}
|
|
559
|
-
this._lockGivenUp = true;
|
|
560
287
|
this.status = LockServiceStatus.RELEASING;
|
|
561
288
|
this.emit("statusChange", LockServiceStatus.RELEASING);
|
|
562
289
|
// Stop heartbeat immediately and clear local lock id first to prevent any
|
|
@@ -605,6 +332,7 @@ export class LockService extends TracedEventEmitter {
|
|
|
605
332
|
clearTimeout(this._inactivityTimeout);
|
|
606
333
|
this._inactivityTimeout = undefined;
|
|
607
334
|
}
|
|
335
|
+
this.clearStartupConnectionTimeout();
|
|
608
336
|
}
|
|
609
337
|
}
|
|
610
338
|
async grabLockImpl() {
|
|
@@ -635,10 +363,8 @@ export class LockService extends TracedEventEmitter {
|
|
|
635
363
|
/**
|
|
636
364
|
* Force takeover of the lock from another user (only allowed when user is inactive)
|
|
637
365
|
*/
|
|
638
|
-
forceTakeover() {
|
|
639
|
-
|
|
640
|
-
}
|
|
641
|
-
async forceTakeoverImpl() {
|
|
366
|
+
async forceTakeover() {
|
|
367
|
+
this.startStartupConnectionTimeoutIfNeeded();
|
|
642
368
|
this.status = LockServiceStatus.ACQUIRING;
|
|
643
369
|
try {
|
|
644
370
|
const lockResponse = await this._rpcClient.call(async (client) => {
|
|
@@ -655,14 +381,10 @@ export class LockService extends TracedEventEmitter {
|
|
|
655
381
|
this.getLogger().info(`[lock-service] lock acquired via force takeover: ${this._lockId}`);
|
|
656
382
|
this.status = LockServiceStatus.LOCKED;
|
|
657
383
|
this.emit("lockAcquired", this._lockId);
|
|
384
|
+
this.evaluateStartupConnectionTimeoutIfNeeded();
|
|
658
385
|
void this.scheduleNextHeartbeat();
|
|
659
386
|
return;
|
|
660
387
|
}
|
|
661
|
-
// As in `acquireLock`: a response carrying no lock is not success.
|
|
662
|
-
// Falling through would leave the status in ACQUIRING, which belongs to
|
|
663
|
-
// neither the claim set nor the reclaim set — so nothing would ever try
|
|
664
|
-
// again and the pod would sit there unable to edit.
|
|
665
|
-
throw new Error("lock force takeover returned no lock");
|
|
666
388
|
}
|
|
667
389
|
catch (error) {
|
|
668
390
|
this.getLogger().error(`[lock-service] ${getErrorMeta(error).error.message}`);
|
|
@@ -697,20 +419,6 @@ export class LockService extends TracedEventEmitter {
|
|
|
697
419
|
this.stopHeartbeat();
|
|
698
420
|
this._lockId = undefined;
|
|
699
421
|
this.status = LockServiceStatus.IDLE;
|
|
700
|
-
// Only take it back if we still wanted it, and only for somebody.
|
|
701
|
-
//
|
|
702
|
-
// A grab left over from before a release lands after the lock is
|
|
703
|
-
// already gone on purpose — acquiring fires a heartbeat immediately,
|
|
704
|
-
// so that overlap is the norm on the attach-and-drop path rather than
|
|
705
|
-
// a rare coincidence. Reacquiring there takes back a lock we chose to
|
|
706
|
-
// give up, either for nobody at all or for a tab that has merely
|
|
707
|
-
// reconnected, which is the app being held away from whoever is
|
|
708
|
-
// actually using it (APPS-4840). A released lock is taken back by an
|
|
709
|
-
// activity ping, never from here.
|
|
710
|
-
if (this._lockGivenUp || !this.hasAttachedClient) {
|
|
711
|
-
this.getLogger().info(`[lock-service] lock lost during heartbeat grab with nothing to take it back for, leaving it (appId=${this.applicationId})`);
|
|
712
|
-
return false;
|
|
713
|
-
}
|
|
714
422
|
try {
|
|
715
423
|
await this.acquireLock();
|
|
716
424
|
this.getLogger().info("[lock-service] lock reacquired after heartbeat grab miss");
|
|
@@ -756,220 +464,50 @@ export class LockService extends TracedEventEmitter {
|
|
|
756
464
|
return false;
|
|
757
465
|
}
|
|
758
466
|
/**
|
|
759
|
-
*
|
|
760
|
-
*
|
|
761
|
-
* The write is itself the demand for the lock, which is what separates this
|
|
762
|
-
* from the heartbeat's check above. That one asks whether a lock we already
|
|
763
|
-
* hold is still there, and answers for a lock id; this one is asked before a
|
|
764
|
-
* sync, when there may be no lock at all.
|
|
765
|
-
*
|
|
766
|
-
* The distinction is load-bearing at boot. Nothing acquires the lock at
|
|
767
|
-
* startup any more — a client attaching is what claims it — but the dev
|
|
768
|
-
* server still uploads the user's local files before any editor connects.
|
|
769
|
-
* With no lock id {@link checkAndUpdateLockIfPossible} skips the grab
|
|
770
|
-
* entirely and reports false without ever asking the server, so routing that
|
|
771
|
-
* upload through it failed the sync and took the dev server down with it.
|
|
772
|
-
*
|
|
773
|
-
* A lock this session handed back is still not taken back here: a write
|
|
774
|
-
* queued before the handback belongs to the session that ended.
|
|
467
|
+
* Cleanup method to be called when the class is destroyed
|
|
468
|
+
* This ensures the lock is released
|
|
775
469
|
*/
|
|
776
|
-
async
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
const held = await this.checkAndUpdateLockIfPossible();
|
|
780
|
-
// The last write of a session that is ending is not gated on still
|
|
781
|
-
// holding the lock. By the time the server closes a session the other
|
|
782
|
-
// one has usually taken the lock already, so the grab above misses and
|
|
783
|
-
// the answer is "no" — which would drop whatever the editor had not
|
|
784
|
-
// synced yet, and make the flush that runs at exactly this moment dead
|
|
785
|
-
// code, since it could never succeed.
|
|
786
|
-
//
|
|
787
|
-
// Writing anyway is safe in the way that matters: the lock is a
|
|
788
|
-
// mutual-exclusion marker between clients, not something the server
|
|
789
|
-
// enforces on writes, and the session being handed to on a local<->cloud
|
|
790
|
-
// switch belongs to the same user whose edits these are.
|
|
791
|
-
return held || this._flushingFinalWork;
|
|
792
|
-
}
|
|
793
|
-
if (this._lockGivenUp) {
|
|
794
|
-
this.getLogger().info(`[lock-service] write wants a lock this session handed back, leaving it (appId=${this.applicationId})`);
|
|
795
|
-
return false;
|
|
796
|
-
}
|
|
797
|
-
// Read before the attempt, because a refusal is what moves the status
|
|
798
|
-
// there: only the first one is news.
|
|
799
|
-
const alreadyRefused = this._status === LockServiceStatus.FAILED_TO_ACQUIRE_LOCK;
|
|
800
|
-
try {
|
|
801
|
-
await this.trackAcquisition(this._lockOperationQueue.enqueue(async () => {
|
|
802
|
-
// Re-read inside the queue: an attach or a handback can have landed
|
|
803
|
-
// while this waited its turn.
|
|
804
|
-
if (this._lockId || this._lockGivenUp) {
|
|
805
|
-
return;
|
|
806
|
-
}
|
|
807
|
-
await this.acquireLock();
|
|
808
|
-
}));
|
|
470
|
+
async shutdown({ serverInitiated, switchingTo, initiatedByEmail, }) {
|
|
471
|
+
if (serverInitiated) {
|
|
472
|
+
this.emit("closedByServer", { initiatedByEmail, switchingTo });
|
|
809
473
|
}
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
474
|
+
// Perform one last sync using the sync callback if it is set
|
|
475
|
+
if (this.syncCallback && this.isLocked) {
|
|
476
|
+
// If the lock is still valid, then we can perform one last sync
|
|
477
|
+
this.getLogger().info("[lock-service] performing one last sync before shutdown");
|
|
478
|
+
try {
|
|
479
|
+
await this.syncCallback();
|
|
480
|
+
this.getLogger().info("[lock-service] sync callback completed");
|
|
814
481
|
}
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
// session to the VS Code extension, so reporting every autosave that
|
|
818
|
-
// runs into the same held lock is a prompt per save.
|
|
819
|
-
if (!alreadyRefused) {
|
|
820
|
-
await this.reportLockConflict(error);
|
|
482
|
+
catch (error) {
|
|
483
|
+
this.getLogger().error("[lock-service] sync callback failed during shutdown, continuing with lock release", getErrorMeta(error));
|
|
821
484
|
}
|
|
822
485
|
}
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
// go ahead is whether the lock is held now, however it was come by.
|
|
826
|
-
//
|
|
827
|
-
// Arming applies to both outcomes for the same reason: a lock taken here
|
|
828
|
-
// was taken for a write, not for a client, and nothing else will ask
|
|
829
|
-
// whether it has gone idle.
|
|
830
|
-
this.ensureIdleEvaluationArmed();
|
|
831
|
-
return this.isLocked;
|
|
832
|
-
}
|
|
833
|
-
/**
|
|
834
|
-
* Flush what this client left behind and hand the lock back.
|
|
835
|
-
*
|
|
836
|
-
* The only way to give the lock up, and the whole of what this service does
|
|
837
|
-
* about a session ending. Whether the process then stops is the caller's
|
|
838
|
-
* business and this cannot tell: the same call serves a tab closing on a
|
|
839
|
-
* sandbox that keeps serving, a `superblocks dev` the user is quitting, and
|
|
840
|
-
* every exit path in `dev()`. So it says nothing about the server's fate —
|
|
841
|
-
* that used to be `shutdown()`'s job, and a lock service asserting a process
|
|
842
|
-
* was exiting is precisely the tangle that made a lock release look like a
|
|
843
|
-
* teardown.
|
|
844
|
-
*
|
|
845
|
-
* Releasing the lock and stopping the dev server used to be one action, so a
|
|
846
|
-
* client going away took the sandbox with it. On the cloud path it did not
|
|
847
|
-
* even stop — it closed the rpc client and returned, leaving a pod still
|
|
848
|
-
* bound to its port with no lock, no heartbeat and nothing behind it: healthy
|
|
849
|
-
* to every probe and unable to do anything. SABS hands such a pod straight
|
|
850
|
-
* back to the next caller, whose requests then go nowhere.
|
|
851
|
-
*
|
|
852
|
-
* Where the pod does stay up, reopening is cheap for the same reason keeping
|
|
853
|
-
* it was worth it: Vite is still running, packages are installed, files are
|
|
854
|
-
* downloaded. A returning editor reacquires through
|
|
855
|
-
* {@link restartInactivityTimer}.
|
|
856
|
-
*/
|
|
857
|
-
async relinquishLock() {
|
|
858
|
-
// Settle first, for the same reason `switchBranch` and `evaluateLockState`
|
|
859
|
-
// do. The attach-time acquisition is awaited by nobody, so an exit can land
|
|
860
|
-
// on top of one: `releaseLock` no-ops while the status is ACQUIRING, the
|
|
861
|
-
// pod leaves having released nothing, and the lock the acquire then lands
|
|
862
|
-
// is held until the server-side TTL - blocking the replacement sandbox,
|
|
863
|
-
// which is the whole reason the exit paths release promptly at all.
|
|
864
|
-
await this.settleClientLockAcquisition();
|
|
865
|
-
await this._lockOperationQueue.enqueue(() => this.relinquishLockNow());
|
|
866
|
-
}
|
|
867
|
-
async relinquishLockNow(shouldRetainLock) {
|
|
868
|
-
await this.flushPendingWork("handing the lock back");
|
|
869
|
-
if (shouldRetainLock?.()) {
|
|
870
|
-
return;
|
|
486
|
+
else {
|
|
487
|
+
this.getLogger().info("[lock-service] no sync callback because lock is not valid anymore");
|
|
871
488
|
}
|
|
872
|
-
const released = await this.releaseLock();
|
|
873
|
-
// `releaseLock` answers false without doing anything when there was no lock
|
|
874
|
-
// to give back. Saying "handed back" regardless tells oncall the
|
|
875
|
-
// server-side lock is clear when it may not be, which is the wrong
|
|
876
|
-
// direction to send someone chasing a stuck sandbox.
|
|
877
|
-
this.getLogger().info(released
|
|
878
|
-
? `[lock-service] lock handed back (appId=${this.applicationId})`
|
|
879
|
-
: `[lock-service] no lock to hand back (status=${this._status}, appId=${this.applicationId})`);
|
|
880
|
-
}
|
|
881
|
-
/**
|
|
882
|
-
* The server closed this session: tell the editor, flush, and stop
|
|
883
|
-
* heartbeating — but do not release.
|
|
884
|
-
*
|
|
885
|
-
* The lock is already gone. `/_sb_disconnect` is the server saying it has
|
|
886
|
-
* taken it, typically to hand the app to a session moving the other way
|
|
887
|
-
* across local↔cloud, so releasing from here would give back something that
|
|
888
|
-
* is not ours; on a switch it belongs to whoever is taking over. What is
|
|
889
|
-
* still ours is the heartbeat, which would otherwise keep refreshing a lock
|
|
890
|
-
* the server reassigned.
|
|
891
|
-
*
|
|
892
|
-
* The editor needs both halves of {@link EventMap.closedByServer}: it has to
|
|
893
|
-
* distinguish "the server closed you" from a socket dropping, and a switch
|
|
894
|
-
* from an ending.
|
|
895
|
-
*/
|
|
896
|
-
async notifyClosedByServer({ switchingTo, initiatedByEmail, }) {
|
|
897
|
-
this.emit("closedByServer", { initiatedByEmail, switchingTo });
|
|
898
|
-
this._closedByServer = true;
|
|
899
|
-
// Record that the lock is no longer ours before anything can act on it
|
|
900
|
-
// being missing. This is the same fact `releaseLock` records, arrived at
|
|
901
|
-
// from the other direction — there we gave it up, here it was taken — and
|
|
902
|
-
// the three places that ask are already written against it: a bare attach
|
|
903
|
-
// must not claim, a grab that misses must not reacquire, and a rejected
|
|
904
|
-
// heartbeat must not be read as losing a lock we still wanted.
|
|
905
|
-
//
|
|
906
|
-
// Without it, each of those fights the session taking over on a local↔cloud
|
|
907
|
-
// switch: the reacquire takes the app straight back off them, and the
|
|
908
|
-
// heartbeat path ends this process mid-`/_sb_disconnect`, before the
|
|
909
|
-
// response is sent. Stopping the heartbeat below is not enough on its own
|
|
910
|
-
// — a grab already in flight is not cancelled by clearing a timer.
|
|
911
|
-
this._lockGivenUp = true;
|
|
912
|
-
// Before the flush, not after it. Every refresh from here on is against
|
|
913
|
-
// something we do not hold, and the flush can take as long as the unsynced
|
|
914
|
-
// work does — which is the window one of those refreshes comes back
|
|
915
|
-
// rejected in.
|
|
916
489
|
this.stopHeartbeat();
|
|
490
|
+
this.clearStartupConnectionTimeout();
|
|
917
491
|
this.clearGatewayLivenessSweep();
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
* Both ways of giving a session up run this first, and neither can be held up
|
|
924
|
-
* by it: the caller is either handing the lock to the next client or on its
|
|
925
|
-
* way out of the process, and a sync that fails is strictly better than one
|
|
926
|
-
* that blocks the handover. An unheld lock means nothing to flush — the files
|
|
927
|
-
* belong to whoever holds it now.
|
|
928
|
-
*/
|
|
929
|
-
async flushPendingWork(reason) {
|
|
930
|
-
if (!this.syncCallback || !this.isLocked) {
|
|
931
|
-
return;
|
|
932
|
-
}
|
|
933
|
-
this.getLogger().info(`[lock-service] performing a final sync before ${reason}`);
|
|
934
|
-
// Tells {@link ensureLockForWrite} that the upload it is about to gate is
|
|
935
|
-
// this flush, which writes on the way out whether or not the lock is still
|
|
936
|
-
// ours.
|
|
937
|
-
this._flushingFinalWork = true;
|
|
938
|
-
try {
|
|
939
|
-
await this.syncCallback();
|
|
940
|
-
}
|
|
941
|
-
catch (error) {
|
|
942
|
-
this.getLogger().error(`[lock-service] sync failed before ${reason}, continuing anyway`, getErrorMeta(error));
|
|
943
|
-
}
|
|
944
|
-
finally {
|
|
945
|
-
this._flushingFinalWork = false;
|
|
492
|
+
if (this.isLocked && !serverInitiated) {
|
|
493
|
+
// Server initiated shutdown means lock is already getting released
|
|
494
|
+
this.getLogger().info("[lock-service] releasing lock before shutdown");
|
|
495
|
+
await this.releaseLock();
|
|
496
|
+
this.getLogger().info("[lock-service] lock released before shutdown");
|
|
946
497
|
}
|
|
498
|
+
this.getLogger().info("[lock-service] closing rpc client before shutdown");
|
|
499
|
+
await this._rpcClient.close();
|
|
500
|
+
this.getLogger().info("[lock-service] shutting down and exiting dev server");
|
|
947
501
|
}
|
|
948
502
|
/**
|
|
949
|
-
*
|
|
950
|
-
*
|
|
951
|
-
* The only method here that ends anything, and it exists for the four callers
|
|
952
|
-
* inside this service that have already decided the pod cannot continue: a
|
|
953
|
-
* heartbeat that failed, a reacquire lost to another user, and the two
|
|
954
|
-
* {@link LockType.LOCAL} disconnect branches, where the dev server is the
|
|
955
|
-
* user's foreground process and they expect their terminal back.
|
|
956
|
-
*
|
|
957
|
-
* Callers outside this service drive {@link relinquishLock} and own their own
|
|
958
|
-
* exit. That is not a style preference: `process.exit` is synchronous, so
|
|
959
|
-
* folding it in here makes everything a caller sequences after it dead code —
|
|
960
|
-
* which is exactly how the fatal-exit paths in `dev()` came to skip their
|
|
961
|
-
* telemetry flush on every pod that had a lock service.
|
|
962
|
-
*
|
|
963
|
-
* The release is best effort and the exit is not. A lock left held TTL-expires
|
|
964
|
-
* in two minutes, the same as any pod that dies; a process that fails to exit
|
|
965
|
-
* is a zombie until something reaps it.
|
|
503
|
+
* Shutdown the lock service and exit the dev server
|
|
966
504
|
*/
|
|
967
505
|
async shutdownAndExit() {
|
|
968
506
|
try {
|
|
969
|
-
await this.
|
|
507
|
+
await this.shutdown({ serverInitiated: false });
|
|
970
508
|
}
|
|
971
509
|
catch (error) {
|
|
972
|
-
this.getLogger().error("[lock-service] failed to
|
|
510
|
+
this.getLogger().error("[lock-service] Lock service failed to gracefully shutdown and will forcefully exit", getErrorMeta(error));
|
|
973
511
|
}
|
|
974
512
|
finally {
|
|
975
513
|
process.exit(1);
|
|
@@ -981,24 +519,15 @@ export class LockService extends TracedEventEmitter {
|
|
|
981
519
|
addConnectedPeerIdToUser(userId, peerId) {
|
|
982
520
|
this._activityTracker.addConnectedPeerIdToUser(userId, peerId);
|
|
983
521
|
this._isRestarting = false;
|
|
984
|
-
this.
|
|
522
|
+
this.clearStartupConnectionTimeout();
|
|
985
523
|
if (this._bufferPeriodTimeout) {
|
|
986
524
|
clearTimeout(this._bufferPeriodTimeout);
|
|
987
525
|
this._bufferPeriodTimeout = undefined;
|
|
988
526
|
this.getLogger().info("[lock-service] user reconnected, cleared pending buffer period timeout");
|
|
989
527
|
}
|
|
990
|
-
this.
|
|
528
|
+
this.armIdleEvaluationAfterReconnectIfNeeded();
|
|
991
529
|
}
|
|
992
|
-
|
|
993
|
-
* Make sure something is still due to ask whether this lock is idle.
|
|
994
|
-
*
|
|
995
|
-
* The inactivity timer is one-shot, and the evaluation it fires returns
|
|
996
|
-
* early on a service that has never held a lock without arming another. So
|
|
997
|
-
* any path that ends up holding one for something other than an attached
|
|
998
|
-
* client has to put the question back on the clock, or the lock is refreshed
|
|
999
|
-
* by the heartbeat for a client that never arrives.
|
|
1000
|
-
*/
|
|
1001
|
-
ensureIdleEvaluationArmed() {
|
|
530
|
+
armIdleEvaluationAfterReconnectIfNeeded() {
|
|
1002
531
|
const alreadyConcludedIdle = this._activityTracker.isReadyForTakeover || this._heartbeatStoppedForIdle;
|
|
1003
532
|
if (this._inactivityTimeout || alreadyConcludedIdle) {
|
|
1004
533
|
return;
|
|
@@ -1041,15 +570,7 @@ export class LockService extends TracedEventEmitter {
|
|
|
1041
570
|
// grab was in flight, a failure here is moot — don't shut the pod down out
|
|
1042
571
|
// from under a user who may still be connected (the lock just TTL-expires).
|
|
1043
572
|
// A genuine heartbeat failure (not an idle stop) still shuts down as before.
|
|
1044
|
-
|
|
1045
|
-
// Same for a lock we handed back on purpose. Acquiring fires a heartbeat
|
|
1046
|
-
// immediately, so on the attach-and-drop path a grab is always still out
|
|
1047
|
-
// when the release lands; reading its failure as "we lost a lock we still
|
|
1048
|
-
// wanted" would take a cloud sandbox down for a lock it had already given
|
|
1049
|
-
// up. Neither the status nor the heartbeat generation can be used to tell
|
|
1050
|
-
// the two apart here — `checkAndUpdateLockIfPossible` resets both while
|
|
1051
|
-
// cleaning up after itself, before the error ever reaches this catch.
|
|
1052
|
-
if (this._heartbeatStoppedForIdle || this._lockGivenUp) {
|
|
573
|
+
if (this._heartbeatStoppedForIdle) {
|
|
1053
574
|
return;
|
|
1054
575
|
}
|
|
1055
576
|
this.getLogger().error("[lock-service] Error sending heartbeat, shutting down", getErrorMeta(error));
|
|
@@ -1082,44 +603,58 @@ export class LockService extends TracedEventEmitter {
|
|
|
1082
603
|
this.getLogger().info("[lock-service] heartbeat stopped");
|
|
1083
604
|
}
|
|
1084
605
|
}
|
|
606
|
+
clearStartupConnectionTimeout() {
|
|
607
|
+
if (this._startupConnectionTimeout) {
|
|
608
|
+
clearTimeout(this._startupConnectionTimeout);
|
|
609
|
+
this._startupConnectionTimeout = undefined;
|
|
610
|
+
}
|
|
611
|
+
this._startupConnectionTimeoutExpired = false;
|
|
612
|
+
this._startupConnectionTimeoutStartedAt = undefined;
|
|
613
|
+
}
|
|
614
|
+
startStartupConnectionTimeoutIfNeeded() {
|
|
615
|
+
if (this._startupConnectionTimeoutStartedAt !== undefined) {
|
|
616
|
+
return;
|
|
617
|
+
}
|
|
618
|
+
this.clearStartupConnectionTimeout();
|
|
619
|
+
this._startupConnectionTimeoutStartedAt = Date.now();
|
|
620
|
+
// info (not debug) so oncall can confirm the watchdog armed when
|
|
621
|
+
// reconstructing a fire incident; fires once per lock acquisition.
|
|
622
|
+
this.getLogger().info(`[lock-service] starting initial connection timeout window (${STARTUP_CONNECTION_TIMEOUT_MS}ms)`);
|
|
623
|
+
this._startupConnectionTimeout = setTimeout(() => {
|
|
624
|
+
this._startupConnectionTimeout = undefined;
|
|
625
|
+
this._startupConnectionTimeoutExpired = true;
|
|
626
|
+
this.evaluateStartupConnectionTimeoutIfNeeded();
|
|
627
|
+
}, STARTUP_CONNECTION_TIMEOUT_MS);
|
|
628
|
+
this._startupConnectionTimeout.unref();
|
|
629
|
+
}
|
|
630
|
+
evaluateStartupConnectionTimeoutIfNeeded() {
|
|
631
|
+
if (!this._startupConnectionTimeoutExpired) {
|
|
632
|
+
return;
|
|
633
|
+
}
|
|
634
|
+
if (!this.isLocked) {
|
|
635
|
+
return;
|
|
636
|
+
}
|
|
637
|
+
if (this._activityTracker.connectedUsers.length > 0) {
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
640
|
+
if (this.hasAttachedGatewayPeer) {
|
|
641
|
+
return;
|
|
642
|
+
}
|
|
643
|
+
this.getLogger().warn(`[lock-service] no socket connection detected within ${STARTUP_CONNECTION_TIMEOUT_MS}ms after initial lock acquisition, marking session inactive (lockId=${this._lockId}, appId=${this.applicationId})`);
|
|
644
|
+
this._activityTracker.markDisconnectedDueToStartupTimeout();
|
|
645
|
+
void this.evaluateLockState().catch((error) => {
|
|
646
|
+
this.getLogger().error(`[lock-service] failed to evaluate lock state after startup connection timeout (lockId=${this._lockId}, appId=${this.applicationId})`, getErrorMeta(error));
|
|
647
|
+
});
|
|
648
|
+
}
|
|
1085
649
|
/**
|
|
1086
650
|
* Evaluate if the lock should be released (user disconnected) or marked as ready for takeover (user inactive but connected)
|
|
1087
651
|
*/
|
|
1088
652
|
async evaluateLockState() {
|
|
1089
|
-
// Single-flight. Several unrelated things ask for an evaluation — a client
|
|
1090
|
-
// disconnecting, the last gateway peer detaching, the inactivity timer —
|
|
1091
|
-
// and the body awaits twice before it decides anything (the acquisition,
|
|
1092
|
-
// then any running generation). The `_bufferPeriodTimeout` guard below is
|
|
1093
|
-
// read before those awaits, so two overlapping callers both saw no timer
|
|
1094
|
-
// and both armed one; the second arming overwrote the first handle, leaving
|
|
1095
|
-
// a timer nothing could cancel. Joining the in-flight evaluation is what
|
|
1096
|
-
// makes the guard mean what it says, and it holds for any await added to
|
|
1097
|
-
// the body later.
|
|
1098
|
-
this._inFlightLockEvaluation ??= this.evaluateLockStateOnce().finally(() => {
|
|
1099
|
-
this._inFlightLockEvaluation = undefined;
|
|
1100
|
-
});
|
|
1101
|
-
return this._inFlightLockEvaluation;
|
|
1102
|
-
}
|
|
1103
|
-
async evaluateLockStateOnce() {
|
|
1104
|
-
// A client can attach and drop again before its lock arrives — the flaky
|
|
1105
|
-
// connect this whole change is downstream of. Deciding now would find
|
|
1106
|
-
// `isLocked` false, release nothing, and leave the lock to land on an empty
|
|
1107
|
-
// sandbox and heartbeat there for nobody.
|
|
1108
|
-
await this.settleClientLockAcquisition();
|
|
1109
653
|
// Skip if already evaluating lock release (buffer timeout exists)
|
|
1110
654
|
if (this._bufferPeriodTimeout) {
|
|
1111
655
|
this.getLogger().info("[lock-service] lock release already in progress, skipping duplicate evaluation");
|
|
1112
656
|
return;
|
|
1113
657
|
}
|
|
1114
|
-
// Nothing to evaluate before a client has ever claimed the lock. The
|
|
1115
|
-
// inactivity timer is armed from the constructor, so on the cloud path it
|
|
1116
|
-
// now elapses on a warm, unclaimed sandbox — that is an empty sandbox, not
|
|
1117
|
-
// a lock going stale. A client that came and went is DISCONNECTED rather
|
|
1118
|
-
// than untouched, and still needs the release path below.
|
|
1119
|
-
if (this._status === LockServiceStatus.IDLE &&
|
|
1120
|
-
!this._activityTracker.isDisconnected) {
|
|
1121
|
-
return;
|
|
1122
|
-
}
|
|
1123
658
|
// 1. Wait for Clark generation to complete if it's generating
|
|
1124
659
|
if (this.waitForGenerationCompleteCallback) {
|
|
1125
660
|
this.getLogger().debug("[lock-service] waiting for Clark generation to complete before evaluating lock state");
|
|
@@ -1141,6 +676,7 @@ export class LockService extends TracedEventEmitter {
|
|
|
1141
676
|
return;
|
|
1142
677
|
}
|
|
1143
678
|
const isUserDisconnected = this._activityTracker.isDisconnected;
|
|
679
|
+
const isAwaitingFirstEditor = this._activityTracker.connectedUsers.length === 0;
|
|
1144
680
|
// Case 1: User is disconnected - release the lock
|
|
1145
681
|
if (isUserDisconnected) {
|
|
1146
682
|
if (this._activityTracker.wasRecentlyActive) {
|
|
@@ -1152,7 +688,7 @@ export class LockService extends TracedEventEmitter {
|
|
|
1152
688
|
void this.shutdownAndExit();
|
|
1153
689
|
}
|
|
1154
690
|
else {
|
|
1155
|
-
void this.
|
|
691
|
+
void this.shutdown({ serverInitiated: false });
|
|
1156
692
|
}
|
|
1157
693
|
}, this._lockBufferMs);
|
|
1158
694
|
this._bufferPeriodTimeout.unref();
|
|
@@ -1164,31 +700,22 @@ export class LockService extends TracedEventEmitter {
|
|
|
1164
700
|
await this.shutdownAndExit();
|
|
1165
701
|
}
|
|
1166
702
|
else {
|
|
1167
|
-
await this.
|
|
703
|
+
await this.shutdown({ serverInitiated: false });
|
|
1168
704
|
}
|
|
1169
705
|
}
|
|
1170
706
|
}
|
|
707
|
+
else if (isAwaitingFirstEditor) {
|
|
708
|
+
this.getLogger().debug("[lock-service] no editor connected yet, leaving the lock to the startup watchdog");
|
|
709
|
+
}
|
|
1171
710
|
else {
|
|
1172
|
-
// Case 2:
|
|
1173
|
-
//
|
|
1174
|
-
//
|
|
1175
|
-
//
|
|
1176
|
-
//
|
|
1177
|
-
//
|
|
1178
|
-
//
|
|
1179
|
-
|
|
1180
|
-
// it. restartInactivityTimer() resumes the heartbeat if this user becomes
|
|
1181
|
-
// active again before the lock expires.
|
|
1182
|
-
// The two cases reach the same handling but are very different
|
|
1183
|
-
// incidents, so the line names which one it is. A lock no client ever
|
|
1184
|
-
// claimed is the state the deleted startup watchdog used to report with
|
|
1185
|
-
// its own distinctive line; without a field here, oncall reconstructing a
|
|
1186
|
-
// stuck-lock incident cannot tell "nobody ever attached" from "someone
|
|
1187
|
-
// stepped away", and both read as routine idleness.
|
|
1188
|
-
const reason = this.hasAttachedClient
|
|
1189
|
-
? "connected client went quiet"
|
|
1190
|
-
: "never claimed by a client";
|
|
1191
|
-
this.getLogger().info(`[lock-service] lock idle (${reason}), marking ready for takeover and stopping heartbeat so it can expire (lockId=${this._lockId}, appId=${this.applicationId})`);
|
|
711
|
+
// Case 2: User is still connected but inactive - mark the lock ready for
|
|
712
|
+
// takeover AND stop the heartbeat. The pod heartbeat is the ONLY thing
|
|
713
|
+
// refreshing the server-side AppBranchLock; if we keep heartbeating while
|
|
714
|
+
// the user is idle, the lock never TTL-expires and no other user can take
|
|
715
|
+
// over the app (APPS-4840). Stopping it lets the lock expire so another
|
|
716
|
+
// user can acquire it normally. restartInactivityTimer() resumes the
|
|
717
|
+
// heartbeat if this user becomes active again before the lock expires.
|
|
718
|
+
this.getLogger().info(`[lock-service] user inactive but still connected, marking lock ready for takeover and stopping heartbeat so the lock can expire (lockId=${this._lockId}, appId=${this.applicationId})`);
|
|
1192
719
|
this._activityTracker.markReadyForTakeover();
|
|
1193
720
|
this._heartbeatStoppedForIdle = true;
|
|
1194
721
|
this.stopHeartbeat();
|
|
@@ -1227,7 +754,10 @@ export class LockService extends TracedEventEmitter {
|
|
|
1227
754
|
this.getLogger().info(`[lock-service] user active again, resuming lock heartbeat (lockId=${this._lockId}, appId=${this.applicationId})`);
|
|
1228
755
|
void this.scheduleNextHeartbeat();
|
|
1229
756
|
}
|
|
1230
|
-
this.
|
|
757
|
+
if (this.canReacquireLock) {
|
|
758
|
+
this.getLogger().info("[lock-service] user active again with no lock held, reacquiring");
|
|
759
|
+
void this.acquireLock();
|
|
760
|
+
}
|
|
1231
761
|
this.scheduleInactivityEvaluation();
|
|
1232
762
|
}
|
|
1233
763
|
scheduleInactivityEvaluation() {
|
|
@@ -1249,10 +779,11 @@ export class LockService extends TracedEventEmitter {
|
|
|
1249
779
|
*
|
|
1250
780
|
* Gateway peers are deliberately not connected-user presence — they are not
|
|
1251
781
|
* a browser editor and must not keep an editor's lock alive — so they need
|
|
1252
|
-
* their own liveness signal. Without it the
|
|
1253
|
-
* sandbox out from under a headless build
|
|
1254
|
-
* per peer id so a reattach that closes the
|
|
1255
|
-
* the sandbox back while its successor is
|
|
782
|
+
* their own liveness signal. Without it the startup watchdog and the
|
|
783
|
+
* inactivity reaper would take the sandbox out from under a headless build
|
|
784
|
+
* that nobody is watching. Tracked per peer id so a reattach that closes the
|
|
785
|
+
* socket it replaced does not hand the sandbox back while its successor is
|
|
786
|
+
* still working.
|
|
1256
787
|
*/
|
|
1257
788
|
markGatewayAttached(peerId) {
|
|
1258
789
|
const isNew = !this._gatewayLivenessByPeerId.has(peerId);
|
|
@@ -1264,10 +795,6 @@ export class LockService extends TracedEventEmitter {
|
|
|
1264
795
|
}
|
|
1265
796
|
this.getLogger().info(`[lock-service] gateway peer attached (peerId=${peerId}, appId=${this.applicationId})`);
|
|
1266
797
|
this.armGatewayLivenessSweep();
|
|
1267
|
-
// No lock is taken here. A gateway peer being attached says only that a
|
|
1268
|
-
// socket is open; {@link acquireLockForTurn} takes write access when there
|
|
1269
|
-
// is actually a turn to write for, which is what keeps the app openable
|
|
1270
|
-
// between prompts.
|
|
1271
798
|
void this.restartInactivityTimer();
|
|
1272
799
|
}
|
|
1273
800
|
/**
|
|
@@ -1279,72 +806,20 @@ export class LockService extends TracedEventEmitter {
|
|
|
1279
806
|
return;
|
|
1280
807
|
}
|
|
1281
808
|
this.getLogger().info(`[lock-service] gateway peer detached (peerId=${peerId}, appId=${this.applicationId})`);
|
|
1282
|
-
if (this._gatewayLivenessByPeerId.size
|
|
1283
|
-
|
|
1284
|
-
}
|
|
1285
|
-
this.clearGatewayLivenessSweep();
|
|
1286
|
-
this.handleLastGatewayPeerGone();
|
|
1287
|
-
}
|
|
1288
|
-
/**
|
|
1289
|
-
* The last gateway peer is gone, so no turn is running.
|
|
1290
|
-
*
|
|
1291
|
-
* Write access was taken for the turn, so it goes back now rather than being
|
|
1292
|
-
* left to TTL out — that would leave the app unopenable for up to the lock's
|
|
1293
|
-
* two-minute lifetime after a headless build had already finished. This is
|
|
1294
|
-
* also the backstop for a gateway that dies mid-turn.
|
|
1295
|
-
*
|
|
1296
|
-
* Deliberately not a shutdown: the gateway drops its socket after every turn
|
|
1297
|
-
* and opens a new one for the next prompt, and in between the pod keeps
|
|
1298
|
-
* serving. The next turn takes the lock again through
|
|
1299
|
-
* {@link acquireLockForTurn}. What bounds the pod's life is SABS's live-edit
|
|
1300
|
-
* expiry, not this.
|
|
1301
|
-
*
|
|
1302
|
-
* Only when no editor is attached. An editor disconnect releases on its own
|
|
1303
|
-
* buffer in {@link evaluateLockState}, and a second releaser over the top of
|
|
1304
|
-
* that would race it.
|
|
1305
|
-
*/
|
|
1306
|
-
handleLastGatewayPeerGone() {
|
|
1307
|
-
if (this._activityTracker.connectedUsers.length > 0) {
|
|
1308
|
-
void this.evaluateLockState().catch((error) => {
|
|
1309
|
-
this.getLogger().error(`[lock-service] failed to evaluate lock state after the last gateway peer detached (lockId=${this._lockId}, appId=${this.applicationId})`, getErrorMeta(error));
|
|
1310
|
-
});
|
|
1311
|
-
return;
|
|
809
|
+
if (this._gatewayLivenessByPeerId.size === 0) {
|
|
810
|
+
this.clearGatewayLivenessSweep();
|
|
1312
811
|
}
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
}
|
|
1317
|
-
/**
|
|
1318
|
-
* Wait for the turn to finish, then hand its lock back.
|
|
1319
|
-
*
|
|
1320
|
-
* The wait sits outside the lock queue, the way {@link evaluateLockState}
|
|
1321
|
-
* does it. That queue serialises the handback itself, branch switches,
|
|
1322
|
-
* `isLockedWhenSettled` and the next turn's acquisition, so waiting inside
|
|
1323
|
-
* it lets a single hung generation stall all of them — including a graceful
|
|
1324
|
-
* shutdown, which awaits the handback with no timeout of its own.
|
|
1325
|
-
*/
|
|
1326
|
-
async handBackTurnLockWhenGenerationSettles() {
|
|
1327
|
-
await this.waitForGenerationCompleteCallback?.();
|
|
1328
|
-
await this._lockOperationQueue.enqueue(async () => {
|
|
1329
|
-
if (this.hasAttachedGatewayPeer ||
|
|
1330
|
-
this._activityTracker.connectedUsers.length > 0) {
|
|
1331
|
-
return;
|
|
1332
|
-
}
|
|
1333
|
-
if (!this.isLocked) {
|
|
1334
|
-
return;
|
|
1335
|
-
}
|
|
1336
|
-
this.getLogger().info(`[lock-service] no gateway peer left, handing the turn lock back and staying up (lockId=${this._lockId}, appId=${this.applicationId})`);
|
|
1337
|
-
await this.relinquishLockNow(() => this.hasAttachedGatewayPeer ||
|
|
1338
|
-
this._activityTracker.connectedUsers.length > 0);
|
|
1339
|
-
});
|
|
812
|
+
// A window that expired while a gateway held the sandbox was deferred, not
|
|
813
|
+
// cancelled; re-evaluate now that the last one is gone.
|
|
814
|
+
this.evaluateStartupConnectionTimeoutIfNeeded();
|
|
1340
815
|
}
|
|
1341
816
|
/**
|
|
1342
817
|
* Watches for the last gateway signal lapsing.
|
|
1343
818
|
*
|
|
1344
|
-
* The
|
|
1345
|
-
* re-examined when something happens. A silent gateway
|
|
1346
|
-
* anything happening, so this is what notices it and puts
|
|
1347
|
-
* for evaluation.
|
|
819
|
+
* The windows a gateway defers (startup watchdog, inactivity reaper) fire
|
|
820
|
+
* once and are then only re-examined when something happens. A silent gateway
|
|
821
|
+
* is the absence of anything happening, so this is what notices it and puts
|
|
822
|
+
* the lock back up for evaluation.
|
|
1348
823
|
*/
|
|
1349
824
|
armGatewayLivenessSweep() {
|
|
1350
825
|
if (this._gatewayLivenessSweep) {
|
|
@@ -1355,9 +830,10 @@ export class LockService extends TracedEventEmitter {
|
|
|
1355
830
|
return;
|
|
1356
831
|
}
|
|
1357
832
|
this.clearGatewayLivenessSweep();
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
833
|
+
this.evaluateStartupConnectionTimeoutIfNeeded();
|
|
834
|
+
void this.evaluateLockState().catch((error) => {
|
|
835
|
+
this.getLogger().error(`[lock-service] failed to evaluate lock state after gateway liveness lapsed (lockId=${this._lockId}, appId=${this.applicationId})`, getErrorMeta(error));
|
|
836
|
+
});
|
|
1361
837
|
}, Math.max(1, Math.floor(this._gatewayLivenessTtlMs / 2)));
|
|
1362
838
|
this._gatewayLivenessSweep.unref?.();
|
|
1363
839
|
}
|