@superblocksteam/vite-plugin-file-sync 2.0.155 → 2.0.156-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-service/agent/prompts/build-base-system-prompt.d.ts +9 -2
- package/dist/ai-service/agent/prompts/build-base-system-prompt.d.ts.map +1 -1
- package/dist/ai-service/agent/prompts/build-base-system-prompt.js +34 -6
- package/dist/ai-service/agent/prompts/build-base-system-prompt.js.map +1 -1
- package/dist/ai-service/agent/prompts/build-security-scan-prompt.d.ts +2 -1
- package/dist/ai-service/agent/prompts/build-security-scan-prompt.d.ts.map +1 -1
- package/dist/ai-service/agent/prompts/build-security-scan-prompt.js +52 -9
- package/dist/ai-service/agent/prompts/build-security-scan-prompt.js.map +1 -1
- package/dist/ai-service/agent/tool-permission-delivery-metrics.d.ts +3 -0
- package/dist/ai-service/agent/tool-permission-delivery-metrics.d.ts.map +1 -0
- package/dist/ai-service/agent/tool-permission-delivery-metrics.js +23 -0
- package/dist/ai-service/agent/tool-permission-delivery-metrics.js.map +1 -0
- package/dist/ai-service/agent/tool-permission-delivery.d.ts +25 -16
- package/dist/ai-service/agent/tool-permission-delivery.d.ts.map +1 -1
- package/dist/ai-service/agent/tool-permission-delivery.js +384 -38
- package/dist/ai-service/agent/tool-permission-delivery.js.map +1 -1
- package/dist/ai-service/agent/tools/apis/get-sdk-api-docs.d.ts +9 -35
- package/dist/ai-service/agent/tools/apis/get-sdk-api-docs.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/apis/get-sdk-api-docs.js +71 -98
- package/dist/ai-service/agent/tools/apis/get-sdk-api-docs.js.map +1 -1
- package/dist/ai-service/agent/tools/apis/test-api.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/apis/test-api.js +3 -2
- package/dist/ai-service/agent/tools/apis/test-api.js.map +1 -1
- package/dist/ai-service/agent/tools/app-environment/update-app-environment.d.ts +1 -1
- package/dist/ai-service/agent/tools/app-environment/update-app-environment.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/app-environment/update-app-environment.js +13 -1
- package/dist/ai-service/agent/tools/app-environment/update-app-environment.js.map +1 -1
- package/dist/ai-service/agent/tools/build-capture-screenshot.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/build-capture-screenshot.js +8 -3
- package/dist/ai-service/agent/tools/build-capture-screenshot.js.map +1 -1
- package/dist/ai-service/agent/tools/build-navigate-preview.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/build-navigate-preview.js +4 -2
- package/dist/ai-service/agent/tools/build-navigate-preview.js.map +1 -1
- package/dist/ai-service/agent/tools/build-reload-file.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/build-reload-file.js +4 -2
- package/dist/ai-service/agent/tools/build-reload-file.js.map +1 -1
- package/dist/ai-service/agent/tools/get-logs.d.ts +2 -2
- package/dist/ai-service/agent/tools/integrations/delete-integration.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/integrations/delete-integration.js +6 -3
- package/dist/ai-service/agent/tools/integrations/delete-integration.js.map +1 -1
- package/dist/ai-service/agent/tools/integrations/execute-request.d.ts +13 -13
- package/dist/ai-service/agent/tools/integrations/integration-setup.d.ts +18 -6
- package/dist/ai-service/agent/tools/integrations/integration-setup.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/integrations/integration-setup.js +41 -15
- package/dist/ai-service/agent/tools/integrations/integration-setup.js.map +1 -1
- package/dist/ai-service/agent/tools/integrations/internal.d.ts.map +1 -1
- package/dist/ai-service/agent/tools/integrations/internal.js +4 -2
- package/dist/ai-service/agent/tools/integrations/internal.js.map +1 -1
- package/dist/ai-service/agent/tools/report-security-findings.d.ts +23 -23
- package/dist/ai-service/agent/tools.d.ts +4 -2
- package/dist/ai-service/agent/tools.d.ts.map +1 -1
- package/dist/ai-service/agent/tools.js +21 -14
- package/dist/ai-service/agent/tools.js.map +1 -1
- package/dist/ai-service/agent/tools2/tools/git.d.ts +2 -2
- package/dist/ai-service/index.d.ts +26 -25
- package/dist/ai-service/index.d.ts.map +1 -1
- package/dist/ai-service/index.js +340 -121
- package/dist/ai-service/index.js.map +1 -1
- package/dist/ai-service/integrations/store.d.ts +2 -1
- package/dist/ai-service/integrations/store.d.ts.map +1 -1
- package/dist/ai-service/integrations/store.js +3 -2
- package/dist/ai-service/integrations/store.js.map +1 -1
- package/dist/ai-service/judge/tools/playwright-action.d.ts +1 -1
- package/dist/ai-service/judge/tools/submit-feedback.d.ts +1 -1
- package/dist/ai-service/llm/client.d.ts +9 -3
- package/dist/ai-service/llm/client.d.ts.map +1 -1
- package/dist/ai-service/llm/client.js +10 -3
- package/dist/ai-service/llm/client.js.map +1 -1
- package/dist/ai-service/llm/context-v2/context-metrics.d.ts.map +1 -1
- package/dist/ai-service/llm/context-v2/context-metrics.js +3 -0
- package/dist/ai-service/llm/context-v2/context-metrics.js.map +1 -1
- package/dist/ai-service/llm/stream/observers/llmobs.d.ts +6 -6
- package/dist/ai-service/llm/stream/observers/llmobs.d.ts.map +1 -1
- package/dist/ai-service/llm/stream/observers/llmobs.js +10 -9
- package/dist/ai-service/llm/stream/observers/llmobs.js.map +1 -1
- package/dist/ai-service/llm/stream/observers/step-token-usage.d.ts +15 -0
- package/dist/ai-service/llm/stream/observers/step-token-usage.d.ts.map +1 -0
- package/dist/ai-service/llm/stream/observers/step-token-usage.js +88 -0
- package/dist/ai-service/llm/stream/observers/step-token-usage.js.map +1 -0
- package/dist/ai-service/llm/stream/orchestrator.d.ts +4 -0
- package/dist/ai-service/llm/stream/orchestrator.d.ts.map +1 -1
- package/dist/ai-service/llm/stream/orchestrator.js +19 -41
- package/dist/ai-service/llm/stream/orchestrator.js.map +1 -1
- package/dist/ai-service/llm/stream/retry-engine.d.ts +1 -1
- package/dist/ai-service/llm/stream/retry-engine.d.ts.map +1 -1
- package/dist/ai-service/llm/stream/retry-engine.js +26 -1
- package/dist/ai-service/llm/stream/retry-engine.js.map +1 -1
- package/dist/ai-service/llm/stream/terminal-reason.d.ts +6 -0
- package/dist/ai-service/llm/stream/terminal-reason.d.ts.map +1 -0
- package/dist/ai-service/llm/stream/terminal-reason.js +73 -0
- package/dist/ai-service/llm/stream/terminal-reason.js.map +1 -0
- package/dist/ai-service/policy-agent-metrics.d.ts +10 -0
- package/dist/ai-service/policy-agent-metrics.d.ts.map +1 -1
- package/dist/ai-service/policy-agent-metrics.js +33 -0
- package/dist/ai-service/policy-agent-metrics.js.map +1 -1
- package/dist/ai-service/state-machine/clark-fsm.d.ts +29 -1
- package/dist/ai-service/state-machine/clark-fsm.d.ts.map +1 -1
- package/dist/ai-service/state-machine/clark-fsm.js +3 -1
- package/dist/ai-service/state-machine/clark-fsm.js.map +1 -1
- package/dist/ai-service/state-machine/handlers/agent-planning.d.ts.map +1 -1
- package/dist/ai-service/state-machine/handlers/agent-planning.js +7 -6
- package/dist/ai-service/state-machine/handlers/agent-planning.js.map +1 -1
- package/dist/ai-service/state-machine/handlers/llm-generating.d.ts.map +1 -1
- package/dist/ai-service/state-machine/handlers/llm-generating.js +18 -5
- package/dist/ai-service/state-machine/handlers/llm-generating.js.map +1 -1
- package/dist/ai-service/state-machine/helpers/fetch-with-reconnect-retry.d.ts.map +1 -1
- package/dist/ai-service/state-machine/helpers/fetch-with-reconnect-retry.js +8 -6
- package/dist/ai-service/state-machine/helpers/fetch-with-reconnect-retry.js.map +1 -1
- package/dist/ai-service/state-machine/helpers/policy-gate-usage-metrics.d.ts +0 -14
- package/dist/ai-service/state-machine/helpers/policy-gate-usage-metrics.d.ts.map +1 -1
- package/dist/ai-service/state-machine/helpers/policy-gate-usage-metrics.js +0 -36
- package/dist/ai-service/state-machine/helpers/policy-gate-usage-metrics.js.map +1 -1
- package/dist/ai-service/state-machine/helpers/pre-session-metrics.d.ts +2 -0
- package/dist/ai-service/state-machine/helpers/pre-session-metrics.d.ts.map +1 -1
- package/dist/ai-service/state-machine/helpers/pre-session-metrics.js +3 -0
- package/dist/ai-service/state-machine/helpers/pre-session-metrics.js.map +1 -1
- package/dist/ai-service/state-machine/helpers/prompt-degradation-metrics.d.ts +28 -0
- package/dist/ai-service/state-machine/helpers/prompt-degradation-metrics.d.ts.map +1 -0
- package/dist/ai-service/state-machine/helpers/prompt-degradation-metrics.js +254 -0
- package/dist/ai-service/state-machine/helpers/prompt-degradation-metrics.js.map +1 -0
- package/dist/ai-service/state-machine/helpers/prompt-request-metrics.d.ts +1 -0
- package/dist/ai-service/state-machine/helpers/prompt-request-metrics.d.ts.map +1 -1
- package/dist/ai-service/state-machine/helpers/prompt-request-metrics.js +60 -27
- package/dist/ai-service/state-machine/helpers/prompt-request-metrics.js.map +1 -1
- package/dist/ai-service/state-machine/helpers/stable-peer.d.ts +12 -1
- package/dist/ai-service/state-machine/helpers/stable-peer.d.ts.map +1 -1
- package/dist/ai-service/state-machine/helpers/stable-peer.js +42 -9
- package/dist/ai-service/state-machine/helpers/stable-peer.js.map +1 -1
- package/dist/ai-service/state-machine/mocks.d.ts.map +1 -1
- package/dist/ai-service/state-machine/mocks.js +6 -0
- package/dist/ai-service/state-machine/mocks.js.map +1 -1
- package/dist/ai-service/util/call-peer.d.ts +11 -0
- package/dist/ai-service/util/call-peer.d.ts.map +1 -0
- package/dist/ai-service/util/call-peer.js +47 -0
- package/dist/ai-service/util/call-peer.js.map +1 -0
- package/dist/ai-service/util/peer-liveness.d.ts +23 -0
- package/dist/ai-service/util/peer-liveness.d.ts.map +1 -0
- package/dist/ai-service/util/peer-liveness.js +42 -0
- package/dist/ai-service/util/peer-liveness.js.map +1 -0
- package/dist/ai-service/util/rpc-timeout.d.ts +12 -5
- package/dist/ai-service/util/rpc-timeout.d.ts.map +1 -1
- package/dist/ai-service/util/rpc-timeout.js +12 -5
- package/dist/ai-service/util/rpc-timeout.js.map +1 -1
- package/dist/file-sync-vite-plugin.d.ts.map +1 -1
- package/dist/file-sync-vite-plugin.js +8 -21
- package/dist/file-sync-vite-plugin.js.map +1 -1
- package/dist/lock-service/activity-tracker.d.ts +0 -5
- package/dist/lock-service/activity-tracker.d.ts.map +1 -1
- package/dist/lock-service/activity-tracker.js +0 -13
- package/dist/lock-service/activity-tracker.js.map +1 -1
- package/dist/lock-service/index.d.ts +308 -27
- package/dist/lock-service/index.d.ts.map +1 -1
- package/dist/lock-service/index.js +680 -156
- package/dist/lock-service/index.js.map +1 -1
- package/dist/policy-gate-callback-mapper.d.ts +3 -2
- package/dist/policy-gate-callback-mapper.d.ts.map +1 -1
- package/dist/policy-gate-callback-mapper.js.map +1 -1
- package/dist/policy-gate-runner.d.ts +4 -2
- package/dist/policy-gate-runner.d.ts.map +1 -1
- package/dist/policy-gate-runner.js +31 -8
- package/dist/policy-gate-runner.js.map +1 -1
- package/dist/scan-scope-merge.d.ts +16 -0
- package/dist/scan-scope-merge.d.ts.map +1 -0
- package/dist/scan-scope-merge.js +90 -0
- package/dist/scan-scope-merge.js.map +1 -0
- package/dist/socket-manager.d.ts +47 -6
- package/dist/socket-manager.d.ts.map +1 -1
- package/dist/socket-manager.js +283 -66
- package/dist/socket-manager.js.map +1 -1
- package/dist/sync-service/index.d.ts.map +1 -1
- package/dist/sync-service/index.js +3 -3
- package/dist/sync-service/index.js.map +1 -1
- package/dist/test-setup.d.ts +2 -0
- package/dist/test-setup.d.ts.map +1 -0
- package/dist/test-setup.js +21 -0
- package/dist/test-setup.js.map +1 -0
- package/package.json +8 -8
|
@@ -2,18 +2,10 @@ import { addTracingToMethods, ConflictError, NotFoundError, TracedEventEmitter,
|
|
|
2
2
|
import { unwrapResponseDto } from "@superblocksteam/shared";
|
|
3
3
|
import { applyErrorHandling, } from "../errors/error-handler.js";
|
|
4
4
|
import { getErrorMeta } from "../util/logger.js";
|
|
5
|
+
import { OperationQueue } from "../util/operation-queue.js";
|
|
5
6
|
import { ActivityTracker } from "./activity-tracker.js";
|
|
6
7
|
const LOCK_HEARTBEAT_INTERVAL_MS = 30 * 1000; // 30 seconds
|
|
7
8
|
const LOCK_BUFFER_MS = 30 * 1000; // 30 seconds
|
|
8
|
-
// Startup no-socket watchdog: if a freshly-acquired lock never sees an editor
|
|
9
|
-
// socket attach, the pod is reclaimed so an abandoned startup can't hold the
|
|
10
|
-
// exclusive per-app-branch lock forever. The window must cover the entire
|
|
11
|
-
// startup — DBFS download + package install, then socket attach — so it sits
|
|
12
|
-
// well above worst-case hydration (~90s observed) instead of racing app size
|
|
13
|
-
// the way the old 90s window did and bricking slow-hydrating apps. Trade-off:
|
|
14
|
-
// an abandoned-but-hydrated pod is reclaimed up to this long rather than
|
|
15
|
-
// sooner; that case is bounded and force-takeover is available.
|
|
16
|
-
const STARTUP_CONNECTION_TIMEOUT_MS = 4 * 60 * 1000; // 4 minutes
|
|
17
9
|
/**
|
|
18
10
|
* How long a gateway peer's liveness signal counts for before it has to be
|
|
19
11
|
* renewed. A gateway refreshes it on every authorized call and, when it has
|
|
@@ -49,6 +41,25 @@ export var LockType;
|
|
|
49
41
|
LockType["LOCAL"] = "local";
|
|
50
42
|
LockType["CSB"] = "csb";
|
|
51
43
|
})(LockType || (LockType = {}));
|
|
44
|
+
/**
|
|
45
|
+
* The methods {@link LockService} reports as spans.
|
|
46
|
+
*
|
|
47
|
+
* `addTracingToMethods` matches by string, so an entry naming a method that no
|
|
48
|
+
* longer exists is dropped at runtime rather than failing to compile — tracing
|
|
49
|
+
* disappears and nothing says so. Exported so a test can assert every name here
|
|
50
|
+
* still resolves to a method.
|
|
51
|
+
*/
|
|
52
|
+
export const TRACED_LOCK_SERVICE_METHODS = [
|
|
53
|
+
"acquireLock",
|
|
54
|
+
"releaseLock",
|
|
55
|
+
"relinquishLock",
|
|
56
|
+
"notifyClosedByServer",
|
|
57
|
+
"checkAndUpdateLockIfPossible",
|
|
58
|
+
"ensureLockForWrite",
|
|
59
|
+
"shutdownAndExit",
|
|
60
|
+
"restartInactivityTimer",
|
|
61
|
+
"switchBranch",
|
|
62
|
+
];
|
|
52
63
|
export class LockService extends TracedEventEmitter {
|
|
53
64
|
applicationId;
|
|
54
65
|
branchName;
|
|
@@ -57,6 +68,7 @@ export class LockService extends TracedEventEmitter {
|
|
|
57
68
|
syncCallback;
|
|
58
69
|
waitForGenerationCompleteCallback;
|
|
59
70
|
onGatewayLivenessLapsed;
|
|
71
|
+
onLockConflict;
|
|
60
72
|
_status = LockServiceStatus.IDLE;
|
|
61
73
|
_lockId;
|
|
62
74
|
_heartbeatTimeout;
|
|
@@ -68,6 +80,22 @@ export class LockService extends TracedEventEmitter {
|
|
|
68
80
|
// (ready for takeover). Distinguishes that case from operational stops
|
|
69
81
|
// (lock release, shutdown, branch switch) so only idle-stops auto-resume.
|
|
70
82
|
_heartbeatStoppedForIdle = false;
|
|
83
|
+
// True from the moment the lock is deliberately given up until one is held
|
|
84
|
+
// again. The durable answer to "did we choose to let this go?", which the
|
|
85
|
+
// status cannot give: `checkAndUpdateLockIfPossible`, `switchBranch` and a
|
|
86
|
+
// failed release all reset it to IDLE, and IDLE is what tells the claim path
|
|
87
|
+
// a lock has never been held. Also the only thing that still remembers by the
|
|
88
|
+
// time a grab left over from before the release fails.
|
|
89
|
+
_lockGivenUp = false;
|
|
90
|
+
_closedByServer = false;
|
|
91
|
+
/**
|
|
92
|
+
* True only while the last-chance sync of a session that is ending is in
|
|
93
|
+
* flight. That write is not gated on still holding the lock — see
|
|
94
|
+
* {@link ensureLockForWrite}.
|
|
95
|
+
*/
|
|
96
|
+
_flushingFinalWork = false;
|
|
97
|
+
/** The evaluation overlapping callers join rather than starting their own. */
|
|
98
|
+
_inFlightLockEvaluation;
|
|
71
99
|
_inactivityTimeout;
|
|
72
100
|
_bufferPeriodTimeout;
|
|
73
101
|
_isRestarting = false;
|
|
@@ -76,9 +104,9 @@ export class LockService extends TracedEventEmitter {
|
|
|
76
104
|
_rpcClient;
|
|
77
105
|
_inactivityThresholdMs;
|
|
78
106
|
_lockBufferMs;
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
107
|
+
/** In-flight acquisition kicked off by a client attaching, if any. */
|
|
108
|
+
_clientLockAcquisition;
|
|
109
|
+
_lockOperationQueue = new OperationQueue();
|
|
82
110
|
_activityTracker;
|
|
83
111
|
/** Gateway peer id -> when its liveness was last refreshed. */
|
|
84
112
|
_gatewayLivenessByPeerId = new Map();
|
|
@@ -107,15 +135,7 @@ export class LockService extends TracedEventEmitter {
|
|
|
107
135
|
acquireLock: { operation: "acquiring lock" },
|
|
108
136
|
releaseLock: { operation: "releasing lock" },
|
|
109
137
|
});
|
|
110
|
-
|
|
111
|
-
"acquireLock",
|
|
112
|
-
"releaseLock",
|
|
113
|
-
"checkAndUpdateLockIfPossible",
|
|
114
|
-
"shutdown",
|
|
115
|
-
"shutdownAndExit",
|
|
116
|
-
"restartInactivityTimer",
|
|
117
|
-
];
|
|
118
|
-
addTracingToMethods(this, methods, this._tracer);
|
|
138
|
+
addTracingToMethods(this, [...TRACED_LOCK_SERVICE_METHODS], this._tracer);
|
|
119
139
|
void this.restartInactivityTimer();
|
|
120
140
|
}
|
|
121
141
|
getLogger() {
|
|
@@ -138,12 +158,28 @@ export class LockService extends TracedEventEmitter {
|
|
|
138
158
|
setOnGatewayLivenessLapsed(callback) {
|
|
139
159
|
this.onGatewayLivenessLapsed = callback;
|
|
140
160
|
}
|
|
161
|
+
/**
|
|
162
|
+
* Called when a client attached and somebody else already holds the lock.
|
|
163
|
+
*
|
|
164
|
+
* What a conflict means is the host's to decide, not this service's: a CLI
|
|
165
|
+
* with a person at it can describe the holder and offer a takeover, a
|
|
166
|
+
* sandbox can only write it down. This is the hook that used to be the
|
|
167
|
+
* `ConflictError` catch around the boot-time acquire.
|
|
168
|
+
*/
|
|
169
|
+
setOnLockConflict(callback) {
|
|
170
|
+
this.onLockConflict = callback;
|
|
171
|
+
}
|
|
141
172
|
get status() {
|
|
142
173
|
return this._status;
|
|
143
174
|
}
|
|
144
175
|
set status(status) {
|
|
145
176
|
if (this._status !== status) {
|
|
146
177
|
this._status = status;
|
|
178
|
+
if (status === LockServiceStatus.LOCKED) {
|
|
179
|
+
// Derived from the transition rather than set by each of the three
|
|
180
|
+
// places that can obtain a lock, so it cannot be forgotten in one.
|
|
181
|
+
this._lockGivenUp = false;
|
|
182
|
+
}
|
|
147
183
|
this.emit("statusChange", status);
|
|
148
184
|
}
|
|
149
185
|
}
|
|
@@ -168,25 +204,185 @@ export class LockService extends TracedEventEmitter {
|
|
|
168
204
|
get timeSinceLastActivity() {
|
|
169
205
|
return this._activityTracker.timeSinceLastActivity;
|
|
170
206
|
}
|
|
207
|
+
/** Whether any client — a browser editor or a headless gateway peer — is here. */
|
|
208
|
+
get hasAttachedClient() {
|
|
209
|
+
return (this._activityTracker.connectedUsers.length > 0 ||
|
|
210
|
+
this.hasAttachedGatewayPeer);
|
|
211
|
+
}
|
|
171
212
|
/**
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
213
|
+
* A client attached and this pod has never held the lock: take it for them.
|
|
214
|
+
*
|
|
215
|
+
* The lock is a mutual-exclusion marker between clients, not a capability the
|
|
216
|
+
* pod needs in order to serve — no write path on either side gates on holding
|
|
217
|
+
* it. So it belongs to whoever is attached, and a pod that grabs it at boot
|
|
218
|
+
* only invents a state nobody wants: a lock held on behalf of nobody, which
|
|
219
|
+
* then needs a watchdog to clean up. Claiming it on attach makes that state
|
|
220
|
+
* unreachable instead.
|
|
221
|
+
*
|
|
222
|
+
* Deliberately narrower than {@link reclaimLockForActiveClient}: neither a
|
|
223
|
+
* lock this pod handed back nor one it was refused is taken here. A tab left
|
|
224
|
+
* open reconnects its socket on its own, so an attach is no evidence its user
|
|
225
|
+
* came back, and treating it as evidence would hand the app to somebody who
|
|
226
|
+
* walked away (APPS-4840).
|
|
175
227
|
*/
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
228
|
+
claimLockForFirstClient() {
|
|
229
|
+
// `IDLE` on its own does not mean "never held". Several paths reset the
|
|
230
|
+
// status to it after a lock we did hold went away — a heartbeat grab that
|
|
231
|
+
// misses, a branch switch, a failed release — so trusting the status alone
|
|
232
|
+
// would turn a lock we handed back into a claimable one, and a tab that
|
|
233
|
+
// reconnects by itself would take the app back from whoever is using it.
|
|
234
|
+
if (this._lockGivenUp) {
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
this.acquireLockForAttachedClient([LockServiceStatus.IDLE]);
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* The client proved it is still there: take the lock if we do not have it.
|
|
241
|
+
*
|
|
242
|
+
* Driven by editor activity pings and by a gateway peer starting a turn —
|
|
243
|
+
* both are a client doing something, which is the evidence a lock we are not
|
|
244
|
+
* holding needs before it is worth asking for again.
|
|
245
|
+
*
|
|
246
|
+
* That includes a lock we were refused. A refusal is what happened to one
|
|
247
|
+
* attempt, not a mode this pod is stuck in: the other session ends, and the
|
|
248
|
+
* user sitting here still wants to edit. Without this a pod that lost one
|
|
249
|
+
* race could never take the lock again for the rest of its life — and since
|
|
250
|
+
* SABS hands running pods to the next caller, it would go on failing for
|
|
251
|
+
* people who had nothing to do with the race.
|
|
252
|
+
*/
|
|
253
|
+
reclaimLockForActiveClient() {
|
|
254
|
+
// Editors only. A gateway's route to the lock is {@link acquireLockForTurn}
|
|
255
|
+
// — an attached gateway peer is a socket, not a turn, and taking the lock
|
|
256
|
+
// for one would hold write access across the gap between prompts.
|
|
257
|
+
if (this._closedByServer ||
|
|
258
|
+
this._activityTracker.connectedUsers.length === 0) {
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
this.acquireLockForAttachedClient([
|
|
262
|
+
LockServiceStatus.IDLE,
|
|
263
|
+
LockServiceStatus.LOCK_RELEASED,
|
|
264
|
+
LockServiceStatus.FAILED_TO_ACQUIRE_LOCK,
|
|
265
|
+
]);
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Fire-and-forget by design: the sockets, pings and branch switches that ask
|
|
269
|
+
* for this have nothing to do with the answer.
|
|
270
|
+
*
|
|
271
|
+
* What keeps a retry from becoming a stream of 409s is the trigger, not the
|
|
272
|
+
* status: the only thing that asks repeatedly is the editor's activity ping,
|
|
273
|
+
* which the pod acts on solely while the user is genuinely active. A blocked
|
|
274
|
+
* session asks about as often as a held lock heartbeats; one nobody is
|
|
275
|
+
* sitting at goes quiet on its own.
|
|
276
|
+
*/
|
|
277
|
+
acquireLockForAttachedClient(from) {
|
|
278
|
+
if (this._clientLockAcquisition || !this.hasAttachedClient) {
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
if (!from.includes(this._status)) {
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
// Only the first refusal is news. The host's handler can prompt on stdin,
|
|
285
|
+
// so reporting each retry would be a prompt every ping.
|
|
286
|
+
const alreadyRefused = this._status === LockServiceStatus.FAILED_TO_ACQUIRE_LOCK;
|
|
287
|
+
if (!alreadyRefused) {
|
|
288
|
+
this.getLogger().info(`[lock-service] client attached with no lock held, acquiring (appId=${this.applicationId})`);
|
|
289
|
+
}
|
|
290
|
+
// Track the queued operation immediately so a lock-state evaluation cannot
|
|
291
|
+
// slip in before the queue starts it. `acquireLock` tracks the RPC itself as
|
|
292
|
+
// well, which covers callers that do not use this attach path.
|
|
293
|
+
void this.trackAcquisition(this._lockOperationQueue.enqueue(async () => {
|
|
294
|
+
if (!this.hasAttachedClient || !from.includes(this._status)) {
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
await this.acquireLock();
|
|
298
|
+
})).catch((error) => alreadyRefused ? undefined : this.reportLockConflict(error));
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Record an acquisition so anything that settles can see it.
|
|
302
|
+
*
|
|
303
|
+
* Every source goes through here — a client attaching, an activity ping, a
|
|
304
|
+
* force takeover, the reacquire after a heartbeat grab misses, a branch
|
|
305
|
+
* switch. Tracking only the attach path left the others invisible:
|
|
306
|
+
* `isLockedWhenSettled` would answer "no lock" for one that was an rpc away
|
|
307
|
+
* from landing, and `relinquishLock` would find nothing to release and let
|
|
308
|
+
* the lock land on a pod that had already gone.
|
|
309
|
+
*
|
|
310
|
+
* What is stored is a swallowed copy. Callers of
|
|
311
|
+
* {@link settleClientLockAcquisition} only want the state to have stopped
|
|
312
|
+
* moving; a rejection surfacing as theirs would turn a refused lock into a
|
|
313
|
+
* failed disconnect. The promise handed back keeps its rejection for whoever
|
|
314
|
+
* asked for the lock.
|
|
315
|
+
*/
|
|
316
|
+
trackAcquisition(acquisition) {
|
|
317
|
+
const tracked = acquisition
|
|
318
|
+
.catch(() => undefined)
|
|
319
|
+
.finally(() => {
|
|
320
|
+
if (this._clientLockAcquisition === tracked) {
|
|
321
|
+
this._clientLockAcquisition = undefined;
|
|
322
|
+
}
|
|
323
|
+
});
|
|
324
|
+
this._clientLockAcquisition = tracked;
|
|
325
|
+
return acquisition;
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Wait for an acquisition a client attach kicked off, if one is in flight.
|
|
329
|
+
*
|
|
330
|
+
* Anything that decides what to do about the lock has to do this first, or it
|
|
331
|
+
* decides on a half-known state. While the RPC is out `isLocked` is false and
|
|
332
|
+
* `_lockId` is unset, so a disconnect would "release" nothing and the lock
|
|
333
|
+
* would land moments later with nobody left to hold it for — and a branch
|
|
334
|
+
* switch would move on and leave it attached to the branch it came from.
|
|
335
|
+
*
|
|
336
|
+
* Never rejects: the acquisition's own catch has already handled failure.
|
|
337
|
+
*/
|
|
338
|
+
async settleClientLockAcquisition() {
|
|
339
|
+
await this._clientLockAcquisition;
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Whether this pod holds the lock after attach and queued turn operations
|
|
343
|
+
* already in flight have settled. The synchronous getter remains the current
|
|
344
|
+
* state for status reporting.
|
|
345
|
+
*/
|
|
346
|
+
async isLockedWhenSettled() {
|
|
347
|
+
await this.settleClientLockAcquisition();
|
|
348
|
+
return await this._lockOperationQueue.enqueue(async () => this.isLocked);
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* Nothing is awaiting the acquisition a client triggers, so this is where a
|
|
352
|
+
* failure stops: swallowing it here is what keeps a conflict from surfacing
|
|
353
|
+
* as an unhandled rejection. `acquireLock` has already logged it and moved
|
|
354
|
+
* the status to `FAILED_TO_ACQUIRE_LOCK`; the host is told separately because
|
|
355
|
+
* only it knows whether there is anyone to tell.
|
|
356
|
+
*/
|
|
357
|
+
async reportLockConflict(error) {
|
|
358
|
+
if (!(error instanceof ConflictError) || !this.onLockConflict) {
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
361
|
+
try {
|
|
362
|
+
await this.onLockConflict(error);
|
|
363
|
+
}
|
|
364
|
+
catch (handlerError) {
|
|
365
|
+
this.getLogger().warn(`[lock-service] onLockConflict threw; continuing (appId=${this.applicationId})`, getErrorMeta(handlerError));
|
|
179
366
|
}
|
|
180
|
-
return (this._status === LockServiceStatus.LOCK_RELEASED ||
|
|
181
|
-
this._status === LockServiceStatus.IDLE);
|
|
182
367
|
}
|
|
368
|
+
/**
|
|
369
|
+
* Serializes branch changes with lock acquisition and handback. Omitting
|
|
370
|
+
* `reacquireLock` preserves the lock state observed inside that operation.
|
|
371
|
+
*/
|
|
183
372
|
async switchBranch(nextBranchName, options) {
|
|
373
|
+
// Git activation runs off the same socket connect that starts an
|
|
374
|
+
// acquisition, so the switch really can arrive while one is out.
|
|
375
|
+
await this.settleClientLockAcquisition();
|
|
376
|
+
await this._lockOperationQueue.enqueue(() => this.switchBranchNow(nextBranchName, options));
|
|
377
|
+
}
|
|
378
|
+
async switchBranchNow(nextBranchName, options) {
|
|
184
379
|
if (this.branchName === nextBranchName) {
|
|
185
380
|
return;
|
|
186
381
|
}
|
|
187
382
|
const previousBranchName = this.branchName;
|
|
188
383
|
const lockIdBeforeSwitch = this._lockId;
|
|
189
|
-
|
|
384
|
+
const reacquireLock = options?.reacquireLock ?? this.isLocked;
|
|
385
|
+
if (reacquireLock && this.isLocked && lockIdBeforeSwitch) {
|
|
190
386
|
this.stopHeartbeat();
|
|
191
387
|
try {
|
|
192
388
|
const lockResponse = await this._rpcClient.call(async (client) => {
|
|
@@ -204,9 +400,6 @@ export class LockService extends TracedEventEmitter {
|
|
|
204
400
|
this.branchName = nextBranchName;
|
|
205
401
|
this._lockId = transferredLock.lockId;
|
|
206
402
|
this.status = LockServiceStatus.LOCKED;
|
|
207
|
-
// A same-pod atomic transfer is not a fresh startup, so the no-socket
|
|
208
|
-
// watchdog is intentionally left untouched: it keeps ticking from the
|
|
209
|
-
// original acquisition rather than restarting on every branch switch.
|
|
210
403
|
this.getLogger().info(`[lock-service] atomically transferred lock branch context from '${previousBranchName}' to '${nextBranchName}'`);
|
|
211
404
|
void this.scheduleNextHeartbeat();
|
|
212
405
|
return;
|
|
@@ -227,19 +420,27 @@ export class LockService extends TracedEventEmitter {
|
|
|
227
420
|
else {
|
|
228
421
|
this.stopHeartbeat();
|
|
229
422
|
this._lockId = undefined;
|
|
230
|
-
this.status
|
|
423
|
+
if (this.status !== LockServiceStatus.FAILED_TO_ACQUIRE_LOCK) {
|
|
424
|
+
this.status = LockServiceStatus.IDLE;
|
|
425
|
+
}
|
|
231
426
|
}
|
|
232
427
|
this.branchName = nextBranchName;
|
|
233
428
|
this.getLogger().info(`[lock-service] switched branch context from '${previousBranchName}' to '${nextBranchName}'`);
|
|
234
|
-
if (
|
|
429
|
+
if (reacquireLock) {
|
|
235
430
|
await this.acquireLock();
|
|
236
431
|
}
|
|
237
432
|
}
|
|
238
433
|
/**
|
|
239
|
-
* Acquire a lock on the specified application and branch
|
|
434
|
+
* Acquire a lock on the specified application and branch.
|
|
435
|
+
*
|
|
436
|
+
* Not `async`: the body has to start and be registered in the same tick, so
|
|
437
|
+
* a caller that immediately settles sees this acquisition rather than
|
|
438
|
+
* missing it by a microtask.
|
|
240
439
|
*/
|
|
241
|
-
|
|
242
|
-
this.
|
|
440
|
+
acquireLock() {
|
|
441
|
+
return this.trackAcquisition(this.acquireLockImpl());
|
|
442
|
+
}
|
|
443
|
+
async acquireLockImpl() {
|
|
243
444
|
this.status = LockServiceStatus.ACQUIRING;
|
|
244
445
|
// Stop any existing heartbeat before acquiring a new lock to prevent duplicate heartbeat loops
|
|
245
446
|
this.stopHeartbeat();
|
|
@@ -259,17 +460,27 @@ export class LockService extends TracedEventEmitter {
|
|
|
259
460
|
this.status = LockServiceStatus.LOCKED;
|
|
260
461
|
this._heartbeatStoppedForIdle = false;
|
|
261
462
|
this.emit("lockAcquired", this._lockId);
|
|
262
|
-
this.evaluateStartupConnectionTimeoutIfNeeded();
|
|
263
463
|
void this.scheduleNextHeartbeat();
|
|
264
464
|
return;
|
|
265
465
|
}
|
|
466
|
+
// A response carrying no lock is not success. Falling through would leave
|
|
467
|
+
// the status in ACQUIRING, which is what the catch below exists to avoid.
|
|
468
|
+
throw new Error("lock acquire returned no lock");
|
|
266
469
|
}
|
|
267
470
|
catch (error) {
|
|
471
|
+
// Whatever went wrong, this attempt is over and the lock is not held.
|
|
472
|
+
// Leaving ACQUIRING behind would be a lie the retry paths cannot see past:
|
|
473
|
+
// it is in neither the claim nor the reclaim set, so a single transient
|
|
474
|
+
// failure would stop this pod ever taking the lock again.
|
|
475
|
+
this.status = LockServiceStatus.FAILED_TO_ACQUIRE_LOCK;
|
|
268
476
|
if (error instanceof ConflictError) {
|
|
269
477
|
this.getLogger().error(`[lock-service] ${error.message}`);
|
|
270
478
|
this.emit("statusChange", LockServiceStatus.FAILED_TO_ACQUIRE_LOCK);
|
|
271
|
-
|
|
272
|
-
|
|
479
|
+
// Deliberately no further teardown. The lock was never held, so there is
|
|
480
|
+
// nothing to release, and the `shutdown` that used to be here only
|
|
481
|
+
// closed the rpc client — leaving a pod bound to its port with no way
|
|
482
|
+
// to reach the server, which is the zombie state. The error goes to the
|
|
483
|
+
// caller, which owns what happens to the process.
|
|
273
484
|
throw error;
|
|
274
485
|
}
|
|
275
486
|
else {
|
|
@@ -277,6 +488,67 @@ export class LockService extends TracedEventEmitter {
|
|
|
277
488
|
}
|
|
278
489
|
}
|
|
279
490
|
}
|
|
491
|
+
/**
|
|
492
|
+
* Take write access for a turn that is about to start.
|
|
493
|
+
*
|
|
494
|
+
* The lock is permission to write for one turn, not a lease on the session: a
|
|
495
|
+
* headless turn takes it on the way in and hands it back when it ends, so the
|
|
496
|
+
* app sits unlocked between prompts and a person can open it in the editor.
|
|
497
|
+
*
|
|
498
|
+
* Deliberately not routed through {@link claimLockForFirstClient} or
|
|
499
|
+
* {@link reclaimLockForActiveClient}. Those weigh up whether a signal is
|
|
500
|
+
* evidence of a client that wants the lock; a turn starting is not evidence
|
|
501
|
+
* of that, it *is* that, so it asks directly and a lock we handed back
|
|
502
|
+
* earlier is taken again without ceremony.
|
|
503
|
+
*
|
|
504
|
+
* Unlike an attach, this is awaited and its failure is the caller's: a turn
|
|
505
|
+
* that cannot write must fail before routing is bound, carrying the server's
|
|
506
|
+
* own reason. Failing does not take the pod down — the lock is per app+branch
|
|
507
|
+
* while a pod is per live edit, so somebody editing in the browser says
|
|
508
|
+
* nothing about whether this pod can serve the turn after them.
|
|
509
|
+
*/
|
|
510
|
+
async acquireLockForTurn() {
|
|
511
|
+
await this.settleClientLockAcquisition();
|
|
512
|
+
await this._lockOperationQueue.enqueue(async () => {
|
|
513
|
+
if (!this.isLocked) {
|
|
514
|
+
await this.acquireLock();
|
|
515
|
+
}
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
/**
|
|
519
|
+
* Hand back the write access a turn took, once that turn has finished.
|
|
520
|
+
*
|
|
521
|
+
* The wait belongs here rather than in the callers. A turn ends in more than
|
|
522
|
+
* one way — the gateway releases it, its socket drops, its liveness lapses —
|
|
523
|
+
* and only the first of those arrives with the generation already complete.
|
|
524
|
+
* The other two reach this while Clark may still be writing, and handing the
|
|
525
|
+
* lock back there gives write access away mid-generation.
|
|
526
|
+
*
|
|
527
|
+
* Before the queue, never inside it, for the reason spelled out on
|
|
528
|
+
* {@link handBackTurnLockWhenGenerationSettles}: that queue serialises the
|
|
529
|
+
* handback, branch switches and the next turn's acquisition, so a wait
|
|
530
|
+
* inside it lets one hung turn stall all of them.
|
|
531
|
+
*
|
|
532
|
+
* `isStillNeeded` is re-read after that wait, because waiting is exactly
|
|
533
|
+
* what makes the caller's decision stale: a replacement gateway can attach
|
|
534
|
+
* and take a turn while the old one's generation is finishing, and releasing
|
|
535
|
+
* then would pull write access out from under it. The caller owns the
|
|
536
|
+
* question — this cannot ask "is a gateway attached?" itself, since the
|
|
537
|
+
* explicit `editor.releaseTurnLock` path runs with one attached by
|
|
538
|
+
* definition.
|
|
539
|
+
*/
|
|
540
|
+
async relinquishLockForTurn(isStillNeeded) {
|
|
541
|
+
await this.waitForGenerationCompleteCallback?.();
|
|
542
|
+
await this._lockOperationQueue.enqueue(async () => {
|
|
543
|
+
if (!this.isLocked ||
|
|
544
|
+
this._activityTracker.connectedUsers.length > 0 ||
|
|
545
|
+
isStillNeeded?.()) {
|
|
546
|
+
return;
|
|
547
|
+
}
|
|
548
|
+
await this.relinquishLockNow(() => this._activityTracker.connectedUsers.length > 0 ||
|
|
549
|
+
(isStillNeeded?.() ?? false));
|
|
550
|
+
});
|
|
551
|
+
}
|
|
280
552
|
/**
|
|
281
553
|
* Release the currently held lock
|
|
282
554
|
*/
|
|
@@ -284,6 +556,7 @@ export class LockService extends TracedEventEmitter {
|
|
|
284
556
|
if (this._status !== LockServiceStatus.LOCKED) {
|
|
285
557
|
return false;
|
|
286
558
|
}
|
|
559
|
+
this._lockGivenUp = true;
|
|
287
560
|
this.status = LockServiceStatus.RELEASING;
|
|
288
561
|
this.emit("statusChange", LockServiceStatus.RELEASING);
|
|
289
562
|
// Stop heartbeat immediately and clear local lock id first to prevent any
|
|
@@ -332,7 +605,6 @@ export class LockService extends TracedEventEmitter {
|
|
|
332
605
|
clearTimeout(this._inactivityTimeout);
|
|
333
606
|
this._inactivityTimeout = undefined;
|
|
334
607
|
}
|
|
335
|
-
this.clearStartupConnectionTimeout();
|
|
336
608
|
}
|
|
337
609
|
}
|
|
338
610
|
async grabLockImpl() {
|
|
@@ -363,8 +635,10 @@ export class LockService extends TracedEventEmitter {
|
|
|
363
635
|
/**
|
|
364
636
|
* Force takeover of the lock from another user (only allowed when user is inactive)
|
|
365
637
|
*/
|
|
366
|
-
|
|
367
|
-
this.
|
|
638
|
+
forceTakeover() {
|
|
639
|
+
return this.trackAcquisition(this.forceTakeoverImpl());
|
|
640
|
+
}
|
|
641
|
+
async forceTakeoverImpl() {
|
|
368
642
|
this.status = LockServiceStatus.ACQUIRING;
|
|
369
643
|
try {
|
|
370
644
|
const lockResponse = await this._rpcClient.call(async (client) => {
|
|
@@ -381,10 +655,14 @@ export class LockService extends TracedEventEmitter {
|
|
|
381
655
|
this.getLogger().info(`[lock-service] lock acquired via force takeover: ${this._lockId}`);
|
|
382
656
|
this.status = LockServiceStatus.LOCKED;
|
|
383
657
|
this.emit("lockAcquired", this._lockId);
|
|
384
|
-
this.evaluateStartupConnectionTimeoutIfNeeded();
|
|
385
658
|
void this.scheduleNextHeartbeat();
|
|
386
659
|
return;
|
|
387
660
|
}
|
|
661
|
+
// As in `acquireLock`: a response carrying no lock is not success.
|
|
662
|
+
// Falling through would leave the status in ACQUIRING, which belongs to
|
|
663
|
+
// neither the claim set nor the reclaim set — so nothing would ever try
|
|
664
|
+
// again and the pod would sit there unable to edit.
|
|
665
|
+
throw new Error("lock force takeover returned no lock");
|
|
388
666
|
}
|
|
389
667
|
catch (error) {
|
|
390
668
|
this.getLogger().error(`[lock-service] ${getErrorMeta(error).error.message}`);
|
|
@@ -419,6 +697,20 @@ export class LockService extends TracedEventEmitter {
|
|
|
419
697
|
this.stopHeartbeat();
|
|
420
698
|
this._lockId = undefined;
|
|
421
699
|
this.status = LockServiceStatus.IDLE;
|
|
700
|
+
// Only take it back if we still wanted it, and only for somebody.
|
|
701
|
+
//
|
|
702
|
+
// A grab left over from before a release lands after the lock is
|
|
703
|
+
// already gone on purpose — acquiring fires a heartbeat immediately,
|
|
704
|
+
// so that overlap is the norm on the attach-and-drop path rather than
|
|
705
|
+
// a rare coincidence. Reacquiring there takes back a lock we chose to
|
|
706
|
+
// give up, either for nobody at all or for a tab that has merely
|
|
707
|
+
// reconnected, which is the app being held away from whoever is
|
|
708
|
+
// actually using it (APPS-4840). A released lock is taken back by an
|
|
709
|
+
// activity ping, never from here.
|
|
710
|
+
if (this._lockGivenUp || !this.hasAttachedClient) {
|
|
711
|
+
this.getLogger().info(`[lock-service] lock lost during heartbeat grab with nothing to take it back for, leaving it (appId=${this.applicationId})`);
|
|
712
|
+
return false;
|
|
713
|
+
}
|
|
422
714
|
try {
|
|
423
715
|
await this.acquireLock();
|
|
424
716
|
this.getLogger().info("[lock-service] lock reacquired after heartbeat grab miss");
|
|
@@ -464,50 +756,220 @@ export class LockService extends TracedEventEmitter {
|
|
|
464
756
|
return false;
|
|
465
757
|
}
|
|
466
758
|
/**
|
|
467
|
-
*
|
|
468
|
-
*
|
|
759
|
+
* Make sure this pod holds the lock a write is about to need.
|
|
760
|
+
*
|
|
761
|
+
* The write is itself the demand for the lock, which is what separates this
|
|
762
|
+
* from the heartbeat's check above. That one asks whether a lock we already
|
|
763
|
+
* hold is still there, and answers for a lock id; this one is asked before a
|
|
764
|
+
* sync, when there may be no lock at all.
|
|
765
|
+
*
|
|
766
|
+
* The distinction is load-bearing at boot. Nothing acquires the lock at
|
|
767
|
+
* startup any more — a client attaching is what claims it — but the dev
|
|
768
|
+
* server still uploads the user's local files before any editor connects.
|
|
769
|
+
* With no lock id {@link checkAndUpdateLockIfPossible} skips the grab
|
|
770
|
+
* entirely and reports false without ever asking the server, so routing that
|
|
771
|
+
* upload through it failed the sync and took the dev server down with it.
|
|
772
|
+
*
|
|
773
|
+
* A lock this session handed back is still not taken back here: a write
|
|
774
|
+
* queued before the handback belongs to the session that ended.
|
|
469
775
|
*/
|
|
470
|
-
async
|
|
471
|
-
|
|
472
|
-
|
|
776
|
+
async ensureLockForWrite() {
|
|
777
|
+
await this.settleClientLockAcquisition();
|
|
778
|
+
if (this._lockId) {
|
|
779
|
+
const held = await this.checkAndUpdateLockIfPossible();
|
|
780
|
+
// The last write of a session that is ending is not gated on still
|
|
781
|
+
// holding the lock. By the time the server closes a session the other
|
|
782
|
+
// one has usually taken the lock already, so the grab above misses and
|
|
783
|
+
// the answer is "no" — which would drop whatever the editor had not
|
|
784
|
+
// synced yet, and make the flush that runs at exactly this moment dead
|
|
785
|
+
// code, since it could never succeed.
|
|
786
|
+
//
|
|
787
|
+
// Writing anyway is safe in the way that matters: the lock is a
|
|
788
|
+
// mutual-exclusion marker between clients, not something the server
|
|
789
|
+
// enforces on writes, and the session being handed to on a local<->cloud
|
|
790
|
+
// switch belongs to the same user whose edits these are.
|
|
791
|
+
return held || this._flushingFinalWork;
|
|
473
792
|
}
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
793
|
+
if (this._lockGivenUp) {
|
|
794
|
+
this.getLogger().info(`[lock-service] write wants a lock this session handed back, leaving it (appId=${this.applicationId})`);
|
|
795
|
+
return false;
|
|
796
|
+
}
|
|
797
|
+
// Read before the attempt, because a refusal is what moves the status
|
|
798
|
+
// there: only the first one is news.
|
|
799
|
+
const alreadyRefused = this._status === LockServiceStatus.FAILED_TO_ACQUIRE_LOCK;
|
|
800
|
+
try {
|
|
801
|
+
await this.trackAcquisition(this._lockOperationQueue.enqueue(async () => {
|
|
802
|
+
// Re-read inside the queue: an attach or a handback can have landed
|
|
803
|
+
// while this waited its turn.
|
|
804
|
+
if (this._lockId || this._lockGivenUp) {
|
|
805
|
+
return;
|
|
806
|
+
}
|
|
807
|
+
await this.acquireLock();
|
|
808
|
+
}));
|
|
809
|
+
}
|
|
810
|
+
catch (error) {
|
|
811
|
+
// A refusal is the other session's answer, not a failure of this write.
|
|
812
|
+
if (!(error instanceof ConflictError)) {
|
|
813
|
+
throw error;
|
|
481
814
|
}
|
|
482
|
-
|
|
483
|
-
|
|
815
|
+
// The host is told because only it knows whether there is anyone to
|
|
816
|
+
// tell — and told once. Its handler can prompt on stdin or hand the
|
|
817
|
+
// session to the VS Code extension, so reporting every autosave that
|
|
818
|
+
// runs into the same held lock is a prompt per save.
|
|
819
|
+
if (!alreadyRefused) {
|
|
820
|
+
await this.reportLockConflict(error);
|
|
484
821
|
}
|
|
485
822
|
}
|
|
486
|
-
|
|
487
|
-
|
|
823
|
+
// Not `false` on the conflict path: that handler can take the lock, as the
|
|
824
|
+
// CLI's does for one whose owner has gone inactive. Whether this write may
|
|
825
|
+
// go ahead is whether the lock is held now, however it was come by.
|
|
826
|
+
//
|
|
827
|
+
// Arming applies to both outcomes for the same reason: a lock taken here
|
|
828
|
+
// was taken for a write, not for a client, and nothing else will ask
|
|
829
|
+
// whether it has gone idle.
|
|
830
|
+
this.ensureIdleEvaluationArmed();
|
|
831
|
+
return this.isLocked;
|
|
832
|
+
}
|
|
833
|
+
/**
|
|
834
|
+
* Flush what this client left behind and hand the lock back.
|
|
835
|
+
*
|
|
836
|
+
* The only way to give the lock up, and the whole of what this service does
|
|
837
|
+
* about a session ending. Whether the process then stops is the caller's
|
|
838
|
+
* business and this cannot tell: the same call serves a tab closing on a
|
|
839
|
+
* sandbox that keeps serving, a `superblocks dev` the user is quitting, and
|
|
840
|
+
* every exit path in `dev()`. So it says nothing about the server's fate —
|
|
841
|
+
* that used to be `shutdown()`'s job, and a lock service asserting a process
|
|
842
|
+
* was exiting is precisely the tangle that made a lock release look like a
|
|
843
|
+
* teardown.
|
|
844
|
+
*
|
|
845
|
+
* Releasing the lock and stopping the dev server used to be one action, so a
|
|
846
|
+
* client going away took the sandbox with it. On the cloud path it did not
|
|
847
|
+
* even stop — it closed the rpc client and returned, leaving a pod still
|
|
848
|
+
* bound to its port with no lock, no heartbeat and nothing behind it: healthy
|
|
849
|
+
* to every probe and unable to do anything. SABS hands such a pod straight
|
|
850
|
+
* back to the next caller, whose requests then go nowhere.
|
|
851
|
+
*
|
|
852
|
+
* Where the pod does stay up, reopening is cheap for the same reason keeping
|
|
853
|
+
* it was worth it: Vite is still running, packages are installed, files are
|
|
854
|
+
* downloaded. A returning editor reacquires through
|
|
855
|
+
* {@link restartInactivityTimer}.
|
|
856
|
+
*/
|
|
857
|
+
async relinquishLock() {
|
|
858
|
+
// Settle first, for the same reason `switchBranch` and `evaluateLockState`
|
|
859
|
+
// do. The attach-time acquisition is awaited by nobody, so an exit can land
|
|
860
|
+
// on top of one: `releaseLock` no-ops while the status is ACQUIRING, the
|
|
861
|
+
// pod leaves having released nothing, and the lock the acquire then lands
|
|
862
|
+
// is held until the server-side TTL - blocking the replacement sandbox,
|
|
863
|
+
// which is the whole reason the exit paths release promptly at all.
|
|
864
|
+
await this.settleClientLockAcquisition();
|
|
865
|
+
await this._lockOperationQueue.enqueue(() => this.relinquishLockNow());
|
|
866
|
+
}
|
|
867
|
+
async relinquishLockNow(shouldRetainLock) {
|
|
868
|
+
await this.flushPendingWork("handing the lock back");
|
|
869
|
+
if (shouldRetainLock?.()) {
|
|
870
|
+
return;
|
|
488
871
|
}
|
|
872
|
+
const released = await this.releaseLock();
|
|
873
|
+
// `releaseLock` answers false without doing anything when there was no lock
|
|
874
|
+
// to give back. Saying "handed back" regardless tells oncall the
|
|
875
|
+
// server-side lock is clear when it may not be, which is the wrong
|
|
876
|
+
// direction to send someone chasing a stuck sandbox.
|
|
877
|
+
this.getLogger().info(released
|
|
878
|
+
? `[lock-service] lock handed back (appId=${this.applicationId})`
|
|
879
|
+
: `[lock-service] no lock to hand back (status=${this._status}, appId=${this.applicationId})`);
|
|
880
|
+
}
|
|
881
|
+
/**
|
|
882
|
+
* The server closed this session: tell the editor, flush, and stop
|
|
883
|
+
* heartbeating — but do not release.
|
|
884
|
+
*
|
|
885
|
+
* The lock is already gone. `/_sb_disconnect` is the server saying it has
|
|
886
|
+
* taken it, typically to hand the app to a session moving the other way
|
|
887
|
+
* across local↔cloud, so releasing from here would give back something that
|
|
888
|
+
* is not ours; on a switch it belongs to whoever is taking over. What is
|
|
889
|
+
* still ours is the heartbeat, which would otherwise keep refreshing a lock
|
|
890
|
+
* the server reassigned.
|
|
891
|
+
*
|
|
892
|
+
* The editor needs both halves of {@link EventMap.closedByServer}: it has to
|
|
893
|
+
* distinguish "the server closed you" from a socket dropping, and a switch
|
|
894
|
+
* from an ending.
|
|
895
|
+
*/
|
|
896
|
+
async notifyClosedByServer({ switchingTo, initiatedByEmail, }) {
|
|
897
|
+
this.emit("closedByServer", { initiatedByEmail, switchingTo });
|
|
898
|
+
this._closedByServer = true;
|
|
899
|
+
// Record that the lock is no longer ours before anything can act on it
|
|
900
|
+
// being missing. This is the same fact `releaseLock` records, arrived at
|
|
901
|
+
// from the other direction — there we gave it up, here it was taken — and
|
|
902
|
+
// the three places that ask are already written against it: a bare attach
|
|
903
|
+
// must not claim, a grab that misses must not reacquire, and a rejected
|
|
904
|
+
// heartbeat must not be read as losing a lock we still wanted.
|
|
905
|
+
//
|
|
906
|
+
// Without it, each of those fights the session taking over on a local↔cloud
|
|
907
|
+
// switch: the reacquire takes the app straight back off them, and the
|
|
908
|
+
// heartbeat path ends this process mid-`/_sb_disconnect`, before the
|
|
909
|
+
// response is sent. Stopping the heartbeat below is not enough on its own
|
|
910
|
+
// — a grab already in flight is not cancelled by clearing a timer.
|
|
911
|
+
this._lockGivenUp = true;
|
|
912
|
+
// Before the flush, not after it. Every refresh from here on is against
|
|
913
|
+
// something we do not hold, and the flush can take as long as the unsynced
|
|
914
|
+
// work does — which is the window one of those refreshes comes back
|
|
915
|
+
// rejected in.
|
|
489
916
|
this.stopHeartbeat();
|
|
490
|
-
this.clearStartupConnectionTimeout();
|
|
491
917
|
this.clearGatewayLivenessSweep();
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
918
|
+
await this.flushPendingWork("the server closed this session");
|
|
919
|
+
}
|
|
920
|
+
/**
|
|
921
|
+
* Push whatever the editor has not synced yet.
|
|
922
|
+
*
|
|
923
|
+
* Both ways of giving a session up run this first, and neither can be held up
|
|
924
|
+
* by it: the caller is either handing the lock to the next client or on its
|
|
925
|
+
* way out of the process, and a sync that fails is strictly better than one
|
|
926
|
+
* that blocks the handover. An unheld lock means nothing to flush — the files
|
|
927
|
+
* belong to whoever holds it now.
|
|
928
|
+
*/
|
|
929
|
+
async flushPendingWork(reason) {
|
|
930
|
+
if (!this.syncCallback || !this.isLocked) {
|
|
931
|
+
return;
|
|
932
|
+
}
|
|
933
|
+
this.getLogger().info(`[lock-service] performing a final sync before ${reason}`);
|
|
934
|
+
// Tells {@link ensureLockForWrite} that the upload it is about to gate is
|
|
935
|
+
// this flush, which writes on the way out whether or not the lock is still
|
|
936
|
+
// ours.
|
|
937
|
+
this._flushingFinalWork = true;
|
|
938
|
+
try {
|
|
939
|
+
await this.syncCallback();
|
|
940
|
+
}
|
|
941
|
+
catch (error) {
|
|
942
|
+
this.getLogger().error(`[lock-service] sync failed before ${reason}, continuing anyway`, getErrorMeta(error));
|
|
943
|
+
}
|
|
944
|
+
finally {
|
|
945
|
+
this._flushingFinalWork = false;
|
|
497
946
|
}
|
|
498
|
-
this.getLogger().info("[lock-service] closing rpc client before shutdown");
|
|
499
|
-
await this._rpcClient.close();
|
|
500
|
-
this.getLogger().info("[lock-service] shutting down and exiting dev server");
|
|
501
947
|
}
|
|
502
948
|
/**
|
|
503
|
-
*
|
|
949
|
+
* Hand the lock back and end the process.
|
|
950
|
+
*
|
|
951
|
+
* The only method here that ends anything, and it exists for the four callers
|
|
952
|
+
* inside this service that have already decided the pod cannot continue: a
|
|
953
|
+
* heartbeat that failed, a reacquire lost to another user, and the two
|
|
954
|
+
* {@link LockType.LOCAL} disconnect branches, where the dev server is the
|
|
955
|
+
* user's foreground process and they expect their terminal back.
|
|
956
|
+
*
|
|
957
|
+
* Callers outside this service drive {@link relinquishLock} and own their own
|
|
958
|
+
* exit. That is not a style preference: `process.exit` is synchronous, so
|
|
959
|
+
* folding it in here makes everything a caller sequences after it dead code —
|
|
960
|
+
* which is exactly how the fatal-exit paths in `dev()` came to skip their
|
|
961
|
+
* telemetry flush on every pod that had a lock service.
|
|
962
|
+
*
|
|
963
|
+
* The release is best effort and the exit is not. A lock left held TTL-expires
|
|
964
|
+
* in two minutes, the same as any pod that dies; a process that fails to exit
|
|
965
|
+
* is a zombie until something reaps it.
|
|
504
966
|
*/
|
|
505
967
|
async shutdownAndExit() {
|
|
506
968
|
try {
|
|
507
|
-
await this.
|
|
969
|
+
await this.relinquishLock();
|
|
508
970
|
}
|
|
509
971
|
catch (error) {
|
|
510
|
-
this.getLogger().error("[lock-service]
|
|
972
|
+
this.getLogger().error("[lock-service] failed to hand the lock back, exiting anyway", getErrorMeta(error));
|
|
511
973
|
}
|
|
512
974
|
finally {
|
|
513
975
|
process.exit(1);
|
|
@@ -519,15 +981,24 @@ export class LockService extends TracedEventEmitter {
|
|
|
519
981
|
addConnectedPeerIdToUser(userId, peerId) {
|
|
520
982
|
this._activityTracker.addConnectedPeerIdToUser(userId, peerId);
|
|
521
983
|
this._isRestarting = false;
|
|
522
|
-
this.
|
|
984
|
+
this.claimLockForFirstClient();
|
|
523
985
|
if (this._bufferPeriodTimeout) {
|
|
524
986
|
clearTimeout(this._bufferPeriodTimeout);
|
|
525
987
|
this._bufferPeriodTimeout = undefined;
|
|
526
988
|
this.getLogger().info("[lock-service] user reconnected, cleared pending buffer period timeout");
|
|
527
989
|
}
|
|
528
|
-
this.
|
|
990
|
+
this.ensureIdleEvaluationArmed();
|
|
529
991
|
}
|
|
530
|
-
|
|
992
|
+
/**
|
|
993
|
+
* Make sure something is still due to ask whether this lock is idle.
|
|
994
|
+
*
|
|
995
|
+
* The inactivity timer is one-shot, and the evaluation it fires returns
|
|
996
|
+
* early on a service that has never held a lock without arming another. So
|
|
997
|
+
* any path that ends up holding one for something other than an attached
|
|
998
|
+
* client has to put the question back on the clock, or the lock is refreshed
|
|
999
|
+
* by the heartbeat for a client that never arrives.
|
|
1000
|
+
*/
|
|
1001
|
+
ensureIdleEvaluationArmed() {
|
|
531
1002
|
const alreadyConcludedIdle = this._activityTracker.isReadyForTakeover || this._heartbeatStoppedForIdle;
|
|
532
1003
|
if (this._inactivityTimeout || alreadyConcludedIdle) {
|
|
533
1004
|
return;
|
|
@@ -570,7 +1041,15 @@ export class LockService extends TracedEventEmitter {
|
|
|
570
1041
|
// grab was in flight, a failure here is moot — don't shut the pod down out
|
|
571
1042
|
// from under a user who may still be connected (the lock just TTL-expires).
|
|
572
1043
|
// A genuine heartbeat failure (not an idle stop) still shuts down as before.
|
|
573
|
-
|
|
1044
|
+
//
|
|
1045
|
+
// Same for a lock we handed back on purpose. Acquiring fires a heartbeat
|
|
1046
|
+
// immediately, so on the attach-and-drop path a grab is always still out
|
|
1047
|
+
// when the release lands; reading its failure as "we lost a lock we still
|
|
1048
|
+
// wanted" would take a cloud sandbox down for a lock it had already given
|
|
1049
|
+
// up. Neither the status nor the heartbeat generation can be used to tell
|
|
1050
|
+
// the two apart here — `checkAndUpdateLockIfPossible` resets both while
|
|
1051
|
+
// cleaning up after itself, before the error ever reaches this catch.
|
|
1052
|
+
if (this._heartbeatStoppedForIdle || this._lockGivenUp) {
|
|
574
1053
|
return;
|
|
575
1054
|
}
|
|
576
1055
|
this.getLogger().error("[lock-service] Error sending heartbeat, shutting down", getErrorMeta(error));
|
|
@@ -603,58 +1082,44 @@ export class LockService extends TracedEventEmitter {
|
|
|
603
1082
|
this.getLogger().info("[lock-service] heartbeat stopped");
|
|
604
1083
|
}
|
|
605
1084
|
}
|
|
606
|
-
clearStartupConnectionTimeout() {
|
|
607
|
-
if (this._startupConnectionTimeout) {
|
|
608
|
-
clearTimeout(this._startupConnectionTimeout);
|
|
609
|
-
this._startupConnectionTimeout = undefined;
|
|
610
|
-
}
|
|
611
|
-
this._startupConnectionTimeoutExpired = false;
|
|
612
|
-
this._startupConnectionTimeoutStartedAt = undefined;
|
|
613
|
-
}
|
|
614
|
-
startStartupConnectionTimeoutIfNeeded() {
|
|
615
|
-
if (this._startupConnectionTimeoutStartedAt !== undefined) {
|
|
616
|
-
return;
|
|
617
|
-
}
|
|
618
|
-
this.clearStartupConnectionTimeout();
|
|
619
|
-
this._startupConnectionTimeoutStartedAt = Date.now();
|
|
620
|
-
// info (not debug) so oncall can confirm the watchdog armed when
|
|
621
|
-
// reconstructing a fire incident; fires once per lock acquisition.
|
|
622
|
-
this.getLogger().info(`[lock-service] starting initial connection timeout window (${STARTUP_CONNECTION_TIMEOUT_MS}ms)`);
|
|
623
|
-
this._startupConnectionTimeout = setTimeout(() => {
|
|
624
|
-
this._startupConnectionTimeout = undefined;
|
|
625
|
-
this._startupConnectionTimeoutExpired = true;
|
|
626
|
-
this.evaluateStartupConnectionTimeoutIfNeeded();
|
|
627
|
-
}, STARTUP_CONNECTION_TIMEOUT_MS);
|
|
628
|
-
this._startupConnectionTimeout.unref();
|
|
629
|
-
}
|
|
630
|
-
evaluateStartupConnectionTimeoutIfNeeded() {
|
|
631
|
-
if (!this._startupConnectionTimeoutExpired) {
|
|
632
|
-
return;
|
|
633
|
-
}
|
|
634
|
-
if (!this.isLocked) {
|
|
635
|
-
return;
|
|
636
|
-
}
|
|
637
|
-
if (this._activityTracker.connectedUsers.length > 0) {
|
|
638
|
-
return;
|
|
639
|
-
}
|
|
640
|
-
if (this.hasAttachedGatewayPeer) {
|
|
641
|
-
return;
|
|
642
|
-
}
|
|
643
|
-
this.getLogger().warn(`[lock-service] no socket connection detected within ${STARTUP_CONNECTION_TIMEOUT_MS}ms after initial lock acquisition, marking session inactive (lockId=${this._lockId}, appId=${this.applicationId})`);
|
|
644
|
-
this._activityTracker.markDisconnectedDueToStartupTimeout();
|
|
645
|
-
void this.evaluateLockState().catch((error) => {
|
|
646
|
-
this.getLogger().error(`[lock-service] failed to evaluate lock state after startup connection timeout (lockId=${this._lockId}, appId=${this.applicationId})`, getErrorMeta(error));
|
|
647
|
-
});
|
|
648
|
-
}
|
|
649
1085
|
/**
|
|
650
1086
|
* Evaluate if the lock should be released (user disconnected) or marked as ready for takeover (user inactive but connected)
|
|
651
1087
|
*/
|
|
652
1088
|
async evaluateLockState() {
|
|
1089
|
+
// Single-flight. Several unrelated things ask for an evaluation — a client
|
|
1090
|
+
// disconnecting, the last gateway peer detaching, the inactivity timer —
|
|
1091
|
+
// and the body awaits twice before it decides anything (the acquisition,
|
|
1092
|
+
// then any running generation). The `_bufferPeriodTimeout` guard below is
|
|
1093
|
+
// read before those awaits, so two overlapping callers both saw no timer
|
|
1094
|
+
// and both armed one; the second arming overwrote the first handle, leaving
|
|
1095
|
+
// a timer nothing could cancel. Joining the in-flight evaluation is what
|
|
1096
|
+
// makes the guard mean what it says, and it holds for any await added to
|
|
1097
|
+
// the body later.
|
|
1098
|
+
this._inFlightLockEvaluation ??= this.evaluateLockStateOnce().finally(() => {
|
|
1099
|
+
this._inFlightLockEvaluation = undefined;
|
|
1100
|
+
});
|
|
1101
|
+
return this._inFlightLockEvaluation;
|
|
1102
|
+
}
|
|
1103
|
+
async evaluateLockStateOnce() {
|
|
1104
|
+
// A client can attach and drop again before its lock arrives — the flaky
|
|
1105
|
+
// connect this whole change is downstream of. Deciding now would find
|
|
1106
|
+
// `isLocked` false, release nothing, and leave the lock to land on an empty
|
|
1107
|
+
// sandbox and heartbeat there for nobody.
|
|
1108
|
+
await this.settleClientLockAcquisition();
|
|
653
1109
|
// Skip if already evaluating lock release (buffer timeout exists)
|
|
654
1110
|
if (this._bufferPeriodTimeout) {
|
|
655
1111
|
this.getLogger().info("[lock-service] lock release already in progress, skipping duplicate evaluation");
|
|
656
1112
|
return;
|
|
657
1113
|
}
|
|
1114
|
+
// Nothing to evaluate before a client has ever claimed the lock. The
|
|
1115
|
+
// inactivity timer is armed from the constructor, so on the cloud path it
|
|
1116
|
+
// now elapses on a warm, unclaimed sandbox — that is an empty sandbox, not
|
|
1117
|
+
// a lock going stale. A client that came and went is DISCONNECTED rather
|
|
1118
|
+
// than untouched, and still needs the release path below.
|
|
1119
|
+
if (this._status === LockServiceStatus.IDLE &&
|
|
1120
|
+
!this._activityTracker.isDisconnected) {
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
658
1123
|
// 1. Wait for Clark generation to complete if it's generating
|
|
659
1124
|
if (this.waitForGenerationCompleteCallback) {
|
|
660
1125
|
this.getLogger().debug("[lock-service] waiting for Clark generation to complete before evaluating lock state");
|
|
@@ -676,7 +1141,6 @@ export class LockService extends TracedEventEmitter {
|
|
|
676
1141
|
return;
|
|
677
1142
|
}
|
|
678
1143
|
const isUserDisconnected = this._activityTracker.isDisconnected;
|
|
679
|
-
const isAwaitingFirstEditor = this._activityTracker.connectedUsers.length === 0;
|
|
680
1144
|
// Case 1: User is disconnected - release the lock
|
|
681
1145
|
if (isUserDisconnected) {
|
|
682
1146
|
if (this._activityTracker.wasRecentlyActive) {
|
|
@@ -688,7 +1152,7 @@ export class LockService extends TracedEventEmitter {
|
|
|
688
1152
|
void this.shutdownAndExit();
|
|
689
1153
|
}
|
|
690
1154
|
else {
|
|
691
|
-
void this.
|
|
1155
|
+
void this.relinquishLock();
|
|
692
1156
|
}
|
|
693
1157
|
}, this._lockBufferMs);
|
|
694
1158
|
this._bufferPeriodTimeout.unref();
|
|
@@ -700,22 +1164,31 @@ export class LockService extends TracedEventEmitter {
|
|
|
700
1164
|
await this.shutdownAndExit();
|
|
701
1165
|
}
|
|
702
1166
|
else {
|
|
703
|
-
await this.
|
|
1167
|
+
await this.relinquishLock();
|
|
704
1168
|
}
|
|
705
1169
|
}
|
|
706
1170
|
}
|
|
707
|
-
else if (isAwaitingFirstEditor) {
|
|
708
|
-
this.getLogger().debug("[lock-service] no editor connected yet, leaving the lock to the startup watchdog");
|
|
709
|
-
}
|
|
710
1171
|
else {
|
|
711
|
-
// Case 2:
|
|
712
|
-
//
|
|
713
|
-
//
|
|
714
|
-
//
|
|
715
|
-
//
|
|
716
|
-
// user can
|
|
717
|
-
//
|
|
718
|
-
|
|
1172
|
+
// Case 2: the lock is idle - either a connected client has gone quiet, or
|
|
1173
|
+
// a boot-time force takeover was never claimed by one. Either way, mark
|
|
1174
|
+
// it ready for takeover AND stop the heartbeat. The pod
|
|
1175
|
+
// heartbeat is the ONLY thing refreshing the server-side AppBranchLock;
|
|
1176
|
+
// if we keep heartbeating while nobody is using it, the lock never
|
|
1177
|
+
// TTL-expires and no other user can take over the app (APPS-4840).
|
|
1178
|
+
// Stopping it lets the lock expire so another user can acquire it
|
|
1179
|
+
// normally, without taking a dev server that is serving fine down with
|
|
1180
|
+
// it. restartInactivityTimer() resumes the heartbeat if this user becomes
|
|
1181
|
+
// active again before the lock expires.
|
|
1182
|
+
// The two cases reach the same handling but are very different
|
|
1183
|
+
// incidents, so the line names which one it is. A lock no client ever
|
|
1184
|
+
// claimed is the state the deleted startup watchdog used to report with
|
|
1185
|
+
// its own distinctive line; without a field here, oncall reconstructing a
|
|
1186
|
+
// stuck-lock incident cannot tell "nobody ever attached" from "someone
|
|
1187
|
+
// stepped away", and both read as routine idleness.
|
|
1188
|
+
const reason = this.hasAttachedClient
|
|
1189
|
+
? "connected client went quiet"
|
|
1190
|
+
: "never claimed by a client";
|
|
1191
|
+
this.getLogger().info(`[lock-service] lock idle (${reason}), marking ready for takeover and stopping heartbeat so it can expire (lockId=${this._lockId}, appId=${this.applicationId})`);
|
|
719
1192
|
this._activityTracker.markReadyForTakeover();
|
|
720
1193
|
this._heartbeatStoppedForIdle = true;
|
|
721
1194
|
this.stopHeartbeat();
|
|
@@ -754,10 +1227,7 @@ export class LockService extends TracedEventEmitter {
|
|
|
754
1227
|
this.getLogger().info(`[lock-service] user active again, resuming lock heartbeat (lockId=${this._lockId}, appId=${this.applicationId})`);
|
|
755
1228
|
void this.scheduleNextHeartbeat();
|
|
756
1229
|
}
|
|
757
|
-
|
|
758
|
-
this.getLogger().info("[lock-service] user active again with no lock held, reacquiring");
|
|
759
|
-
void this.acquireLock();
|
|
760
|
-
}
|
|
1230
|
+
this.reclaimLockForActiveClient();
|
|
761
1231
|
this.scheduleInactivityEvaluation();
|
|
762
1232
|
}
|
|
763
1233
|
scheduleInactivityEvaluation() {
|
|
@@ -779,11 +1249,10 @@ export class LockService extends TracedEventEmitter {
|
|
|
779
1249
|
*
|
|
780
1250
|
* Gateway peers are deliberately not connected-user presence — they are not
|
|
781
1251
|
* a browser editor and must not keep an editor's lock alive — so they need
|
|
782
|
-
* their own liveness signal. Without it the
|
|
783
|
-
*
|
|
784
|
-
*
|
|
785
|
-
*
|
|
786
|
-
* still working.
|
|
1252
|
+
* their own liveness signal. Without it the inactivity reaper would take the
|
|
1253
|
+
* sandbox out from under a headless build that nobody is watching. Tracked
|
|
1254
|
+
* per peer id so a reattach that closes the socket it replaced does not hand
|
|
1255
|
+
* the sandbox back while its successor is still working.
|
|
787
1256
|
*/
|
|
788
1257
|
markGatewayAttached(peerId) {
|
|
789
1258
|
const isNew = !this._gatewayLivenessByPeerId.has(peerId);
|
|
@@ -795,6 +1264,10 @@ export class LockService extends TracedEventEmitter {
|
|
|
795
1264
|
}
|
|
796
1265
|
this.getLogger().info(`[lock-service] gateway peer attached (peerId=${peerId}, appId=${this.applicationId})`);
|
|
797
1266
|
this.armGatewayLivenessSweep();
|
|
1267
|
+
// No lock is taken here. A gateway peer being attached says only that a
|
|
1268
|
+
// socket is open; {@link acquireLockForTurn} takes write access when there
|
|
1269
|
+
// is actually a turn to write for, which is what keeps the app openable
|
|
1270
|
+
// between prompts.
|
|
798
1271
|
void this.restartInactivityTimer();
|
|
799
1272
|
}
|
|
800
1273
|
/**
|
|
@@ -806,20 +1279,72 @@ export class LockService extends TracedEventEmitter {
|
|
|
806
1279
|
return;
|
|
807
1280
|
}
|
|
808
1281
|
this.getLogger().info(`[lock-service] gateway peer detached (peerId=${peerId}, appId=${this.applicationId})`);
|
|
809
|
-
if (this._gatewayLivenessByPeerId.size
|
|
810
|
-
|
|
1282
|
+
if (this._gatewayLivenessByPeerId.size !== 0) {
|
|
1283
|
+
return;
|
|
811
1284
|
}
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
1285
|
+
this.clearGatewayLivenessSweep();
|
|
1286
|
+
this.handleLastGatewayPeerGone();
|
|
1287
|
+
}
|
|
1288
|
+
/**
|
|
1289
|
+
* The last gateway peer is gone, so no turn is running.
|
|
1290
|
+
*
|
|
1291
|
+
* Write access was taken for the turn, so it goes back now rather than being
|
|
1292
|
+
* left to TTL out — that would leave the app unopenable for up to the lock's
|
|
1293
|
+
* two-minute lifetime after a headless build had already finished. This is
|
|
1294
|
+
* also the backstop for a gateway that dies mid-turn.
|
|
1295
|
+
*
|
|
1296
|
+
* Deliberately not a shutdown: the gateway drops its socket after every turn
|
|
1297
|
+
* and opens a new one for the next prompt, and in between the pod keeps
|
|
1298
|
+
* serving. The next turn takes the lock again through
|
|
1299
|
+
* {@link acquireLockForTurn}. What bounds the pod's life is SABS's live-edit
|
|
1300
|
+
* expiry, not this.
|
|
1301
|
+
*
|
|
1302
|
+
* Only when no editor is attached. An editor disconnect releases on its own
|
|
1303
|
+
* buffer in {@link evaluateLockState}, and a second releaser over the top of
|
|
1304
|
+
* that would race it.
|
|
1305
|
+
*/
|
|
1306
|
+
handleLastGatewayPeerGone() {
|
|
1307
|
+
if (this._activityTracker.connectedUsers.length > 0) {
|
|
1308
|
+
void this.evaluateLockState().catch((error) => {
|
|
1309
|
+
this.getLogger().error(`[lock-service] failed to evaluate lock state after the last gateway peer detached (lockId=${this._lockId}, appId=${this.applicationId})`, getErrorMeta(error));
|
|
1310
|
+
});
|
|
1311
|
+
return;
|
|
1312
|
+
}
|
|
1313
|
+
void this.handBackTurnLockWhenGenerationSettles().catch((error) => {
|
|
1314
|
+
this.getLogger().error(`[lock-service] failed to hand the turn lock back after the last gateway peer detached (lockId=${this._lockId}, appId=${this.applicationId})`, getErrorMeta(error));
|
|
1315
|
+
});
|
|
1316
|
+
}
|
|
1317
|
+
/**
|
|
1318
|
+
* Wait for the turn to finish, then hand its lock back.
|
|
1319
|
+
*
|
|
1320
|
+
* The wait sits outside the lock queue, the way {@link evaluateLockState}
|
|
1321
|
+
* does it. That queue serialises the handback itself, branch switches,
|
|
1322
|
+
* `isLockedWhenSettled` and the next turn's acquisition, so waiting inside
|
|
1323
|
+
* it lets a single hung generation stall all of them — including a graceful
|
|
1324
|
+
* shutdown, which awaits the handback with no timeout of its own.
|
|
1325
|
+
*/
|
|
1326
|
+
async handBackTurnLockWhenGenerationSettles() {
|
|
1327
|
+
await this.waitForGenerationCompleteCallback?.();
|
|
1328
|
+
await this._lockOperationQueue.enqueue(async () => {
|
|
1329
|
+
if (this.hasAttachedGatewayPeer ||
|
|
1330
|
+
this._activityTracker.connectedUsers.length > 0) {
|
|
1331
|
+
return;
|
|
1332
|
+
}
|
|
1333
|
+
if (!this.isLocked) {
|
|
1334
|
+
return;
|
|
1335
|
+
}
|
|
1336
|
+
this.getLogger().info(`[lock-service] no gateway peer left, handing the turn lock back and staying up (lockId=${this._lockId}, appId=${this.applicationId})`);
|
|
1337
|
+
await this.relinquishLockNow(() => this.hasAttachedGatewayPeer ||
|
|
1338
|
+
this._activityTracker.connectedUsers.length > 0);
|
|
1339
|
+
});
|
|
815
1340
|
}
|
|
816
1341
|
/**
|
|
817
1342
|
* Watches for the last gateway signal lapsing.
|
|
818
1343
|
*
|
|
819
|
-
* The
|
|
820
|
-
*
|
|
821
|
-
*
|
|
822
|
-
*
|
|
1344
|
+
* The inactivity reaper a gateway defers fires once and is then only
|
|
1345
|
+
* re-examined when something happens. A silent gateway is the absence of
|
|
1346
|
+
* anything happening, so this is what notices it and puts the lock back up
|
|
1347
|
+
* for evaluation.
|
|
823
1348
|
*/
|
|
824
1349
|
armGatewayLivenessSweep() {
|
|
825
1350
|
if (this._gatewayLivenessSweep) {
|
|
@@ -830,10 +1355,9 @@ export class LockService extends TracedEventEmitter {
|
|
|
830
1355
|
return;
|
|
831
1356
|
}
|
|
832
1357
|
this.clearGatewayLivenessSweep();
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
});
|
|
1358
|
+
// A gateway that went silent is a turn that ended without saying so, so
|
|
1359
|
+
// it takes the same route as a clean detach.
|
|
1360
|
+
this.handleLastGatewayPeerGone();
|
|
837
1361
|
}, Math.max(1, Math.floor(this._gatewayLivenessTtlMs / 2)));
|
|
838
1362
|
this._gatewayLivenessSweep.unref?.();
|
|
839
1363
|
}
|