@superblocksteam/sdk 2.0.156 → 2.0.157-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/cli-replacement/automatic-upgrades.d.ts.map +1 -1
- package/dist/cli-replacement/automatic-upgrades.js +4 -8
- package/dist/cli-replacement/automatic-upgrades.js.map +1 -1
- package/dist/cli-replacement/automatic-upgrades.test.js +2 -1
- package/dist/cli-replacement/automatic-upgrades.test.js.map +1 -1
- package/dist/cli-replacement/dev-s3-restore.test.mjs +6 -2
- package/dist/cli-replacement/dev-s3-restore.test.mjs.map +1 -1
- package/dist/cli-replacement/dev-startup-git-before-dbfs-order.test.mjs +6 -2
- package/dist/cli-replacement/dev-startup-git-before-dbfs-order.test.mjs.map +1 -1
- package/dist/cli-replacement/dev-startup-lock-acquisition.test.d.mts +2 -0
- package/dist/cli-replacement/dev-startup-lock-acquisition.test.d.mts.map +1 -0
- package/dist/cli-replacement/dev-startup-lock-acquisition.test.mjs +487 -0
- package/dist/cli-replacement/dev-startup-lock-acquisition.test.mjs.map +1 -0
- package/dist/cli-replacement/dev.d.mts +6 -6
- package/dist/cli-replacement/dev.d.mts.map +1 -1
- package/dist/cli-replacement/dev.interception.test.mjs +12 -12
- package/dist/cli-replacement/dev.interception.test.mjs.map +1 -1
- package/dist/cli-replacement/dev.mjs +100 -42
- package/dist/cli-replacement/dev.mjs.map +1 -1
- package/dist/dev-utils/dev-server.d.mts +34 -1
- package/dist/dev-utils/dev-server.d.mts.map +1 -1
- package/dist/dev-utils/dev-server.mjs +69 -70
- package/dist/dev-utils/dev-server.mjs.map +1 -1
- package/dist/dev-utils/dev-server.shutdown-lock.test.d.mts +2 -0
- package/dist/dev-utils/dev-server.shutdown-lock.test.d.mts.map +1 -0
- package/dist/dev-utils/dev-server.shutdown-lock.test.mjs +48 -0
- package/dist/dev-utils/dev-server.shutdown-lock.test.mjs.map +1 -0
- package/dist/dev-utils/fatal-process-barrier.d.mts +71 -0
- package/dist/dev-utils/fatal-process-barrier.d.mts.map +1 -0
- package/dist/dev-utils/fatal-process-barrier.mjs +161 -0
- package/dist/dev-utils/fatal-process-barrier.mjs.map +1 -0
- package/dist/dev-utils/fatal-process-barrier.spawn-fixture.d.mts +2 -0
- package/dist/dev-utils/fatal-process-barrier.spawn-fixture.d.mts.map +1 -0
- package/dist/dev-utils/fatal-process-barrier.spawn-fixture.mjs +9 -0
- package/dist/dev-utils/fatal-process-barrier.spawn-fixture.mjs.map +1 -0
- package/dist/dev-utils/fatal-process-barrier.test.d.mts +2 -0
- package/dist/dev-utils/fatal-process-barrier.test.d.mts.map +1 -0
- package/dist/dev-utils/fatal-process-barrier.test.mjs +281 -0
- package/dist/dev-utils/fatal-process-barrier.test.mjs.map +1 -0
- package/dist/telemetry/logging.js +1 -1
- package/dist/telemetry/logging.js.map +1 -1
- package/dist/telemetry/safe-stringify.d.ts.map +1 -1
- package/dist/telemetry/safe-stringify.js +25 -9
- package/dist/telemetry/safe-stringify.js.map +1 -1
- package/package.json +10 -6
- package/src/cli-replacement/automatic-upgrades.test.ts +2 -1
- package/src/cli-replacement/automatic-upgrades.ts +4 -11
- package/src/cli-replacement/dev-s3-restore.test.mts +6 -2
- package/src/cli-replacement/dev-startup-git-before-dbfs-order.test.mts +6 -2
- package/src/cli-replacement/dev-startup-lock-acquisition.test.mts +573 -0
- package/src/cli-replacement/dev.interception.test.mts +12 -12
- package/src/cli-replacement/dev.mts +112 -56
- package/src/dev-utils/dev-server.mts +93 -88
- package/src/dev-utils/dev-server.shutdown-lock.test.mts +58 -0
- package/src/dev-utils/fatal-process-barrier.mts +262 -0
- package/src/dev-utils/fatal-process-barrier.spawn-fixture.mts +12 -0
- package/src/dev-utils/fatal-process-barrier.test.mts +346 -0
- package/src/telemetry/logging.ts +1 -1
- package/src/telemetry/safe-stringify.ts +25 -10
- package/test/safe-stringify.test.mts +25 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -16,10 +16,10 @@ import type {
|
|
|
16
16
|
DependencyUpgradeError,
|
|
17
17
|
ServerError,
|
|
18
18
|
} from "@superblocksteam/library-shared/types";
|
|
19
|
+
import type { ConflictError } from "@superblocksteam/shared";
|
|
19
20
|
import {
|
|
20
21
|
buildGithubSuperblocksSyncWorkflow,
|
|
21
22
|
buildGithubSuperblocksSyncWorkflowFromBaseUrl,
|
|
22
|
-
ConflictError,
|
|
23
23
|
DEFAULT_NON_GIT_BRANCH,
|
|
24
24
|
isGitAuthErrorMessage,
|
|
25
25
|
isGitHubRemoteUrl,
|
|
@@ -176,6 +176,19 @@ async function handleInactiveLockTakeover(
|
|
|
176
176
|
throw new Error("Lock takeover prompt shown");
|
|
177
177
|
}
|
|
178
178
|
|
|
179
|
+
// Only ask if somebody can answer. `promptUser` reads stdin, which on a
|
|
180
|
+
// sandbox nothing is attached to — it would wait forever for a keystroke
|
|
181
|
+
// that cannot come. Reachable now that conflicts surface when a client
|
|
182
|
+
// attaches rather than only during a CLI startup.
|
|
183
|
+
if (!process.stdin.isTTY) {
|
|
184
|
+
logger.warn(`${warningMessage} Run with --force-takeover to take it.`);
|
|
185
|
+
span.setStatus({
|
|
186
|
+
code: SpanStatusCode.ERROR,
|
|
187
|
+
message: "Lock held by an inactive user, nobody available to prompt",
|
|
188
|
+
});
|
|
189
|
+
throw new Error(warningMessage);
|
|
190
|
+
}
|
|
191
|
+
|
|
179
192
|
// Prompt user to force takeover (CLI only)
|
|
180
193
|
logger.warn(warningMessage);
|
|
181
194
|
const shouldTakeover = await getBooleanUserResponse(prompt);
|
|
@@ -1019,12 +1032,12 @@ export async function flushTelemetryWithTimeout(): Promise<boolean> {
|
|
|
1019
1032
|
* MUST BE CALLED LAST in any shutdown sequence. This does not merely flush: it
|
|
1020
1033
|
* awaits `shutdownTelemetry()`, which tears the provider down, so every signal
|
|
1021
1034
|
* emitted after it goes into a dead pipeline. It can also block for up to
|
|
1022
|
-
* `TELEMETRY_FLUSH_TIMEOUT_MS`, and delaying `lockService.
|
|
1023
|
-
* much delays the lock release that lets a replacement sandbox take over —
|
|
1035
|
+
* `TELEMETRY_FLUSH_TIMEOUT_MS`, and delaying `lockService.relinquishLock()` by
|
|
1036
|
+
* that much delays the lock release that lets a replacement sandbox take over —
|
|
1024
1037
|
* which is the entire recovery story this exit path exists to enable.
|
|
1025
1038
|
*
|
|
1026
1039
|
* "Last" is only reachable because the callers drive the lock service with
|
|
1027
|
-
* `
|
|
1040
|
+
* `relinquishLock()` rather than `shutdownAndExit()`. The latter ends in an
|
|
1028
1041
|
* unconditional `finally { process.exit(1) }`, and `process.exit` is
|
|
1029
1042
|
* synchronous, so anything sequenced after it never runs at all. See the two
|
|
1030
1043
|
* fatal-exit blocks in `dev()`.
|
|
@@ -1075,8 +1088,8 @@ const FATAL_EXIT_STEP_TIMEOUT_MS = 5000;
|
|
|
1075
1088
|
|
|
1076
1089
|
/**
|
|
1077
1090
|
* One teardown step. Returning `undefined` is allowed so callers can pass the
|
|
1078
|
-
* optional-chained call directly (`() => lockService?.
|
|
1079
|
-
* having to re-narrow a mutable binding at the call site.
|
|
1091
|
+
* optional-chained call directly (`() => lockService?.relinquishLock()`)
|
|
1092
|
+
* without having to re-narrow a mutable binding at the call site.
|
|
1080
1093
|
*/
|
|
1081
1094
|
type FatalExitStep = () => Promise<unknown> | undefined;
|
|
1082
1095
|
|
|
@@ -1130,7 +1143,7 @@ async function runFatalExitStep(
|
|
|
1130
1143
|
*
|
|
1131
1144
|
* Order is load-bearing:
|
|
1132
1145
|
* 1. `removeIntegrationCache` — ai-service state that must not outlive us.
|
|
1133
|
-
* 2. `
|
|
1146
|
+
* 2. `relinquishLock` — hands the AppBranchLock back. Recovery-critical:
|
|
1134
1147
|
* the heartbeat dies with the process, so skipping this makes the
|
|
1135
1148
|
* replacement sandbox's `acquireLock()` fail with a ConflictError until
|
|
1136
1149
|
* the server-side TTL expires, and the replacement is the entire point of
|
|
@@ -1145,15 +1158,15 @@ async function runFatalExitStep(
|
|
|
1145
1158
|
*
|
|
1146
1159
|
* This does NOT exit. The caller owns `process.exit(1)` from a `finally`, so
|
|
1147
1160
|
* the exit still happens even if this function is somehow bypassed. That is
|
|
1148
|
-
* also why callers must drive the lock service through `
|
|
1149
|
-
* `shutdownAndExit()`: the latter ends in its own unconditional
|
|
1161
|
+
* also why callers must drive the lock service through `relinquishLock()` and
|
|
1162
|
+
* not `shutdownAndExit()`: the latter ends in its own unconditional
|
|
1150
1163
|
* `finally { process.exit(1) }`, and `process.exit` is synchronous, so
|
|
1151
1164
|
* everything sequenced after it — including the flush — silently never ran.
|
|
1152
1165
|
*/
|
|
1153
1166
|
export async function runFatalExitShutdown(steps: {
|
|
1154
1167
|
logger: Logger;
|
|
1155
1168
|
removeIntegrationCache?: FatalExitStep;
|
|
1156
|
-
|
|
1169
|
+
relinquishLock?: FatalExitStep;
|
|
1157
1170
|
}): Promise<void> {
|
|
1158
1171
|
const { logger } = steps;
|
|
1159
1172
|
await runFatalExitStep(
|
|
@@ -1161,11 +1174,7 @@ export async function runFatalExitShutdown(steps: {
|
|
|
1161
1174
|
"ai-service integration cache removal",
|
|
1162
1175
|
steps.removeIntegrationCache,
|
|
1163
1176
|
);
|
|
1164
|
-
await runFatalExitStep(
|
|
1165
|
-
logger,
|
|
1166
|
-
"lock service shutdown",
|
|
1167
|
-
steps.shutdownLockService,
|
|
1168
|
-
);
|
|
1177
|
+
await runFatalExitStep(logger, "lock hand-back", steps.relinquishLock);
|
|
1169
1178
|
await flushTelemetryBeforeExit(logger);
|
|
1170
1179
|
}
|
|
1171
1180
|
|
|
@@ -1635,35 +1644,84 @@ export async function dev(options: {
|
|
|
1635
1644
|
|
|
1636
1645
|
logger.info("Checking if local files are synced with the server");
|
|
1637
1646
|
|
|
1647
|
+
const lockSvc = lockService;
|
|
1648
|
+
|
|
1649
|
+
// Nobody takes the lock to boot: nothing startup does is gated on
|
|
1650
|
+
// holding one, and a lock taken before any client exists is held on
|
|
1651
|
+
// behalf of nobody. `LockService` takes it when the first client
|
|
1652
|
+
// attaches, and tells us here if somebody else already has it —
|
|
1653
|
+
// this is the `ConflictError` catch that used to wrap the boot-time
|
|
1654
|
+
// acquire, moved to where acquisition actually happens (ENG-6090).
|
|
1655
|
+
lockSvc.setOnLockConflict(async (error) => {
|
|
1656
|
+
let stopForExtension = false;
|
|
1657
|
+
await tracer.startActiveSpan("handlingLockConflict", async (span) => {
|
|
1658
|
+
try {
|
|
1659
|
+
await handleLockConflict(error, lockSvc, span, logger);
|
|
1660
|
+
} catch (conflictError) {
|
|
1661
|
+
// Reporting is all this owes the caller. The holder is someone
|
|
1662
|
+
// else's live session and the pod is still serving preview, so
|
|
1663
|
+
// there is nothing to tear down — the lock does not decide how
|
|
1664
|
+
// long this process lives (ENG-6032).
|
|
1665
|
+
logger.warn(
|
|
1666
|
+
`Could not take the live-edit lock: ${conflictError instanceof Error ? conflictError.message : String(conflictError)}`,
|
|
1667
|
+
);
|
|
1668
|
+
// Except under the VS Code extension, which reads the `cliprompt`
|
|
1669
|
+
// above, asks the user, and answers by restarting this command
|
|
1670
|
+
// with `--force-takeover`. That hand-off has always ended with
|
|
1671
|
+
// this process gone: it used to abort startup before anything
|
|
1672
|
+
// bound a port. Staying up now would leave the replacement
|
|
1673
|
+
// fighting us for :5173.
|
|
1674
|
+
if (process.env.SUPERBLOCKS_VSCODE === "true") {
|
|
1675
|
+
logger.info(
|
|
1676
|
+
"Stopping so the extension can restart with --force-takeover",
|
|
1677
|
+
);
|
|
1678
|
+
stopForExtension = true;
|
|
1679
|
+
}
|
|
1680
|
+
} finally {
|
|
1681
|
+
span.end();
|
|
1682
|
+
}
|
|
1683
|
+
});
|
|
1684
|
+
if (!stopForExtension) {
|
|
1685
|
+
return;
|
|
1686
|
+
}
|
|
1687
|
+
// Outside the span, so `span.end()` above has already run and the
|
|
1688
|
+
// flush below can carry it. This is the same teardown the two
|
|
1689
|
+
// fatal-exit blocks in `dev()` use, and for the same reason: the
|
|
1690
|
+
// `shutdownAndExit()` that used to be inside the `catch` exits
|
|
1691
|
+
// synchronously from its own `finally`, which made both the
|
|
1692
|
+
// `span.end()` and the telemetry flush unreachable — the trace for
|
|
1693
|
+
// the one event this handler exists to record never left the process.
|
|
1694
|
+
try {
|
|
1695
|
+
await runFatalExitShutdown({
|
|
1696
|
+
logger,
|
|
1697
|
+
removeIntegrationCache: () => aiService?.removeIntegrationCache(),
|
|
1698
|
+
relinquishLock: () => lockSvc.relinquishLock(),
|
|
1699
|
+
});
|
|
1700
|
+
} finally {
|
|
1701
|
+
process.exit(1);
|
|
1702
|
+
}
|
|
1703
|
+
});
|
|
1704
|
+
|
|
1638
1705
|
try {
|
|
1639
1706
|
await maskUnixSignals(async () => {
|
|
1640
|
-
|
|
1641
|
-
|
|
1707
|
+
// `--force-takeover` is the one thing that still acquires at boot:
|
|
1708
|
+
// it is an explicit "take it from them now" rather than a pod
|
|
1709
|
+
// helping itself to a lock nobody asked it to hold.
|
|
1710
|
+
if (options.forceTakeover) {
|
|
1642
1711
|
await tracer.startActiveSpan(
|
|
1643
|
-
"
|
|
1712
|
+
"forcingInitialLockTakeover",
|
|
1644
1713
|
async (span) => {
|
|
1645
1714
|
try {
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
"Lock acquired successfully via force takeover.",
|
|
1654
|
-
);
|
|
1655
|
-
} else {
|
|
1656
|
-
await lockSvc.acquireLock();
|
|
1657
|
-
}
|
|
1715
|
+
logger.info(
|
|
1716
|
+
"Force takeover requested, taking over lock...",
|
|
1717
|
+
);
|
|
1718
|
+
await lockSvc.forceTakeover();
|
|
1719
|
+
logger.info(
|
|
1720
|
+
"Lock acquired successfully via force takeover.",
|
|
1721
|
+
);
|
|
1658
1722
|
} catch (error) {
|
|
1659
|
-
// If we got a conflict error (409), inspect the lock for more information
|
|
1660
|
-
if (error instanceof ConflictError) {
|
|
1661
|
-
await handleLockConflict(error, lockSvc, span, logger);
|
|
1662
|
-
return;
|
|
1663
|
-
}
|
|
1664
|
-
|
|
1665
1723
|
logger.error(
|
|
1666
|
-
"Failed to
|
|
1724
|
+
"Failed to force takeover of lock on application",
|
|
1667
1725
|
getErrorMeta(error),
|
|
1668
1726
|
);
|
|
1669
1727
|
span.setStatus({
|
|
@@ -2337,7 +2395,7 @@ export async function dev(options: {
|
|
|
2337
2395
|
try {
|
|
2338
2396
|
logger.info("Releasing lock before restarting the dev server");
|
|
2339
2397
|
await aiService?.removeIntegrationCache();
|
|
2340
|
-
await lockService?.
|
|
2398
|
+
await lockService?.relinquishLock();
|
|
2341
2399
|
} catch (e) {
|
|
2342
2400
|
logger.error(
|
|
2343
2401
|
"Error releasing lock before restarting the dev server",
|
|
@@ -2414,13 +2472,13 @@ export async function dev(options: {
|
|
|
2414
2472
|
}),
|
|
2415
2473
|
);
|
|
2416
2474
|
try {
|
|
2417
|
-
// `
|
|
2418
|
-
// unconditional `finally { process.exit(1) }`; `process.exit`
|
|
2419
|
-
// synchronous, so every statement below it would be dead code
|
|
2475
|
+
// `relinquishLock()`, NOT `shutdownAndExit()`. The latter ends in
|
|
2476
|
+
// an unconditional `finally { process.exit(1) }`; `process.exit`
|
|
2477
|
+
// is synchronous, so every statement below it would be dead code
|
|
2420
2478
|
// whenever `lockService` is set — which is the normal state in a
|
|
2421
2479
|
// SABS pod, exactly the population this exit path targets. The
|
|
2422
|
-
// telemetry flush was silently never running. `
|
|
2423
|
-
// the same teardown without owning the exit, matching how
|
|
2480
|
+
// telemetry flush was silently never running. `relinquishLock()`
|
|
2481
|
+
// does the same teardown without owning the exit, matching how
|
|
2424
2482
|
// `runGracefulShutdown` in dev-server.mts drives it; the
|
|
2425
2483
|
// `finally` below performs the exit instead.
|
|
2426
2484
|
//
|
|
@@ -2431,8 +2489,7 @@ export async function dev(options: {
|
|
|
2431
2489
|
logger,
|
|
2432
2490
|
removeIntegrationCache: () =>
|
|
2433
2491
|
aiService?.removeIntegrationCache(),
|
|
2434
|
-
|
|
2435
|
-
lockService?.shutdown({ serverInitiated: false }),
|
|
2492
|
+
relinquishLock: () => lockService?.relinquishLock(),
|
|
2436
2493
|
});
|
|
2437
2494
|
} finally {
|
|
2438
2495
|
// The only exit on this path now that the lock service no longer
|
|
@@ -2489,15 +2546,14 @@ export async function dev(options: {
|
|
|
2489
2546
|
}),
|
|
2490
2547
|
);
|
|
2491
2548
|
try {
|
|
2492
|
-
// `
|
|
2549
|
+
// `relinquishLock()`, NOT `shutdownAndExit()` — same reason as the
|
|
2493
2550
|
// sync/lock catch above: `shutdownAndExit()` exits the process
|
|
2494
2551
|
// synchronously in its own `finally`, which made everything
|
|
2495
2552
|
// sequenced after it unreachable.
|
|
2496
2553
|
await runFatalExitShutdown({
|
|
2497
2554
|
logger,
|
|
2498
2555
|
removeIntegrationCache: () => aiService?.removeIntegrationCache(),
|
|
2499
|
-
|
|
2500
|
-
lockService?.shutdown({ serverInitiated: false }),
|
|
2556
|
+
relinquishLock: () => lockService?.relinquishLock(),
|
|
2501
2557
|
});
|
|
2502
2558
|
} finally {
|
|
2503
2559
|
// The only exit on this path; here so a thrown shutdown path can't
|
|
@@ -2695,7 +2751,8 @@ export async function dev(options: {
|
|
|
2695
2751
|
logger.warn(`Error removing integration cache: ${error}`);
|
|
2696
2752
|
});
|
|
2697
2753
|
lockService?.shutdownAndExit().catch(() => {
|
|
2698
|
-
//
|
|
2754
|
+
// Redundant — `shutdownAndExit` exits from its own `finally` — but
|
|
2755
|
+
// kept so the process still goes away if that ever stops being true.
|
|
2699
2756
|
process.exit(1);
|
|
2700
2757
|
});
|
|
2701
2758
|
});
|
|
@@ -3094,7 +3151,7 @@ async function ensureRuntimeDbfsBranchConsistency({
|
|
|
3094
3151
|
currentBranchName: string;
|
|
3095
3152
|
}): Promise<string> {
|
|
3096
3153
|
interface BranchSwitchingLockService {
|
|
3097
|
-
|
|
3154
|
+
isLockedWhenSettled(): Promise<boolean>;
|
|
3098
3155
|
switchBranch(
|
|
3099
3156
|
nextBranchName: string,
|
|
3100
3157
|
options?: { reacquireLock?: boolean },
|
|
@@ -3124,7 +3181,8 @@ async function ensureRuntimeDbfsBranchConsistency({
|
|
|
3124
3181
|
|
|
3125
3182
|
const lockSvc = lockService as unknown as BranchSwitchingLockService;
|
|
3126
3183
|
const syncSvc = syncService as unknown as BranchSwitchingSyncService;
|
|
3127
|
-
|
|
3184
|
+
// Retain the pre-switch state for explicit fallback and rollback paths.
|
|
3185
|
+
const wasLocked = await lockSvc.isLockedWhenSettled();
|
|
3128
3186
|
logger.info(
|
|
3129
3187
|
`Switching runtime DBFS branch context from '${currentBranchName}' to '${targetBranchName}'`,
|
|
3130
3188
|
);
|
|
@@ -3132,14 +3190,12 @@ async function ensureRuntimeDbfsBranchConsistency({
|
|
|
3132
3190
|
try {
|
|
3133
3191
|
syncSvc.setBranchName(targetBranchName);
|
|
3134
3192
|
try {
|
|
3135
|
-
await lockSvc.switchBranch(targetBranchName
|
|
3136
|
-
reacquireLock: wasLocked,
|
|
3137
|
-
});
|
|
3193
|
+
await lockSvc.switchBranch(targetBranchName);
|
|
3138
3194
|
} catch (switchError) {
|
|
3139
3195
|
// Disconnecting native git can invalidate the previous live branch
|
|
3140
3196
|
// context. Fall back to explicit release/reacquire on the target branch.
|
|
3141
3197
|
logger.warn(
|
|
3142
|
-
`Lock branch transfer failed for '${currentBranchName}' -> '${targetBranchName}', performing
|
|
3198
|
+
`Lock branch transfer failed for '${currentBranchName}' -> '${targetBranchName}', performing an explicit release and reacquire: ${switchError instanceof Error ? switchError.message : String(switchError)}`,
|
|
3143
3199
|
);
|
|
3144
3200
|
await lockSvc.switchBranch(targetBranchName, { reacquireLock: false });
|
|
3145
3201
|
if (wasLocked) {
|
|
@@ -3156,7 +3212,7 @@ async function ensureRuntimeDbfsBranchConsistency({
|
|
|
3156
3212
|
syncSvc.setBranchName(currentBranchName);
|
|
3157
3213
|
try {
|
|
3158
3214
|
await lockSvc.switchBranch(currentBranchName, {
|
|
3159
|
-
reacquireLock: lockSvc.
|
|
3215
|
+
reacquireLock: (await lockSvc.isLockedWhenSettled()) || wasLocked,
|
|
3160
3216
|
});
|
|
3161
3217
|
} catch (rollbackError) {
|
|
3162
3218
|
logger.warn(
|
|
@@ -69,11 +69,12 @@ import {
|
|
|
69
69
|
type DevServerFailureType,
|
|
70
70
|
devServerMetrics,
|
|
71
71
|
} from "./dev-server-metrics.mjs";
|
|
72
|
+
import { isRecoverableShutdownError, parseIsWarm } from "./fatal-exit.mjs";
|
|
72
73
|
import {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
} from "./fatal-
|
|
74
|
+
fatalAwareExitCode,
|
|
75
|
+
installFatalProcessBarrier,
|
|
76
|
+
type FatalProcessHandler,
|
|
77
|
+
} from "./fatal-process-barrier.mjs";
|
|
77
78
|
import { OPTIMIZE_DEPS_CONFIG } from "./optimize-deps-config.mjs";
|
|
78
79
|
import { prepareInPlaceRestart } from "./prepare-in-place-restart.mjs";
|
|
79
80
|
import { handleRecoverableBuildError } from "./recoverable-build-error.mjs";
|
|
@@ -546,7 +547,7 @@ export function buildStatusPayload<T extends object>(
|
|
|
546
547
|
return { ...base, serverErrors: devServerStatus?.serverErrors ?? [] };
|
|
547
548
|
}
|
|
548
549
|
|
|
549
|
-
type ProcessErrorHandler =
|
|
550
|
+
type ProcessErrorHandler = FatalProcessHandler;
|
|
550
551
|
|
|
551
552
|
// Both listeners log a recoverable shutdown, but with different wording, and
|
|
552
553
|
// these lines are what on-call greps when a pod stays up instead of restarting.
|
|
@@ -557,6 +558,59 @@ const RECOVERABLE_SHUTDOWN_LOG_MESSAGE: Record<ProcessErrorHandler, string> = {
|
|
|
557
558
|
"Ignoring recoverable shutdown rejection (unhandledRejection)",
|
|
558
559
|
};
|
|
559
560
|
|
|
561
|
+
/**
|
|
562
|
+
* What the dev server needs of the lock service on its way out. Structural so a
|
|
563
|
+
* test does not have to stand up a real one.
|
|
564
|
+
*/
|
|
565
|
+
type ExitingLockService = Pick<
|
|
566
|
+
LockService,
|
|
567
|
+
"notifyClosedByServer" | "relinquishLock"
|
|
568
|
+
>;
|
|
569
|
+
|
|
570
|
+
/**
|
|
571
|
+
* Give the AppBranchLock to whoever should have it next, then let the caller
|
|
572
|
+
* exit.
|
|
573
|
+
*
|
|
574
|
+
* Which of the two things to do is the dev server's knowledge, not the lock
|
|
575
|
+
* service's — it used to be a `serverInitiated` flag passed down into a
|
|
576
|
+
* `shutdown()` that branched on it, which is how a lock service came to have an
|
|
577
|
+
* opinion about why a process was stopping.
|
|
578
|
+
*
|
|
579
|
+
* Normally this process is the one stopping, so the lock goes back: the
|
|
580
|
+
* heartbeat dies with the process, and anything not released here makes the
|
|
581
|
+
* replacement sandbox's `acquireLock()` fail until the server-side TTL expires
|
|
582
|
+
* — and the replacement is the entire point of exiting.
|
|
583
|
+
*
|
|
584
|
+
* When the server closed us, the lock is already gone. `/_sb_disconnect` is
|
|
585
|
+
* usually a session moving across local↔cloud, so releasing would take the lock
|
|
586
|
+
* from whoever is taking over. The editor is told instead, because it has to
|
|
587
|
+
* distinguish "the server closed you" from a socket dropping.
|
|
588
|
+
*
|
|
589
|
+
* Extracted so both branches are testable: `runGracefulShutdown` is a closure
|
|
590
|
+
* over a booted dev server, so a unit test cannot otherwise reach either one.
|
|
591
|
+
*/
|
|
592
|
+
export async function handOverLockOnExit(
|
|
593
|
+
lockService: ExitingLockService | undefined,
|
|
594
|
+
{
|
|
595
|
+
serverInitiated,
|
|
596
|
+
switchingTo,
|
|
597
|
+
initiatedByEmail,
|
|
598
|
+
}: {
|
|
599
|
+
serverInitiated: boolean;
|
|
600
|
+
switchingTo?: "local" | "cloud" | "none";
|
|
601
|
+
initiatedByEmail?: string;
|
|
602
|
+
},
|
|
603
|
+
): Promise<void> {
|
|
604
|
+
if (!lockService) {
|
|
605
|
+
return;
|
|
606
|
+
}
|
|
607
|
+
if (serverInitiated) {
|
|
608
|
+
await lockService.notifyClosedByServer({ switchingTo, initiatedByEmail });
|
|
609
|
+
return;
|
|
610
|
+
}
|
|
611
|
+
await lockService.relinquishLock();
|
|
612
|
+
}
|
|
613
|
+
|
|
560
614
|
/**
|
|
561
615
|
* Decides whether a process-level error keeps the pod alive, and performs the
|
|
562
616
|
* recording for the cases that do. Returns `true` when the caller must return
|
|
@@ -871,7 +925,7 @@ export async function createDevServer({
|
|
|
871
925
|
}): Promise<void> {
|
|
872
926
|
try {
|
|
873
927
|
stopMemoryMetrics();
|
|
874
|
-
await lockService
|
|
928
|
+
await handOverLockOnExit(lockService, {
|
|
875
929
|
serverInitiated,
|
|
876
930
|
switchingTo,
|
|
877
931
|
initiatedByEmail,
|
|
@@ -882,16 +936,23 @@ export async function createDevServer({
|
|
|
882
936
|
} catch (e) {
|
|
883
937
|
logger.error(`Error during graceful shutdown: ${e}`);
|
|
884
938
|
} finally {
|
|
939
|
+
// The statement about this process, made by the thing that owns its exit.
|
|
940
|
+
// It used to be the last line of `LockService.shutdown()`, prefixed
|
|
941
|
+
// `[lock-service]` — a lock service announcing a process exit, which is
|
|
942
|
+
// the tangle this change removes.
|
|
943
|
+
//
|
|
944
|
+
// Load bearing: cli-system-tests waits on this exact substring.
|
|
945
|
+
logger.info("[dev-server] shutting down and exiting dev server");
|
|
885
946
|
const closeTimeout = setTimeout(() => {
|
|
886
947
|
logger.info("Dev server closed");
|
|
887
|
-
process.exit(
|
|
948
|
+
process.exit(fatalAwareExitCode());
|
|
888
949
|
}, 1000); // Fallback to exit if the HTTP server doesn't close within 1 second
|
|
889
950
|
|
|
890
951
|
// Then close the HTTP server
|
|
891
952
|
httpServer.close(() => {
|
|
892
953
|
logger.info("Dev server closed");
|
|
893
954
|
clearTimeout(closeTimeout);
|
|
894
|
-
process.exit(
|
|
955
|
+
process.exit(fatalAwareExitCode());
|
|
895
956
|
});
|
|
896
957
|
}
|
|
897
958
|
}
|
|
@@ -1266,18 +1327,11 @@ export async function createDevServer({
|
|
|
1266
1327
|
// even if the pod is activated later.
|
|
1267
1328
|
const isWarm = parseIsWarm(process.env.SUPERBLOCKS_WARM_STANDBY);
|
|
1268
1329
|
|
|
1269
|
-
// Only the first fatal event writes a fatal-exit line and starts shutdown.
|
|
1270
|
-
// Without this, one crash could write several lines: an unhandled rejection
|
|
1271
|
-
// fires once per rejected promise, so a burst of failures (say a dead DB pool
|
|
1272
|
-
// rejecting every in-flight query) would each write a line and each start
|
|
1273
|
-
// shutdown, making one crash look like many in the logs and the crash count.
|
|
1274
|
-
let fatalExitHandled = false;
|
|
1275
|
-
|
|
1276
1330
|
// Signal handlers attach `.catch` so a synchronous throw in
|
|
1277
1331
|
// `runGracefulShutdown` (e.g. logger init or lockService shutdown throwing
|
|
1278
1332
|
// before the first `await`) is logged rather than surfacing as an
|
|
1279
1333
|
// unhandled rejection across N handlers (one per fired signal). Symmetric
|
|
1280
|
-
// with the
|
|
1334
|
+
// with the process barrier below.
|
|
1281
1335
|
process.on("SIGINT", () => {
|
|
1282
1336
|
logger.info("SIGINT received");
|
|
1283
1337
|
gracefulShutdown({
|
|
@@ -1311,86 +1365,37 @@ export async function createDevServer({
|
|
|
1311
1365
|
);
|
|
1312
1366
|
});
|
|
1313
1367
|
|
|
1314
|
-
//
|
|
1315
|
-
//
|
|
1316
|
-
//
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
}
|
|
1321
|
-
if (
|
|
1368
|
+
// Preload may already have armed this from a separate module instance
|
|
1369
|
+
// (node_modules export vs this esbuild bundle). A second call updates
|
|
1370
|
+
// shutdown hooks on the shared host state and does not register another
|
|
1371
|
+
// pair of listeners.
|
|
1372
|
+
installFatalProcessBarrier({
|
|
1373
|
+
isNonFatal: (handler, reason) =>
|
|
1322
1374
|
handleNonFatalProcessError({
|
|
1323
|
-
devServerStatus,
|
|
1324
1375
|
devServerMetrics,
|
|
1325
|
-
handler: "uncaughtException",
|
|
1326
|
-
logger,
|
|
1327
|
-
reason: error,
|
|
1328
|
-
})
|
|
1329
|
-
) {
|
|
1330
|
-
return;
|
|
1331
|
-
}
|
|
1332
|
-
fatalExitHandled = true;
|
|
1333
|
-
// Write the fatal-exit line first, and synchronously, so the reason reaches
|
|
1334
|
-
// the logs before anything else runs. Shutdown below may finish first and
|
|
1335
|
-
// call `process.exit(0)`, so the container's real exit code can be 0 even
|
|
1336
|
-
// though this was a crash. That is why `exit_code=1` here is the code this
|
|
1337
|
-
// handler means to use, not a promise of what the container reports: for the
|
|
1338
|
-
// JS-handler lines, trust that the line exists (and the `signal` field) over
|
|
1339
|
-
// its `exit_code` when comparing against the pod's exit status.
|
|
1340
|
-
logger.error(
|
|
1341
|
-
buildFatalExitLog({ handler: "uncaughtException", exitCode: 1, isWarm }),
|
|
1342
|
-
getErrorMeta(error),
|
|
1343
|
-
);
|
|
1344
|
-
gracefulShutdown({
|
|
1345
|
-
logger,
|
|
1346
|
-
serverInitiated: false,
|
|
1347
|
-
source: "uncaughtException",
|
|
1348
|
-
})
|
|
1349
|
-
.catch((shutdownError) => {
|
|
1350
|
-
logger.error(
|
|
1351
|
-
"Error during shutdown after uncaught exception:",
|
|
1352
|
-
getErrorMeta(shutdownError),
|
|
1353
|
-
);
|
|
1354
|
-
})
|
|
1355
|
-
.finally(() => process.exit(1));
|
|
1356
|
-
});
|
|
1357
|
-
|
|
1358
|
-
// Without its own listener, Node turns an unhandled promise rejection into an
|
|
1359
|
-
// uncaught exception, which would log it under the wrong cause. Handle it here
|
|
1360
|
-
// so a rejection is labelled as a rejection, then exit through the same
|
|
1361
|
-
// shutdown path as an uncaught exception.
|
|
1362
|
-
process.on("unhandledRejection", (reason) => {
|
|
1363
|
-
if (fatalExitHandled) {
|
|
1364
|
-
return;
|
|
1365
|
-
}
|
|
1366
|
-
if (
|
|
1367
|
-
handleNonFatalProcessError({
|
|
1368
1376
|
devServerStatus,
|
|
1369
|
-
|
|
1370
|
-
handler: "unhandledRejection",
|
|
1377
|
+
handler,
|
|
1371
1378
|
logger,
|
|
1372
1379
|
reason,
|
|
1373
|
-
})
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
serverInitiated: false,
|
|
1385
|
-
source: "unhandledRejection",
|
|
1386
|
-
})
|
|
1387
|
-
.catch((shutdownError) => {
|
|
1380
|
+
}),
|
|
1381
|
+
isWarm,
|
|
1382
|
+
log: (message, meta) => logger.error(message, meta),
|
|
1383
|
+
onFatal: async (handler) => {
|
|
1384
|
+
try {
|
|
1385
|
+
await gracefulShutdown({
|
|
1386
|
+
logger,
|
|
1387
|
+
serverInitiated: false,
|
|
1388
|
+
source: handler,
|
|
1389
|
+
});
|
|
1390
|
+
} catch (shutdownError) {
|
|
1388
1391
|
logger.error(
|
|
1389
|
-
|
|
1392
|
+
handler === "uncaughtException"
|
|
1393
|
+
? "Error during shutdown after uncaught exception:"
|
|
1394
|
+
: "Error during shutdown after unhandled rejection:",
|
|
1390
1395
|
getErrorMeta(shutdownError),
|
|
1391
1396
|
);
|
|
1392
|
-
}
|
|
1393
|
-
|
|
1397
|
+
}
|
|
1398
|
+
},
|
|
1394
1399
|
});
|
|
1395
1400
|
|
|
1396
1401
|
if (existingServer) {
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { describe, expect, it, vi } from "vitest";
|
|
2
|
+
|
|
3
|
+
import { handOverLockOnExit } from "./dev-server.mjs";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* ENG-6128. The dev server's exit path used to drive the lock service through a
|
|
7
|
+
* `shutdown({ serverInitiated })` that decided, inside the lock service, whether
|
|
8
|
+
* to release. That flag is the dev server's knowledge, not the lock service's,
|
|
9
|
+
* and getting it wrong in either direction is a live-edit outage: releasing a
|
|
10
|
+
* lock the server has already reassigned takes the app away from the session
|
|
11
|
+
* taking over, and failing to release on a normal exit makes the replacement
|
|
12
|
+
* sandbox wait out the TTL.
|
|
13
|
+
*/
|
|
14
|
+
describe("handOverLockOnExit", () => {
|
|
15
|
+
const makeLockService = () => ({
|
|
16
|
+
notifyClosedByServer: vi.fn(async () => undefined),
|
|
17
|
+
relinquishLock: vi.fn(async () => undefined),
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it("hands the lock back when the dev server is the one stopping", async () => {
|
|
21
|
+
const lockService = makeLockService();
|
|
22
|
+
|
|
23
|
+
await handOverLockOnExit(lockService, { serverInitiated: false });
|
|
24
|
+
|
|
25
|
+
// Prompt release is the whole recovery story for this path: the heartbeat
|
|
26
|
+
// dies with the process, so anything not released here blocks the
|
|
27
|
+
// replacement sandbox for the two-minute TTL.
|
|
28
|
+
expect(lockService.relinquishLock).toHaveBeenCalledOnce();
|
|
29
|
+
expect(lockService.notifyClosedByServer).not.toHaveBeenCalled();
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it("only notifies when the server closed the session", async () => {
|
|
33
|
+
const lockService = makeLockService();
|
|
34
|
+
|
|
35
|
+
await handOverLockOnExit(lockService, {
|
|
36
|
+
serverInitiated: true,
|
|
37
|
+
switchingTo: "local",
|
|
38
|
+
initiatedByEmail: "someone@example.com",
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
// The server has already taken the lock — on a local↔cloud switch it now
|
|
42
|
+
// belongs to the session taking over, so releasing would take it away from
|
|
43
|
+
// them.
|
|
44
|
+
expect(lockService.relinquishLock).not.toHaveBeenCalled();
|
|
45
|
+
expect(lockService.notifyClosedByServer).toHaveBeenCalledWith({
|
|
46
|
+
switchingTo: "local",
|
|
47
|
+
initiatedByEmail: "someone@example.com",
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it("is a no-op when the dev server never had a lock service", async () => {
|
|
52
|
+
// Startup can fail before the lock service exists, and the exit path still
|
|
53
|
+
// runs.
|
|
54
|
+
await expect(
|
|
55
|
+
handOverLockOnExit(undefined, { serverInitiated: false }),
|
|
56
|
+
).resolves.toBeUndefined();
|
|
57
|
+
});
|
|
58
|
+
});
|