@opengeni/worker-bundle 0.23.4-canary.0 → 0.24.2-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +7 -0
- package/dist/activities/agent-turn/model-usage.d.ts +5 -0
- package/dist/activities/agent-turn/quiescence.d.ts +2 -0
- package/dist/activities/agent-turn/run.d.ts +9 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
- package/dist/activities/agent-turn/session-title.d.ts +21 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
- package/dist/activities/agent-turn/turn-context.d.ts +2 -1
- package/dist/activities/agent-turn.d.ts +1 -1
- package/dist/activities/capabilities.d.ts +1 -0
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/{activities-control-SGU2WEVB.js → activities-control-ZWH7J4KZ.js} +102 -94
- package/dist/activities-control-ZWH7J4KZ.js.map +1 -0
- package/dist/{activities-turn-7EAQXW6Q.js → activities-turn-I4OPLQLS.js} +613 -284
- package/dist/activities-turn-I4OPLQLS.js.map +1 -0
- package/dist/{chunk-TJIPCHMP.js → chunk-65JEWP2Q.js} +105 -6
- package/dist/chunk-65JEWP2Q.js.map +1 -0
- package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
- package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
- package/dist/{chunk-S2QVRVJW.js → chunk-RQNH6CGR.js} +28 -41
- package/dist/chunk-RQNH6CGR.js.map +1 -0
- package/dist/index.js +16 -3
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +33 -7
- package/dist/sandbox-reaper-timeout.d.ts +2 -2
- package/dist/sandbox-resume.d.ts +6 -1
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +19 -0
- package/src/activities/agent-turn/claim.ts +9 -1
- package/src/activities/agent-turn/compaction-prep.ts +6 -1
- package/src/activities/agent-turn/finalization.ts +31 -17
- package/src/activities/agent-turn/model-usage.ts +33 -4
- package/src/activities/agent-turn/quiescence.ts +11 -2
- package/src/activities/agent-turn/run.ts +74 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
- package/src/activities/agent-turn/session-title.ts +55 -2
- package/src/activities/agent-turn/stream-attempt.ts +258 -135
- package/src/activities/agent-turn/tool-environment.ts +41 -7
- package/src/activities/agent-turn/turn-context.ts +4 -2
- package/src/activities/agent-turn.ts +5 -1
- package/src/activities/api-integrations.ts +11 -1
- package/src/activities/capabilities.ts +57 -0
- package/src/activities/context-compaction.ts +3 -0
- package/src/activities/goals.ts +1 -0
- package/src/activities/mcp-credentials.ts +53 -0
- package/src/activities/rig-verification.ts +5 -3
- package/src/activities/scheduled-tasks.ts +111 -98
- package/src/index.ts +15 -0
- package/src/observability-metrics.ts +152 -14
- package/src/sandbox-reaper-timeout.ts +10 -4
- package/src/sandbox-resume.ts +58 -8
- package/dist/activities-control-SGU2WEVB.js.map +0 -1
- package/dist/activities-turn-7EAQXW6Q.js.map +0 -1
- package/dist/chunk-S2QVRVJW.js.map +0 -1
- package/dist/chunk-TJIPCHMP.js.map +0 -1
|
@@ -34,6 +34,10 @@ export type SessionRecoveryBacklog = {
|
|
|
34
34
|
quiescence_missing: number;
|
|
35
35
|
projection_stale: number;
|
|
36
36
|
};
|
|
37
|
+
export type ContextCompactionPendingSummary = {
|
|
38
|
+
pendingCount: number;
|
|
39
|
+
oldestStartedAt: Date | null;
|
|
40
|
+
};
|
|
37
41
|
|
|
38
42
|
export type TemporalTurnTaskQueueStats = {
|
|
39
43
|
approximateBacklogCount?: unknown;
|
|
@@ -49,6 +53,7 @@ const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
|
49
53
|
const modelRequestTrackers = new WeakMap<Observability, ModelRequestLifecycleMetrics>();
|
|
50
54
|
const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
|
|
51
55
|
const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
|
|
56
|
+
const initializedContextCompactionMetrics = new WeakSet<Observability>();
|
|
52
57
|
|
|
53
58
|
// A worker process reaching one trillion tokens in one provider/cache counter is
|
|
54
59
|
// already far outside an ordinary scrape lifetime. Refuse further increments
|
|
@@ -61,6 +66,18 @@ const WORKER_DEATH_RECOVERY_OUTCOMES: readonly WorkerDeathRecoveryOutcome[] = [
|
|
|
61
66
|
"exhausted",
|
|
62
67
|
];
|
|
63
68
|
const WORKER_DEATH_TIMEOUT_TYPES = ["heartbeat", "schedule_to_start"] as const;
|
|
69
|
+
const CONTEXT_COMPACTION_TRIGGERS = ["auto", "operator", "proactive", "overflow"] as const;
|
|
70
|
+
|
|
71
|
+
export type ContextCompactionTrigger = (typeof CONTEXT_COMPACTION_TRIGGERS)[number];
|
|
72
|
+
|
|
73
|
+
const CONTEXT_COMPACTION_STARTS_METRIC = {
|
|
74
|
+
name: "opengeni_context_compaction_starts_total",
|
|
75
|
+
help: "Total durable context compaction starts, by trigger.",
|
|
76
|
+
} as const;
|
|
77
|
+
const CONTEXT_COMPACTIONS_METRIC = {
|
|
78
|
+
name: "opengeni_context_compactions_total",
|
|
79
|
+
help: "Total completed context compactions, by trigger.",
|
|
80
|
+
} as const;
|
|
64
81
|
|
|
65
82
|
export function observabilityEventLogger(observability: Observability): EventLogger {
|
|
66
83
|
return {
|
|
@@ -346,6 +363,29 @@ export function initializeWorkerOutcomeMetrics(observability: Observability): vo
|
|
|
346
363
|
}
|
|
347
364
|
}
|
|
348
365
|
|
|
366
|
+
/**
|
|
367
|
+
* Publish every bounded compaction lifecycle series before the first event.
|
|
368
|
+
* The zero counters make the closed trigger catalog explicit before a scrape.
|
|
369
|
+
* Alerting uses the separate durable control-worker projection below; these
|
|
370
|
+
* process-local counters are lifecycle-rate diagnostics only.
|
|
371
|
+
*/
|
|
372
|
+
export function initializeContextCompactionMetrics(observability: Observability): void {
|
|
373
|
+
if (initializedContextCompactionMetrics.has(observability)) return;
|
|
374
|
+
for (const trigger of CONTEXT_COMPACTION_TRIGGERS) {
|
|
375
|
+
observability.incrementCounter({
|
|
376
|
+
...CONTEXT_COMPACTION_STARTS_METRIC,
|
|
377
|
+
labels: { trigger },
|
|
378
|
+
amount: 0,
|
|
379
|
+
});
|
|
380
|
+
observability.incrementCounter({
|
|
381
|
+
...CONTEXT_COMPACTIONS_METRIC,
|
|
382
|
+
labels: { trigger },
|
|
383
|
+
amount: 0,
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
initializedContextCompactionMetrics.add(observability);
|
|
387
|
+
}
|
|
388
|
+
|
|
349
389
|
export function recordWorkerDeathRecoveryMetrics(
|
|
350
390
|
observability: Observability,
|
|
351
391
|
input: {
|
|
@@ -830,6 +870,96 @@ export function startSessionRecoveryMonitor(input: {
|
|
|
830
870
|
};
|
|
831
871
|
}
|
|
832
872
|
|
|
873
|
+
export function recordContextCompactionPendingGauges(
|
|
874
|
+
observability: Observability,
|
|
875
|
+
summary: ContextCompactionPendingSummary,
|
|
876
|
+
nowMs = Date.now(),
|
|
877
|
+
): void {
|
|
878
|
+
const oldestStartedAtMs = summary.oldestStartedAt?.getTime();
|
|
879
|
+
const oldestPendingAgeSeconds =
|
|
880
|
+
oldestStartedAtMs === undefined || !Number.isFinite(oldestStartedAtMs)
|
|
881
|
+
? 0
|
|
882
|
+
: Math.max(0, (nowMs - oldestStartedAtMs) / 1_000);
|
|
883
|
+
observability.setGauge({
|
|
884
|
+
name: "opengeni_context_compaction_pending",
|
|
885
|
+
help: "Current exact active attempts whose latest automatic compaction landmark is still started.",
|
|
886
|
+
value: nonnegativeFinite(summary.pendingCount),
|
|
887
|
+
});
|
|
888
|
+
observability.setGauge({
|
|
889
|
+
name: "opengeni_context_compaction_oldest_pending_age_seconds",
|
|
890
|
+
help: "Age in seconds of the oldest durably pending automatic compaction, or zero when none are pending.",
|
|
891
|
+
value: oldestPendingAgeSeconds,
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
export function startContextCompactionPendingMonitor(input: {
|
|
896
|
+
observability: Observability;
|
|
897
|
+
read: () => Promise<ContextCompactionPendingSummary>;
|
|
898
|
+
intervalMs?: number;
|
|
899
|
+
now?: () => number;
|
|
900
|
+
}): { close: () => Promise<void> } {
|
|
901
|
+
const intervalMs = input.intervalMs ?? 60_000;
|
|
902
|
+
const now = input.now ?? Date.now;
|
|
903
|
+
const startedAt = now();
|
|
904
|
+
let lastSuccessAt: number | null = null;
|
|
905
|
+
let lastReadSucceeded = false;
|
|
906
|
+
let stopped = false;
|
|
907
|
+
let running: Promise<void> | null = null;
|
|
908
|
+
const recordStatus = () => {
|
|
909
|
+
const observedAt = now();
|
|
910
|
+
const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
|
|
911
|
+
const set = (name: string, help: string, value: number) =>
|
|
912
|
+
input.observability.setGauge({ name, help, value });
|
|
913
|
+
set(
|
|
914
|
+
"opengeni_context_compaction_monitor_last_read_success",
|
|
915
|
+
"Whether the latest durable context-compaction aggregate read completed successfully.",
|
|
916
|
+
lastReadSucceeded ? 1 : 0,
|
|
917
|
+
);
|
|
918
|
+
set(
|
|
919
|
+
"opengeni_context_compaction_monitor_last_success_timestamp_seconds",
|
|
920
|
+
"Unix timestamp of the latest successful durable context-compaction aggregate read, or zero before one succeeds.",
|
|
921
|
+
lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
|
|
922
|
+
);
|
|
923
|
+
set(
|
|
924
|
+
"opengeni_context_compaction_monitor_fresh",
|
|
925
|
+
"Whether durable context-compaction pending gauges have a successful read within three monitor intervals.",
|
|
926
|
+
lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
|
|
927
|
+
);
|
|
928
|
+
};
|
|
929
|
+
const refresh = () => {
|
|
930
|
+
recordStatus();
|
|
931
|
+
if (stopped || running) return;
|
|
932
|
+
running = input
|
|
933
|
+
.read()
|
|
934
|
+
.then((summary) => {
|
|
935
|
+
recordContextCompactionPendingGauges(input.observability, summary, now());
|
|
936
|
+
lastSuccessAt = now();
|
|
937
|
+
lastReadSucceeded = true;
|
|
938
|
+
recordStatus();
|
|
939
|
+
})
|
|
940
|
+
.catch((error) => {
|
|
941
|
+
lastReadSucceeded = false;
|
|
942
|
+
recordStatus();
|
|
943
|
+
input.observability.warn("context compaction monitor: durable aggregate read failed", {
|
|
944
|
+
error: error instanceof Error ? error.message : String(error),
|
|
945
|
+
});
|
|
946
|
+
})
|
|
947
|
+
.finally(() => {
|
|
948
|
+
running = null;
|
|
949
|
+
});
|
|
950
|
+
};
|
|
951
|
+
refresh();
|
|
952
|
+
const timer = setInterval(refresh, intervalMs);
|
|
953
|
+
timer.unref?.();
|
|
954
|
+
return {
|
|
955
|
+
close: async () => {
|
|
956
|
+
stopped = true;
|
|
957
|
+
clearInterval(timer);
|
|
958
|
+
await running;
|
|
959
|
+
},
|
|
960
|
+
};
|
|
961
|
+
}
|
|
962
|
+
|
|
833
963
|
export function recordSandboxLeaseGauges(
|
|
834
964
|
observability: Observability,
|
|
835
965
|
counts: Partial<Record<SandboxLeaseLiveness, number>>,
|
|
@@ -1751,15 +1881,14 @@ export function recordSessionEventPublishLatency(
|
|
|
1751
1881
|
});
|
|
1752
1882
|
}
|
|
1753
1883
|
|
|
1754
|
-
// Context tokens per response span a wide range; buckets
|
|
1755
|
-
//
|
|
1884
|
+
// Context tokens per response span a wide range; buckets retain context-pressure
|
|
1885
|
+
// diagnostics while compaction alerting follows the durable model-aware start.
|
|
1756
1886
|
const MODEL_INPUT_TOKENS_BUCKETS = [
|
|
1757
1887
|
1_000, 5_000, 10_000, 25_000, 50_000, 100_000, 150_000, 200_000, 300_000, 500_000, 1_000_000,
|
|
1758
1888
|
];
|
|
1759
1889
|
|
|
1760
|
-
/** Observed input (context) tokens per model response
|
|
1761
|
-
*
|
|
1762
|
-
* "compaction never firing while contexts run hot" failure mode expressible. */
|
|
1890
|
+
/** Observed input (context) tokens per model response for context-pressure
|
|
1891
|
+
* dashboards and provider diagnostics. */
|
|
1763
1892
|
export function recordModelInputTokens(
|
|
1764
1893
|
observability: Observability,
|
|
1765
1894
|
provider: string,
|
|
@@ -1778,15 +1907,24 @@ export function recordModelInputTokens(
|
|
|
1778
1907
|
});
|
|
1779
1908
|
}
|
|
1780
1909
|
|
|
1781
|
-
/** A context compaction
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1910
|
+
/** A context compaction successfully completed, by bounded trigger. */
|
|
1911
|
+
export function recordContextCompaction(
|
|
1912
|
+
observability: Observability,
|
|
1913
|
+
trigger: ContextCompactionTrigger,
|
|
1914
|
+
): void {
|
|
1915
|
+
observability.incrementCounter({ ...CONTEXT_COMPACTIONS_METRIC, labels: { trigger } });
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
/**
|
|
1919
|
+
* A durable context-compaction start, recorded only after the attempt-fenced
|
|
1920
|
+
* `compaction.started` transaction commits. This is the model-aware threshold
|
|
1921
|
+
* signal; it does not guess from a static token count shared by unlike models.
|
|
1922
|
+
*/
|
|
1923
|
+
export function recordContextCompactionStarted(
|
|
1924
|
+
observability: Observability,
|
|
1925
|
+
trigger: ContextCompactionTrigger,
|
|
1926
|
+
): void {
|
|
1927
|
+
observability.incrementCounter({ ...CONTEXT_COMPACTION_STARTS_METRIC, labels: { trigger } });
|
|
1790
1928
|
}
|
|
1791
1929
|
|
|
1792
1930
|
const MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS = [1, 8, 32, 128, 512, 2_048, 8_192];
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,
|
|
3
3
|
SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS,
|
|
4
|
+
effectiveSandboxDrainSnapshotTimeoutMs,
|
|
4
5
|
sandboxArchiveCaptureTimeoutMs,
|
|
5
6
|
type Settings,
|
|
6
7
|
} from "@opengeni/config";
|
|
@@ -13,16 +14,21 @@ import {
|
|
|
13
14
|
|
|
14
15
|
const STRICT_TIMEOUT_FENCE_MS = 1;
|
|
15
16
|
|
|
16
|
-
export function sandboxDrainTiming(
|
|
17
|
+
export function sandboxDrainTiming(
|
|
18
|
+
settings: Pick<Settings, "sandboxSnapshotTimeoutMs" | "sandboxDrainSnapshotTimeoutMs">,
|
|
19
|
+
): {
|
|
17
20
|
snapshotTimeoutMs: number;
|
|
18
21
|
captureTimeoutMs: number;
|
|
19
22
|
timeoutClass: SandboxDrainTimeoutClass;
|
|
20
23
|
activityTimeoutMs: number;
|
|
21
24
|
} {
|
|
22
|
-
const
|
|
25
|
+
const snapshotTimeoutMs = effectiveSandboxDrainSnapshotTimeoutMs(settings);
|
|
26
|
+
const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs({
|
|
27
|
+
sandboxSnapshotTimeoutMs: snapshotTimeoutMs,
|
|
28
|
+
});
|
|
23
29
|
const timeoutClass = sandboxDrainTimeoutClass(captureTimeoutMs);
|
|
24
30
|
return {
|
|
25
|
-
snapshotTimeoutMs
|
|
31
|
+
snapshotTimeoutMs,
|
|
26
32
|
captureTimeoutMs,
|
|
27
33
|
timeoutClass,
|
|
28
34
|
activityTimeoutMs: sandboxDrainActivityTimeoutMs(timeoutClass),
|
|
@@ -30,7 +36,7 @@ export function sandboxDrainTiming(settings: Pick<Settings, "sandboxSnapshotTime
|
|
|
30
36
|
}
|
|
31
37
|
|
|
32
38
|
export function assertSandboxReaperActivityTimeout(
|
|
33
|
-
settings: Pick<Settings, "sandboxSnapshotTimeoutMs">,
|
|
39
|
+
settings: Pick<Settings, "sandboxSnapshotTimeoutMs" | "sandboxDrainSnapshotTimeoutMs">,
|
|
34
40
|
): void {
|
|
35
41
|
const timing = sandboxDrainTiming(settings);
|
|
36
42
|
if (timing.snapshotTimeoutMs >= timing.captureTimeoutMs) {
|
package/src/sandbox-resume.ts
CHANGED
|
@@ -709,7 +709,13 @@ export async function persistSandboxDeadlineRotationCheckpoint(
|
|
|
709
709
|
* the user's machine IS the persistence), or a snapshot newer than the
|
|
710
710
|
* interval already rides the lease.
|
|
711
711
|
*/
|
|
712
|
-
export
|
|
712
|
+
export type WarmWorkspaceSnapshotPromise = Promise<boolean> & {
|
|
713
|
+
/** Settles only after any provider capture continuation has published or
|
|
714
|
+
* failed and released its exact workspace-capture admission. */
|
|
715
|
+
readonly settled: Promise<void>;
|
|
716
|
+
};
|
|
717
|
+
|
|
718
|
+
export function maybePersistWarmWorkspaceSnapshot(
|
|
713
719
|
services: SandboxResumeServices,
|
|
714
720
|
ids: {
|
|
715
721
|
accountId: string;
|
|
@@ -723,6 +729,50 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
723
729
|
leaseEpoch: number,
|
|
724
730
|
signal?: AbortSignal,
|
|
725
731
|
force = false,
|
|
732
|
+
): WarmWorkspaceSnapshotPromise {
|
|
733
|
+
let continuationRegistered = false;
|
|
734
|
+
let resolvePhysicalSettlement: () => void = () => undefined;
|
|
735
|
+
const physicalSettlement = new Promise<void>((resolve) => {
|
|
736
|
+
resolvePhysicalSettlement = resolve;
|
|
737
|
+
});
|
|
738
|
+
const result = persistWarmWorkspaceSnapshot(
|
|
739
|
+
services,
|
|
740
|
+
ids,
|
|
741
|
+
session,
|
|
742
|
+
leaseEpoch,
|
|
743
|
+
signal,
|
|
744
|
+
force,
|
|
745
|
+
(continuation) => {
|
|
746
|
+
continuationRegistered = true;
|
|
747
|
+
void continuation.then(resolvePhysicalSettlement, resolvePhysicalSettlement);
|
|
748
|
+
},
|
|
749
|
+
);
|
|
750
|
+
void result.then(
|
|
751
|
+
() => {
|
|
752
|
+
if (!continuationRegistered) resolvePhysicalSettlement();
|
|
753
|
+
},
|
|
754
|
+
() => {
|
|
755
|
+
if (!continuationRegistered) resolvePhysicalSettlement();
|
|
756
|
+
},
|
|
757
|
+
);
|
|
758
|
+
return Object.assign(result, { settled: physicalSettlement });
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
async function persistWarmWorkspaceSnapshot(
|
|
762
|
+
services: SandboxResumeServices,
|
|
763
|
+
ids: {
|
|
764
|
+
accountId: string;
|
|
765
|
+
workspaceId: string;
|
|
766
|
+
sessionId: string;
|
|
767
|
+
turnId: string;
|
|
768
|
+
attemptId: string;
|
|
769
|
+
sandboxGroupId: string;
|
|
770
|
+
},
|
|
771
|
+
session: unknown,
|
|
772
|
+
leaseEpoch: number,
|
|
773
|
+
signal: AbortSignal | undefined,
|
|
774
|
+
force: boolean,
|
|
775
|
+
registerOwnedContinuation: (continuation: Promise<void>) => void,
|
|
726
776
|
): Promise<boolean> {
|
|
727
777
|
const { db, settings } = services;
|
|
728
778
|
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
@@ -989,6 +1039,7 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
989
1039
|
return { kind: "settled" as const, persisted: false };
|
|
990
1040
|
},
|
|
991
1041
|
);
|
|
1042
|
+
registerOwnedContinuation(settled.then(() => undefined));
|
|
992
1043
|
let cancelListener: (() => void) | undefined;
|
|
993
1044
|
const outcome = await Promise.race([
|
|
994
1045
|
settled,
|
|
@@ -1223,13 +1274,12 @@ export async function resumeBoxForTurn(
|
|
|
1223
1274
|
// holder is gone / its attempt is no longer the active writer" from "the lease
|
|
1224
1275
|
// could not be extended" (rotation requested, draining, another epoch). The
|
|
1225
1276
|
// second is NOT a reason to release: once a provider-deadline rotation is
|
|
1226
|
-
// requested the
|
|
1227
|
-
//
|
|
1228
|
-
//
|
|
1229
|
-
//
|
|
1230
|
-
//
|
|
1231
|
-
//
|
|
1232
|
-
// "holder alive -> never release"; this loop keeps touching the holder.
|
|
1277
|
+
// requested the turn-side heartbeat immediately preempts the attempt, whose
|
|
1278
|
+
// finalizer retains this holder until every attempt-owned writer is physically
|
|
1279
|
+
// drained. Dropping it here would let the reaper capture or terminate while a
|
|
1280
|
+
// provider operation may still be mutating /workspace. Both loops therefore
|
|
1281
|
+
// converge on "holder alive -> never release"; finalization performs the
|
|
1282
|
+
// proof-bearing release.
|
|
1233
1283
|
const releaseDeadHolder = (reason: string): void => {
|
|
1234
1284
|
const message = "sandbox resume holder released: holder no longer live";
|
|
1235
1285
|
const fields = {
|