@opengeni/worker-bundle 0.23.4 → 0.24.2-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +17 -2
- package/dist/activities/agent-turn/claim.d.ts +1 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -2
- package/dist/activities/agent-turn/list-models.d.ts +16 -0
- package/dist/activities/agent-turn/model-usage.d.ts +15 -1
- package/dist/activities/agent-turn/quiescence.d.ts +2 -0
- package/dist/activities/agent-turn/run.d.ts +9 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
- package/dist/activities/agent-turn/session-title.d.ts +21 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +2 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +36 -5
- package/dist/activities/agent-turn/turn-context.d.ts +4 -1
- package/dist/activities/agent-turn.d.ts +1 -1
- package/dist/activities/automations.d.ts +1 -0
- package/dist/activities/capabilities.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/activities/goals.d.ts +11 -0
- package/dist/activities/image-generation-references.d.ts +2 -1
- package/dist/activities/scheduled-tasks.d.ts +1 -0
- package/dist/activities/types.d.ts +8 -0
- package/dist/{activities-control-52W5IDWM.js → activities-control-ZWH7J4KZ.js} +297 -180
- package/dist/activities-control-ZWH7J4KZ.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-A2G2KLSY.js → activities-turn-I4OPLQLS.js} +898 -333
- package/dist/activities-turn-I4OPLQLS.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-GFNXVZVQ.js → chunk-65JEWP2Q.js} +107 -7
- package/dist/chunk-65JEWP2Q.js.map +1 -0
- package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
- package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
- package/dist/{chunk-IPVMCAW5.js → chunk-RQNH6CGR.js} +111 -62
- package/dist/chunk-RQNH6CGR.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +43 -7
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +34 -7
- package/dist/sandbox-reaper-timeout.d.ts +2 -2
- package/dist/sandbox-resume.d.ts +6 -1
- package/dist/workflow-bundle.js +26 -3
- package/dist/workflows/activities.d.ts +5 -0
- package/dist/workflows/automations.d.ts +5 -0
- package/package.json +17 -17
- package/src/activities/agent-run-admission.ts +7 -1
- package/src/activities/agent-turn/admission.ts +47 -6
- package/src/activities/agent-turn/claim.ts +33 -5
- package/src/activities/agent-turn/codex.ts +2 -2
- package/src/activities/agent-turn/compaction-prep.ts +8 -1
- package/src/activities/agent-turn/finalization.ts +31 -17
- package/src/activities/agent-turn/list-models.ts +101 -0
- package/src/activities/agent-turn/model-usage.ts +120 -45
- package/src/activities/agent-turn/quiescence.ts +11 -2
- package/src/activities/agent-turn/run.ts +80 -4
- package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
- package/src/activities/agent-turn/session-title.ts +55 -2
- package/src/activities/agent-turn/stream-attempt.ts +266 -135
- package/src/activities/agent-turn/tool-environment.ts +102 -7
- package/src/activities/agent-turn/tool-policy.ts +58 -5
- package/src/activities/agent-turn/turn-context.ts +8 -2
- package/src/activities/agent-turn.ts +5 -1
- package/src/activities/api-integrations.ts +11 -1
- package/src/activities/automations.ts +122 -67
- package/src/activities/capabilities.ts +129 -2
- package/src/activities/context-compaction.ts +3 -0
- package/src/activities/goals.ts +108 -31
- package/src/activities/image-generation-references.ts +9 -9
- package/src/activities/mcp-credentials.ts +53 -0
- package/src/activities/rig-verification.ts +5 -3
- package/src/activities/scheduled-tasks.ts +217 -110
- package/src/activities/site-auth-maintenance.ts +12 -12
- package/src/activities/types.ts +6 -0
- package/src/activities.ts +1 -0
- package/src/activity-services.ts +4 -1
- package/src/github-rest-mcp.ts +2 -1
- package/src/index.ts +34 -4
- package/src/observability-metrics.ts +154 -14
- package/src/sandbox-reaper-timeout.ts +10 -4
- package/src/sandbox-resume.ts +58 -8
- package/src/workflows/activities.ts +15 -0
- package/src/workflows/automations.ts +21 -2
- package/dist/activities-control-52W5IDWM.js.map +0 -1
- package/dist/activities-turn-A2G2KLSY.js.map +0 -1
- package/dist/chunk-GFNXVZVQ.js.map +0 -1
- package/dist/chunk-IPVMCAW5.js.map +0 -1
package/src/index.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
assertRuntimeDatabasePosture,
|
|
12
12
|
countSessionRecoveryBacklog,
|
|
13
13
|
createDb,
|
|
14
|
+
getContextCompactionPendingSummary,
|
|
14
15
|
markSessionWorkflowWakeDelivered,
|
|
15
16
|
type Database,
|
|
16
17
|
type RuntimeDatabasePostureOptions,
|
|
@@ -56,14 +57,17 @@ import {
|
|
|
56
57
|
type WorkerLifecycleState,
|
|
57
58
|
} from "./http";
|
|
58
59
|
import {
|
|
60
|
+
initializeContextCompactionMetrics,
|
|
59
61
|
initializeWorkerOutcomeMetrics,
|
|
60
62
|
normalizeTurnTaskQueueStats,
|
|
61
63
|
observabilityEventLogger,
|
|
64
|
+
startContextCompactionPendingMonitor,
|
|
62
65
|
startSessionRecoveryMonitor,
|
|
63
66
|
startTurnCapacityMonitor,
|
|
64
67
|
type TurnTaskQueueStats,
|
|
65
68
|
} from "./observability-metrics";
|
|
66
69
|
import {
|
|
70
|
+
resolveCatalogSettings,
|
|
67
71
|
SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
68
72
|
SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
69
73
|
SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
|
|
@@ -224,6 +228,9 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
224
228
|
options.activityDependencies?.observability ??
|
|
225
229
|
createObservability(settings, { component: `worker-${options.role}` });
|
|
226
230
|
initializeWorkerOutcomeMetrics(observability);
|
|
231
|
+
if (options.role === "turn") {
|
|
232
|
+
initializeContextCompactionMetrics(observability);
|
|
233
|
+
}
|
|
227
234
|
if (options.role === "turn" && options.workflowBundle) {
|
|
228
235
|
throw new Error("workflowBundle is valid only for the control worker role");
|
|
229
236
|
}
|
|
@@ -799,11 +806,24 @@ export async function createOpenGeniWorkerService(
|
|
|
799
806
|
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
800
807
|
let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
|
|
801
808
|
let sessionRecoveryMonitor: ReturnType<typeof startSessionRecoveryMonitor> | undefined;
|
|
809
|
+
let contextCompactionPendingMonitor:
|
|
810
|
+
| ReturnType<typeof startContextCompactionPendingMonitor>
|
|
811
|
+
| undefined;
|
|
802
812
|
const schedules: Array<{ close: () => Promise<void> }> = [];
|
|
803
813
|
let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
|
|
804
814
|
let memoryPressureGuard: TurnWorkerMemoryPressureGuard | undefined;
|
|
805
815
|
|
|
806
816
|
try {
|
|
817
|
+
const resolvedCatalog = await retryStartupDependency(
|
|
818
|
+
"model catalog",
|
|
819
|
+
() => resolveCatalogSettings(options.activityDependencies.db, settings),
|
|
820
|
+
{ ...retryOptions, onRetry },
|
|
821
|
+
);
|
|
822
|
+
observability.info("OpenGeni model catalog resolved", {
|
|
823
|
+
role: options.role,
|
|
824
|
+
catalogSource: resolvedCatalog.source,
|
|
825
|
+
catalogVersion: resolvedCatalog.version,
|
|
826
|
+
});
|
|
807
827
|
const needsSignaler =
|
|
808
828
|
options.role === "turn" ||
|
|
809
829
|
!options.activityDependencies.wakeSessionWorkflow ||
|
|
@@ -877,6 +897,10 @@ export async function createOpenGeniWorkerService(
|
|
|
877
897
|
observability,
|
|
878
898
|
read: async () => await countSessionRecoveryBacklog(options.activityDependencies.db),
|
|
879
899
|
});
|
|
900
|
+
contextCompactionPendingMonitor = startContextCompactionPendingMonitor({
|
|
901
|
+
observability,
|
|
902
|
+
read: async () => await getContextCompactionPendingSummary(options.activityDependencies.db),
|
|
903
|
+
});
|
|
880
904
|
}
|
|
881
905
|
|
|
882
906
|
if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
|
|
@@ -911,14 +935,18 @@ export async function createOpenGeniWorkerService(
|
|
|
911
935
|
}
|
|
912
936
|
|
|
913
937
|
if (options.http !== false) {
|
|
938
|
+
const databaseReady = dbReadyCheck(
|
|
939
|
+
options.http?.readinessDb ?? options.activityDependencies.db,
|
|
940
|
+
options.databasePosture,
|
|
941
|
+
);
|
|
914
942
|
httpServer = startWorkerHttpServer({
|
|
915
943
|
settings,
|
|
916
944
|
observability,
|
|
917
945
|
checks: {
|
|
918
|
-
db:
|
|
919
|
-
|
|
920
|
-
options.
|
|
921
|
-
|
|
946
|
+
db: async () => {
|
|
947
|
+
await databaseReady();
|
|
948
|
+
await resolveCatalogSettings(options.activityDependencies.db, settings);
|
|
949
|
+
},
|
|
922
950
|
nats: natsReadyCheck(options.activityDependencies.bus),
|
|
923
951
|
temporal: temporalReadyCheck(workerBundle.connection),
|
|
924
952
|
},
|
|
@@ -931,6 +959,7 @@ export async function createOpenGeniWorkerService(
|
|
|
931
959
|
await Promise.allSettled([
|
|
932
960
|
turnCapacityMonitor?.close(),
|
|
933
961
|
sessionRecoveryMonitor?.close(),
|
|
962
|
+
contextCompactionPendingMonitor?.close(),
|
|
934
963
|
workerBundle?.connection.close(),
|
|
935
964
|
signaler?.close(),
|
|
936
965
|
...schedules.map((schedule) => schedule.close()),
|
|
@@ -954,6 +983,7 @@ export async function createOpenGeniWorkerService(
|
|
|
954
983
|
await Promise.allSettled([
|
|
955
984
|
turnCapacityMonitor?.close(),
|
|
956
985
|
sessionRecoveryMonitor?.close(),
|
|
986
|
+
contextCompactionPendingMonitor?.close(),
|
|
957
987
|
activeWorkerBundle.connection.close(),
|
|
958
988
|
activeSignaler?.close(),
|
|
959
989
|
...schedules.map((schedule) => schedule.close()),
|
|
@@ -34,6 +34,10 @@ export type SessionRecoveryBacklog = {
|
|
|
34
34
|
quiescence_missing: number;
|
|
35
35
|
projection_stale: number;
|
|
36
36
|
};
|
|
37
|
+
export type ContextCompactionPendingSummary = {
|
|
38
|
+
pendingCount: number;
|
|
39
|
+
oldestStartedAt: Date | null;
|
|
40
|
+
};
|
|
37
41
|
|
|
38
42
|
export type TemporalTurnTaskQueueStats = {
|
|
39
43
|
approximateBacklogCount?: unknown;
|
|
@@ -49,6 +53,7 @@ const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
|
49
53
|
const modelRequestTrackers = new WeakMap<Observability, ModelRequestLifecycleMetrics>();
|
|
50
54
|
const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
|
|
51
55
|
const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
|
|
56
|
+
const initializedContextCompactionMetrics = new WeakSet<Observability>();
|
|
52
57
|
|
|
53
58
|
// A worker process reaching one trillion tokens in one provider/cache counter is
|
|
54
59
|
// already far outside an ordinary scrape lifetime. Refuse further increments
|
|
@@ -61,6 +66,18 @@ const WORKER_DEATH_RECOVERY_OUTCOMES: readonly WorkerDeathRecoveryOutcome[] = [
|
|
|
61
66
|
"exhausted",
|
|
62
67
|
];
|
|
63
68
|
const WORKER_DEATH_TIMEOUT_TYPES = ["heartbeat", "schedule_to_start"] as const;
|
|
69
|
+
const CONTEXT_COMPACTION_TRIGGERS = ["auto", "operator", "proactive", "overflow"] as const;
|
|
70
|
+
|
|
71
|
+
export type ContextCompactionTrigger = (typeof CONTEXT_COMPACTION_TRIGGERS)[number];
|
|
72
|
+
|
|
73
|
+
const CONTEXT_COMPACTION_STARTS_METRIC = {
|
|
74
|
+
name: "opengeni_context_compaction_starts_total",
|
|
75
|
+
help: "Total durable context compaction starts, by trigger.",
|
|
76
|
+
} as const;
|
|
77
|
+
const CONTEXT_COMPACTIONS_METRIC = {
|
|
78
|
+
name: "opengeni_context_compactions_total",
|
|
79
|
+
help: "Total completed context compactions, by trigger.",
|
|
80
|
+
} as const;
|
|
64
81
|
|
|
65
82
|
export function observabilityEventLogger(observability: Observability): EventLogger {
|
|
66
83
|
return {
|
|
@@ -346,6 +363,29 @@ export function initializeWorkerOutcomeMetrics(observability: Observability): vo
|
|
|
346
363
|
}
|
|
347
364
|
}
|
|
348
365
|
|
|
366
|
+
/**
|
|
367
|
+
* Publish every bounded compaction lifecycle series before the first event.
|
|
368
|
+
* The zero counters make the closed trigger catalog explicit before a scrape.
|
|
369
|
+
* Alerting uses the separate durable control-worker projection below; these
|
|
370
|
+
* process-local counters are lifecycle-rate diagnostics only.
|
|
371
|
+
*/
|
|
372
|
+
export function initializeContextCompactionMetrics(observability: Observability): void {
|
|
373
|
+
if (initializedContextCompactionMetrics.has(observability)) return;
|
|
374
|
+
for (const trigger of CONTEXT_COMPACTION_TRIGGERS) {
|
|
375
|
+
observability.incrementCounter({
|
|
376
|
+
...CONTEXT_COMPACTION_STARTS_METRIC,
|
|
377
|
+
labels: { trigger },
|
|
378
|
+
amount: 0,
|
|
379
|
+
});
|
|
380
|
+
observability.incrementCounter({
|
|
381
|
+
...CONTEXT_COMPACTIONS_METRIC,
|
|
382
|
+
labels: { trigger },
|
|
383
|
+
amount: 0,
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
initializedContextCompactionMetrics.add(observability);
|
|
387
|
+
}
|
|
388
|
+
|
|
349
389
|
export function recordWorkerDeathRecoveryMetrics(
|
|
350
390
|
observability: Observability,
|
|
351
391
|
input: {
|
|
@@ -830,6 +870,96 @@ export function startSessionRecoveryMonitor(input: {
|
|
|
830
870
|
};
|
|
831
871
|
}
|
|
832
872
|
|
|
873
|
+
export function recordContextCompactionPendingGauges(
|
|
874
|
+
observability: Observability,
|
|
875
|
+
summary: ContextCompactionPendingSummary,
|
|
876
|
+
nowMs = Date.now(),
|
|
877
|
+
): void {
|
|
878
|
+
const oldestStartedAtMs = summary.oldestStartedAt?.getTime();
|
|
879
|
+
const oldestPendingAgeSeconds =
|
|
880
|
+
oldestStartedAtMs === undefined || !Number.isFinite(oldestStartedAtMs)
|
|
881
|
+
? 0
|
|
882
|
+
: Math.max(0, (nowMs - oldestStartedAtMs) / 1_000);
|
|
883
|
+
observability.setGauge({
|
|
884
|
+
name: "opengeni_context_compaction_pending",
|
|
885
|
+
help: "Current exact active attempts whose latest automatic compaction landmark is still started.",
|
|
886
|
+
value: nonnegativeFinite(summary.pendingCount),
|
|
887
|
+
});
|
|
888
|
+
observability.setGauge({
|
|
889
|
+
name: "opengeni_context_compaction_oldest_pending_age_seconds",
|
|
890
|
+
help: "Age in seconds of the oldest durably pending automatic compaction, or zero when none are pending.",
|
|
891
|
+
value: oldestPendingAgeSeconds,
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
export function startContextCompactionPendingMonitor(input: {
|
|
896
|
+
observability: Observability;
|
|
897
|
+
read: () => Promise<ContextCompactionPendingSummary>;
|
|
898
|
+
intervalMs?: number;
|
|
899
|
+
now?: () => number;
|
|
900
|
+
}): { close: () => Promise<void> } {
|
|
901
|
+
const intervalMs = input.intervalMs ?? 60_000;
|
|
902
|
+
const now = input.now ?? Date.now;
|
|
903
|
+
const startedAt = now();
|
|
904
|
+
let lastSuccessAt: number | null = null;
|
|
905
|
+
let lastReadSucceeded = false;
|
|
906
|
+
let stopped = false;
|
|
907
|
+
let running: Promise<void> | null = null;
|
|
908
|
+
const recordStatus = () => {
|
|
909
|
+
const observedAt = now();
|
|
910
|
+
const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
|
|
911
|
+
const set = (name: string, help: string, value: number) =>
|
|
912
|
+
input.observability.setGauge({ name, help, value });
|
|
913
|
+
set(
|
|
914
|
+
"opengeni_context_compaction_monitor_last_read_success",
|
|
915
|
+
"Whether the latest durable context-compaction aggregate read completed successfully.",
|
|
916
|
+
lastReadSucceeded ? 1 : 0,
|
|
917
|
+
);
|
|
918
|
+
set(
|
|
919
|
+
"opengeni_context_compaction_monitor_last_success_timestamp_seconds",
|
|
920
|
+
"Unix timestamp of the latest successful durable context-compaction aggregate read, or zero before one succeeds.",
|
|
921
|
+
lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
|
|
922
|
+
);
|
|
923
|
+
set(
|
|
924
|
+
"opengeni_context_compaction_monitor_fresh",
|
|
925
|
+
"Whether durable context-compaction pending gauges have a successful read within three monitor intervals.",
|
|
926
|
+
lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
|
|
927
|
+
);
|
|
928
|
+
};
|
|
929
|
+
const refresh = () => {
|
|
930
|
+
recordStatus();
|
|
931
|
+
if (stopped || running) return;
|
|
932
|
+
running = input
|
|
933
|
+
.read()
|
|
934
|
+
.then((summary) => {
|
|
935
|
+
recordContextCompactionPendingGauges(input.observability, summary, now());
|
|
936
|
+
lastSuccessAt = now();
|
|
937
|
+
lastReadSucceeded = true;
|
|
938
|
+
recordStatus();
|
|
939
|
+
})
|
|
940
|
+
.catch((error) => {
|
|
941
|
+
lastReadSucceeded = false;
|
|
942
|
+
recordStatus();
|
|
943
|
+
input.observability.warn("context compaction monitor: durable aggregate read failed", {
|
|
944
|
+
error: error instanceof Error ? error.message : String(error),
|
|
945
|
+
});
|
|
946
|
+
})
|
|
947
|
+
.finally(() => {
|
|
948
|
+
running = null;
|
|
949
|
+
});
|
|
950
|
+
};
|
|
951
|
+
refresh();
|
|
952
|
+
const timer = setInterval(refresh, intervalMs);
|
|
953
|
+
timer.unref?.();
|
|
954
|
+
return {
|
|
955
|
+
close: async () => {
|
|
956
|
+
stopped = true;
|
|
957
|
+
clearInterval(timer);
|
|
958
|
+
await running;
|
|
959
|
+
},
|
|
960
|
+
};
|
|
961
|
+
}
|
|
962
|
+
|
|
833
963
|
export function recordSandboxLeaseGauges(
|
|
834
964
|
observability: Observability,
|
|
835
965
|
counts: Partial<Record<SandboxLeaseLiveness, number>>,
|
|
@@ -1023,6 +1153,7 @@ export function recordSandboxRotationBacklogGauges(
|
|
|
1023
1153
|
turnBlocked: number;
|
|
1024
1154
|
directBlocked: number;
|
|
1025
1155
|
processBlocked: number;
|
|
1156
|
+
interactionBlocked: number;
|
|
1026
1157
|
},
|
|
1027
1158
|
): void {
|
|
1028
1159
|
const values = {
|
|
@@ -1031,6 +1162,7 @@ export function recordSandboxRotationBacklogGauges(
|
|
|
1031
1162
|
turn_blocked: backlog.turnBlocked,
|
|
1032
1163
|
direct_blocked: backlog.directBlocked,
|
|
1033
1164
|
process_blocked: backlog.processBlocked,
|
|
1165
|
+
interaction_blocked: backlog.interactionBlocked,
|
|
1034
1166
|
} as const;
|
|
1035
1167
|
for (const [kind, value] of Object.entries(values)) {
|
|
1036
1168
|
observability.setGauge({
|
|
@@ -1749,15 +1881,14 @@ export function recordSessionEventPublishLatency(
|
|
|
1749
1881
|
});
|
|
1750
1882
|
}
|
|
1751
1883
|
|
|
1752
|
-
// Context tokens per response span a wide range; buckets
|
|
1753
|
-
//
|
|
1884
|
+
// Context tokens per response span a wide range; buckets retain context-pressure
|
|
1885
|
+
// diagnostics while compaction alerting follows the durable model-aware start.
|
|
1754
1886
|
const MODEL_INPUT_TOKENS_BUCKETS = [
|
|
1755
1887
|
1_000, 5_000, 10_000, 25_000, 50_000, 100_000, 150_000, 200_000, 300_000, 500_000, 1_000_000,
|
|
1756
1888
|
];
|
|
1757
1889
|
|
|
1758
|
-
/** Observed input (context) tokens per model response
|
|
1759
|
-
*
|
|
1760
|
-
* "compaction never firing while contexts run hot" failure mode expressible. */
|
|
1890
|
+
/** Observed input (context) tokens per model response for context-pressure
|
|
1891
|
+
* dashboards and provider diagnostics. */
|
|
1761
1892
|
export function recordModelInputTokens(
|
|
1762
1893
|
observability: Observability,
|
|
1763
1894
|
provider: string,
|
|
@@ -1776,15 +1907,24 @@ export function recordModelInputTokens(
|
|
|
1776
1907
|
});
|
|
1777
1908
|
}
|
|
1778
1909
|
|
|
1779
|
-
/** A context compaction
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1910
|
+
/** A context compaction successfully completed, by bounded trigger. */
|
|
1911
|
+
export function recordContextCompaction(
|
|
1912
|
+
observability: Observability,
|
|
1913
|
+
trigger: ContextCompactionTrigger,
|
|
1914
|
+
): void {
|
|
1915
|
+
observability.incrementCounter({ ...CONTEXT_COMPACTIONS_METRIC, labels: { trigger } });
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
/**
|
|
1919
|
+
* A durable context-compaction start, recorded only after the attempt-fenced
|
|
1920
|
+
* `compaction.started` transaction commits. This is the model-aware threshold
|
|
1921
|
+
* signal; it does not guess from a static token count shared by unlike models.
|
|
1922
|
+
*/
|
|
1923
|
+
export function recordContextCompactionStarted(
|
|
1924
|
+
observability: Observability,
|
|
1925
|
+
trigger: ContextCompactionTrigger,
|
|
1926
|
+
): void {
|
|
1927
|
+
observability.incrementCounter({ ...CONTEXT_COMPACTION_STARTS_METRIC, labels: { trigger } });
|
|
1788
1928
|
}
|
|
1789
1929
|
|
|
1790
1930
|
const MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS = [1, 8, 32, 128, 512, 2_048, 8_192];
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,
|
|
3
3
|
SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS,
|
|
4
|
+
effectiveSandboxDrainSnapshotTimeoutMs,
|
|
4
5
|
sandboxArchiveCaptureTimeoutMs,
|
|
5
6
|
type Settings,
|
|
6
7
|
} from "@opengeni/config";
|
|
@@ -13,16 +14,21 @@ import {
|
|
|
13
14
|
|
|
14
15
|
const STRICT_TIMEOUT_FENCE_MS = 1;
|
|
15
16
|
|
|
16
|
-
export function sandboxDrainTiming(
|
|
17
|
+
export function sandboxDrainTiming(
|
|
18
|
+
settings: Pick<Settings, "sandboxSnapshotTimeoutMs" | "sandboxDrainSnapshotTimeoutMs">,
|
|
19
|
+
): {
|
|
17
20
|
snapshotTimeoutMs: number;
|
|
18
21
|
captureTimeoutMs: number;
|
|
19
22
|
timeoutClass: SandboxDrainTimeoutClass;
|
|
20
23
|
activityTimeoutMs: number;
|
|
21
24
|
} {
|
|
22
|
-
const
|
|
25
|
+
const snapshotTimeoutMs = effectiveSandboxDrainSnapshotTimeoutMs(settings);
|
|
26
|
+
const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs({
|
|
27
|
+
sandboxSnapshotTimeoutMs: snapshotTimeoutMs,
|
|
28
|
+
});
|
|
23
29
|
const timeoutClass = sandboxDrainTimeoutClass(captureTimeoutMs);
|
|
24
30
|
return {
|
|
25
|
-
snapshotTimeoutMs
|
|
31
|
+
snapshotTimeoutMs,
|
|
26
32
|
captureTimeoutMs,
|
|
27
33
|
timeoutClass,
|
|
28
34
|
activityTimeoutMs: sandboxDrainActivityTimeoutMs(timeoutClass),
|
|
@@ -30,7 +36,7 @@ export function sandboxDrainTiming(settings: Pick<Settings, "sandboxSnapshotTime
|
|
|
30
36
|
}
|
|
31
37
|
|
|
32
38
|
export function assertSandboxReaperActivityTimeout(
|
|
33
|
-
settings: Pick<Settings, "sandboxSnapshotTimeoutMs">,
|
|
39
|
+
settings: Pick<Settings, "sandboxSnapshotTimeoutMs" | "sandboxDrainSnapshotTimeoutMs">,
|
|
34
40
|
): void {
|
|
35
41
|
const timing = sandboxDrainTiming(settings);
|
|
36
42
|
if (timing.snapshotTimeoutMs >= timing.captureTimeoutMs) {
|
package/src/sandbox-resume.ts
CHANGED
|
@@ -709,7 +709,13 @@ export async function persistSandboxDeadlineRotationCheckpoint(
|
|
|
709
709
|
* the user's machine IS the persistence), or a snapshot newer than the
|
|
710
710
|
* interval already rides the lease.
|
|
711
711
|
*/
|
|
712
|
-
export
|
|
712
|
+
export type WarmWorkspaceSnapshotPromise = Promise<boolean> & {
|
|
713
|
+
/** Settles only after any provider capture continuation has published or
|
|
714
|
+
* failed and released its exact workspace-capture admission. */
|
|
715
|
+
readonly settled: Promise<void>;
|
|
716
|
+
};
|
|
717
|
+
|
|
718
|
+
export function maybePersistWarmWorkspaceSnapshot(
|
|
713
719
|
services: SandboxResumeServices,
|
|
714
720
|
ids: {
|
|
715
721
|
accountId: string;
|
|
@@ -723,6 +729,50 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
723
729
|
leaseEpoch: number,
|
|
724
730
|
signal?: AbortSignal,
|
|
725
731
|
force = false,
|
|
732
|
+
): WarmWorkspaceSnapshotPromise {
|
|
733
|
+
let continuationRegistered = false;
|
|
734
|
+
let resolvePhysicalSettlement: () => void = () => undefined;
|
|
735
|
+
const physicalSettlement = new Promise<void>((resolve) => {
|
|
736
|
+
resolvePhysicalSettlement = resolve;
|
|
737
|
+
});
|
|
738
|
+
const result = persistWarmWorkspaceSnapshot(
|
|
739
|
+
services,
|
|
740
|
+
ids,
|
|
741
|
+
session,
|
|
742
|
+
leaseEpoch,
|
|
743
|
+
signal,
|
|
744
|
+
force,
|
|
745
|
+
(continuation) => {
|
|
746
|
+
continuationRegistered = true;
|
|
747
|
+
void continuation.then(resolvePhysicalSettlement, resolvePhysicalSettlement);
|
|
748
|
+
},
|
|
749
|
+
);
|
|
750
|
+
void result.then(
|
|
751
|
+
() => {
|
|
752
|
+
if (!continuationRegistered) resolvePhysicalSettlement();
|
|
753
|
+
},
|
|
754
|
+
() => {
|
|
755
|
+
if (!continuationRegistered) resolvePhysicalSettlement();
|
|
756
|
+
},
|
|
757
|
+
);
|
|
758
|
+
return Object.assign(result, { settled: physicalSettlement });
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
async function persistWarmWorkspaceSnapshot(
|
|
762
|
+
services: SandboxResumeServices,
|
|
763
|
+
ids: {
|
|
764
|
+
accountId: string;
|
|
765
|
+
workspaceId: string;
|
|
766
|
+
sessionId: string;
|
|
767
|
+
turnId: string;
|
|
768
|
+
attemptId: string;
|
|
769
|
+
sandboxGroupId: string;
|
|
770
|
+
},
|
|
771
|
+
session: unknown,
|
|
772
|
+
leaseEpoch: number,
|
|
773
|
+
signal: AbortSignal | undefined,
|
|
774
|
+
force: boolean,
|
|
775
|
+
registerOwnedContinuation: (continuation: Promise<void>) => void,
|
|
726
776
|
): Promise<boolean> {
|
|
727
777
|
const { db, settings } = services;
|
|
728
778
|
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
@@ -989,6 +1039,7 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
989
1039
|
return { kind: "settled" as const, persisted: false };
|
|
990
1040
|
},
|
|
991
1041
|
);
|
|
1042
|
+
registerOwnedContinuation(settled.then(() => undefined));
|
|
992
1043
|
let cancelListener: (() => void) | undefined;
|
|
993
1044
|
const outcome = await Promise.race([
|
|
994
1045
|
settled,
|
|
@@ -1223,13 +1274,12 @@ export async function resumeBoxForTurn(
|
|
|
1223
1274
|
// holder is gone / its attempt is no longer the active writer" from "the lease
|
|
1224
1275
|
// could not be extended" (rotation requested, draining, another epoch). The
|
|
1225
1276
|
// second is NOT a reason to release: once a provider-deadline rotation is
|
|
1226
|
-
// requested the
|
|
1227
|
-
//
|
|
1228
|
-
//
|
|
1229
|
-
//
|
|
1230
|
-
//
|
|
1231
|
-
//
|
|
1232
|
-
// "holder alive -> never release"; this loop keeps touching the holder.
|
|
1277
|
+
// requested the turn-side heartbeat immediately preempts the attempt, whose
|
|
1278
|
+
// finalizer retains this holder until every attempt-owned writer is physically
|
|
1279
|
+
// drained. Dropping it here would let the reaper capture or terminate while a
|
|
1280
|
+
// provider operation may still be mutating /workspace. Both loops therefore
|
|
1281
|
+
// converge on "holder alive -> never release"; finalization performs the
|
|
1282
|
+
// proof-bearing release.
|
|
1233
1283
|
const releaseDeadHolder = (reason: string): void => {
|
|
1234
1284
|
const message = "sandbox resume holder released: holder no longer live";
|
|
1235
1285
|
const fields = {
|
|
@@ -68,6 +68,21 @@ export const automationActivity = proxyActivities<Pick<typeof activities, "dispa
|
|
|
68
68
|
},
|
|
69
69
|
);
|
|
70
70
|
|
|
71
|
+
/** Dispatch has a bounded retry window, but an accepted automation event must
|
|
72
|
+
* not remain durably `dispatching` after that window closes. This compensating
|
|
73
|
+
* database write is idempotent and retries without a cap until the run reaches
|
|
74
|
+
* a terminal state (or is observed as already dispatched). */
|
|
75
|
+
export const automationFailureActivity = proxyActivities<
|
|
76
|
+
Pick<typeof activities, "settleAutomationRunFailure">
|
|
77
|
+
>({
|
|
78
|
+
startToCloseTimeout: "2 minutes",
|
|
79
|
+
retry: {
|
|
80
|
+
initialInterval: "1 second",
|
|
81
|
+
backoffCoefficient: 2,
|
|
82
|
+
maximumInterval: "30 seconds",
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
|
|
71
86
|
/** Goal continuation evaluates a durable Postgres obligation at an idle
|
|
72
87
|
* boundary. A transient failure gets a short retry window, then records an
|
|
73
88
|
* explicit delayed outbox wake instead of relying on an unrelated mutation or
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { automationActivity } from "./activities";
|
|
1
|
+
import { automationActivity, automationFailureActivity } from "./activities";
|
|
2
2
|
|
|
3
3
|
export type AutomationRunWorkflowInput = {
|
|
4
4
|
accountId: string;
|
|
@@ -6,6 +6,25 @@ export type AutomationRunWorkflowInput = {
|
|
|
6
6
|
runId: string;
|
|
7
7
|
};
|
|
8
8
|
|
|
9
|
+
export type AutomationRunWorkflowActivities = {
|
|
10
|
+
dispatchAutomationRun: (input: AutomationRunWorkflowInput) => Promise<unknown>;
|
|
11
|
+
settleAutomationRunFailure: (input: AutomationRunWorkflowInput) => Promise<void>;
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
export async function runAutomationRunWorkflow(
|
|
15
|
+
input: AutomationRunWorkflowInput,
|
|
16
|
+
activities: AutomationRunWorkflowActivities,
|
|
17
|
+
): Promise<void> {
|
|
18
|
+
try {
|
|
19
|
+
await activities.dispatchAutomationRun(input);
|
|
20
|
+
} catch {
|
|
21
|
+
await activities.settleAutomationRunFailure(input);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
9
25
|
export async function automationRunWorkflow(input: AutomationRunWorkflowInput): Promise<void> {
|
|
10
|
-
await
|
|
26
|
+
await runAutomationRunWorkflow(input, {
|
|
27
|
+
dispatchAutomationRun: automationActivity.dispatchAutomationRun,
|
|
28
|
+
settleAutomationRunFailure: automationFailureActivity.settleAutomationRunFailure,
|
|
29
|
+
});
|
|
11
30
|
}
|