@opengeni/worker-bundle 0.16.21 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +145 -32
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +3 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -7
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +2 -2
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +129 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +2 -2
- package/dist/activities/run-input.d.ts +5 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +11 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -147
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/host-export.js +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +675 -16460
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +40 -7
- package/dist/sandbox-routing.d.ts +13 -6
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1044 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1905 -559
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +16 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +47 -6
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +52 -60
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +1317 -0
- package/src/activities/rig-verification.ts +567 -77
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +40 -10
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +96 -7
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +77 -13
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +50 -118
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +202 -7
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +331 -157
- package/src/sandbox-routing.ts +49 -39
- package/src/worker-service-lifecycle.ts +82 -12
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +6 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
- package/src/activities/toolspace-token-renewal.ts +0 -57
|
@@ -15,6 +15,23 @@ export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
|
|
|
15
15
|
export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
|
|
16
16
|
export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
|
|
17
17
|
export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
|
|
18
|
+
export type TurnTaskQueueStats = {
|
|
19
|
+
/** Temporal activity tasks ready for a turn worker to accept. */
|
|
20
|
+
eligibleBacklog: number;
|
|
21
|
+
oldestBacklogAgeSeconds: number;
|
|
22
|
+
tasksAddRate: number;
|
|
23
|
+
tasksDispatchRate: number;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export type TemporalTurnTaskQueueStats = {
|
|
27
|
+
approximateBacklogCount?: unknown;
|
|
28
|
+
approximateBacklogAge?: {
|
|
29
|
+
seconds?: unknown;
|
|
30
|
+
nanos?: unknown;
|
|
31
|
+
} | null;
|
|
32
|
+
tasksAddRate?: unknown;
|
|
33
|
+
tasksDispatchRate?: unknown;
|
|
34
|
+
} | null;
|
|
18
35
|
|
|
19
36
|
const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
20
37
|
const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
|
|
@@ -37,11 +54,11 @@ function eventAttributes(attributes: Record<string, unknown> | undefined): Attri
|
|
|
37
54
|
if (!attributes) {
|
|
38
55
|
return undefined;
|
|
39
56
|
}
|
|
40
|
-
const
|
|
57
|
+
const projected: Attributes = {};
|
|
41
58
|
for (const [key, value] of Object.entries(attributes)) {
|
|
42
|
-
|
|
59
|
+
projected[key] = eventAttributeValue(value);
|
|
43
60
|
}
|
|
44
|
-
return
|
|
61
|
+
return projected;
|
|
45
62
|
}
|
|
46
63
|
|
|
47
64
|
function eventAttributeValue(value: unknown): AttributeValue {
|
|
@@ -78,23 +95,24 @@ export function runtimeMetricsHooksForObservability(
|
|
|
78
95
|
value: durationSeconds,
|
|
79
96
|
});
|
|
80
97
|
},
|
|
81
|
-
onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
|
|
98
|
+
onSandboxCreate: ({ backend, imageSource, outcome, durationSeconds }) => {
|
|
82
99
|
observability.incrementCounter({
|
|
83
100
|
name: "opengeni_sandbox_creates_total",
|
|
84
101
|
help: "Total sandbox create attempts by backend and outcome.",
|
|
85
|
-
labels: { backend, outcome },
|
|
102
|
+
labels: { backend, image_source: imageSource, outcome },
|
|
86
103
|
});
|
|
87
104
|
observability.observeHistogram({
|
|
88
105
|
name: "opengeni_sandbox_create_duration_seconds",
|
|
89
106
|
help: "Sandbox create duration in seconds by backend.",
|
|
90
|
-
labels: { backend },
|
|
107
|
+
labels: { backend, image_source: imageSource },
|
|
91
108
|
value: durationSeconds,
|
|
92
109
|
});
|
|
93
110
|
},
|
|
94
|
-
onSandboxWarmingTimeout: () => {
|
|
111
|
+
onSandboxWarmingTimeout: ({ backend, stage }) => {
|
|
95
112
|
observability.incrementCounter({
|
|
96
113
|
name: "opengeni_sandbox_warming_timeouts_total",
|
|
97
114
|
help: "Total sandbox warming timeouts.",
|
|
115
|
+
labels: { backend, stage },
|
|
98
116
|
});
|
|
99
117
|
},
|
|
100
118
|
onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
|
|
@@ -330,6 +348,152 @@ export function recordTurnsQueuedGauge(observability: Observability, value: numb
|
|
|
330
348
|
});
|
|
331
349
|
}
|
|
332
350
|
|
|
351
|
+
/**
|
|
352
|
+
* Record the authoritative turn activity queue rather than aggregate Postgres
|
|
353
|
+
* prompts. A paused human prompt is durable queue truth but is not runnable and
|
|
354
|
+
* never reaches this Temporal task queue. DescribeTaskQueue's approximate
|
|
355
|
+
* backlog count and age are explicitly documented by Temporal as autoscaling
|
|
356
|
+
* signals.
|
|
357
|
+
*/
|
|
358
|
+
export function recordTurnTaskQueueStats(
|
|
359
|
+
observability: Observability,
|
|
360
|
+
stats: TurnTaskQueueStats,
|
|
361
|
+
): void {
|
|
362
|
+
observability.setGauge({
|
|
363
|
+
name: "opengeni_turn_eligible_backlog",
|
|
364
|
+
help: "Temporal runAgentTurn activity tasks eligible for immediate worker admission.",
|
|
365
|
+
value: nonnegativeFinite(stats.eligibleBacklog),
|
|
366
|
+
});
|
|
367
|
+
observability.setGauge({
|
|
368
|
+
name: "opengeni_turn_eligible_backlog_oldest_age_seconds",
|
|
369
|
+
help: "Approximate age of the oldest eligible runAgentTurn activity task.",
|
|
370
|
+
value: nonnegativeFinite(stats.oldestBacklogAgeSeconds),
|
|
371
|
+
});
|
|
372
|
+
observability.setGauge({
|
|
373
|
+
name: "opengeni_turn_eligible_tasks_add_rate",
|
|
374
|
+
help: "Temporal runAgentTurn tasks added per second over its rolling window.",
|
|
375
|
+
value: nonnegativeFinite(stats.tasksAddRate),
|
|
376
|
+
});
|
|
377
|
+
observability.setGauge({
|
|
378
|
+
name: "opengeni_turn_eligible_tasks_dispatch_rate",
|
|
379
|
+
help: "Temporal runAgentTurn tasks dispatched per second over its rolling window.",
|
|
380
|
+
value: nonnegativeFinite(stats.tasksDispatchRate),
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
export function normalizeTurnTaskQueueStats(
|
|
385
|
+
stats: TemporalTurnTaskQueueStats | undefined,
|
|
386
|
+
): TurnTaskQueueStats {
|
|
387
|
+
if (!stats) {
|
|
388
|
+
throw new Error("Temporal DescribeTaskQueue response omitted required stats");
|
|
389
|
+
}
|
|
390
|
+
const eligibleBacklog = requiredTemporalNumber(
|
|
391
|
+
stats.approximateBacklogCount,
|
|
392
|
+
"approximateBacklogCount",
|
|
393
|
+
{ integer: true },
|
|
394
|
+
);
|
|
395
|
+
let oldestBacklogAgeSeconds = 0;
|
|
396
|
+
if (stats.approximateBacklogAge) {
|
|
397
|
+
const seconds = optionalTemporalNumber(
|
|
398
|
+
stats.approximateBacklogAge.seconds,
|
|
399
|
+
"approximateBacklogAge.seconds",
|
|
400
|
+
{ integer: true },
|
|
401
|
+
);
|
|
402
|
+
const nanos = optionalTemporalNumber(
|
|
403
|
+
stats.approximateBacklogAge.nanos,
|
|
404
|
+
"approximateBacklogAge.nanos",
|
|
405
|
+
{ integer: true },
|
|
406
|
+
);
|
|
407
|
+
if (nanos >= 1_000_000_000) {
|
|
408
|
+
throw new Error("Temporal approximateBacklogAge.nanos must be less than one second");
|
|
409
|
+
}
|
|
410
|
+
oldestBacklogAgeSeconds = seconds + nanos / 1_000_000_000;
|
|
411
|
+
} else if (eligibleBacklog > 0) {
|
|
412
|
+
throw new Error("Temporal stats omitted approximateBacklogAge for a nonzero backlog");
|
|
413
|
+
}
|
|
414
|
+
return {
|
|
415
|
+
eligibleBacklog,
|
|
416
|
+
oldestBacklogAgeSeconds,
|
|
417
|
+
tasksAddRate: optionalTemporalNumber(stats.tasksAddRate, "tasksAddRate"),
|
|
418
|
+
tasksDispatchRate: optionalTemporalNumber(stats.tasksDispatchRate, "tasksDispatchRate"),
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
export function startTurnCapacityMonitor(input: {
|
|
423
|
+
observability: Observability;
|
|
424
|
+
read: () => Promise<TurnTaskQueueStats>;
|
|
425
|
+
intervalMs?: number;
|
|
426
|
+
now?: () => number;
|
|
427
|
+
}): { close: () => Promise<void> } {
|
|
428
|
+
const intervalMs = input.intervalMs ?? 15_000;
|
|
429
|
+
const now = input.now ?? Date.now;
|
|
430
|
+
const startedAt = now();
|
|
431
|
+
let lastSuccessAt: number | null = null;
|
|
432
|
+
let lastReadSucceeded = false;
|
|
433
|
+
let stopped = false;
|
|
434
|
+
let running: Promise<void> | null = null;
|
|
435
|
+
const recordStatus = () => {
|
|
436
|
+
const observedAt = now();
|
|
437
|
+
const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
|
|
438
|
+
const set = (name: string, help: string, value: number) =>
|
|
439
|
+
input.observability.setGauge({ name, help, value });
|
|
440
|
+
set(
|
|
441
|
+
"opengeni_turn_capacity_monitor_last_read_success",
|
|
442
|
+
"Whether the latest Temporal turn-queue capacity read completed successfully.",
|
|
443
|
+
lastReadSucceeded ? 1 : 0,
|
|
444
|
+
);
|
|
445
|
+
set(
|
|
446
|
+
"opengeni_turn_capacity_monitor_last_success_timestamp_seconds",
|
|
447
|
+
"Unix timestamp of the latest successful Temporal turn-queue capacity read, or zero before one succeeds.",
|
|
448
|
+
lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
|
|
449
|
+
);
|
|
450
|
+
set(
|
|
451
|
+
"opengeni_turn_capacity_monitor_last_success_age_seconds",
|
|
452
|
+
"Age of the latest successful Temporal turn-queue capacity read, or monitor age before one succeeds.",
|
|
453
|
+
Math.max(0, successAgeMs / 1_000),
|
|
454
|
+
);
|
|
455
|
+
set(
|
|
456
|
+
"opengeni_turn_capacity_monitor_fresh",
|
|
457
|
+
"Whether eligible-backlog gauges have a successful Temporal read within three monitor intervals.",
|
|
458
|
+
lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
|
|
459
|
+
);
|
|
460
|
+
};
|
|
461
|
+
const refresh = () => {
|
|
462
|
+
// Advance freshness age even while a Temporal read is hung. Old backlog
|
|
463
|
+
// values remain observable for diagnosis but cease to be authoritative.
|
|
464
|
+
recordStatus();
|
|
465
|
+
if (stopped || running) return;
|
|
466
|
+
running = input
|
|
467
|
+
.read()
|
|
468
|
+
.then((stats) => {
|
|
469
|
+
recordTurnTaskQueueStats(input.observability, stats);
|
|
470
|
+
lastSuccessAt = now();
|
|
471
|
+
lastReadSucceeded = true;
|
|
472
|
+
recordStatus();
|
|
473
|
+
})
|
|
474
|
+
.catch((error) => {
|
|
475
|
+
lastReadSucceeded = false;
|
|
476
|
+
recordStatus();
|
|
477
|
+
input.observability.warn("turn capacity monitor: Temporal task-queue stats failed", {
|
|
478
|
+
error: error instanceof Error ? error.message : String(error),
|
|
479
|
+
});
|
|
480
|
+
})
|
|
481
|
+
.finally(() => {
|
|
482
|
+
running = null;
|
|
483
|
+
});
|
|
484
|
+
};
|
|
485
|
+
refresh();
|
|
486
|
+
const timer = setInterval(refresh, intervalMs);
|
|
487
|
+
timer.unref?.();
|
|
488
|
+
return {
|
|
489
|
+
close: async () => {
|
|
490
|
+
stopped = true;
|
|
491
|
+
clearInterval(timer);
|
|
492
|
+
await running;
|
|
493
|
+
},
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
|
|
333
497
|
export function recordSandboxLeaseGauges(
|
|
334
498
|
observability: Observability,
|
|
335
499
|
counts: Partial<Record<SandboxLeaseLiveness, number>>,
|
|
@@ -948,6 +1112,37 @@ function incrementBoundedModelCacheCounter(
|
|
|
948
1112
|
totals.set(key, current + input.amount);
|
|
949
1113
|
}
|
|
950
1114
|
|
|
1115
|
+
function nonnegativeFinite(value: number): number {
|
|
1116
|
+
return Number.isFinite(value) && value > 0 ? value : 0;
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
function requiredTemporalNumber(
|
|
1120
|
+
value: unknown,
|
|
1121
|
+
field: string,
|
|
1122
|
+
options: { integer?: boolean } = {},
|
|
1123
|
+
): number {
|
|
1124
|
+
if (value === null || value === undefined) {
|
|
1125
|
+
throw new Error(`Temporal stats omitted required ${field}`);
|
|
1126
|
+
}
|
|
1127
|
+
const parsed = Number(value);
|
|
1128
|
+
if (
|
|
1129
|
+
!Number.isFinite(parsed) ||
|
|
1130
|
+
parsed < 0 ||
|
|
1131
|
+
(options.integer && !Number.isSafeInteger(parsed))
|
|
1132
|
+
) {
|
|
1133
|
+
throw new Error(`Temporal stats returned invalid ${field}: ${String(value)}`);
|
|
1134
|
+
}
|
|
1135
|
+
return parsed;
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
function optionalTemporalNumber(
|
|
1139
|
+
value: unknown,
|
|
1140
|
+
field: string,
|
|
1141
|
+
options: { integer?: boolean } = {},
|
|
1142
|
+
): number {
|
|
1143
|
+
return value === null || value === undefined ? 0 : requiredTemporalNumber(value, field, options);
|
|
1144
|
+
}
|
|
1145
|
+
|
|
951
1146
|
/**
|
|
952
1147
|
* An opaque, stable, non-reversible tag for a codex credential (account) — the
|
|
953
1148
|
* per-call log dimension the account-switch hypothesis correlates against. It is
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
export const SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS = 5 * 60_000;
|
|
2
|
+
export const SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS = 5 * 60_000;
|
|
3
|
+
export const SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS = 65 * 60_000;
|
|
4
|
+
export const SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS = 60_000;
|
|
5
|
+
export const SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS = 20_000;
|
|
6
|
+
export const SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS = 5_000;
|
|
7
|
+
// Match the DB inventory's fair 500-row bound so one sweep never discovers a
|
|
8
|
+
// due box and then needlessly defers it for another schedule period. Five
|
|
9
|
+
// hundred small StartChild commands remain far below Temporal history limits;
|
|
10
|
+
// actual provider I/O stays bounded by worker activity concurrency.
|
|
11
|
+
export const SANDBOX_REAPER_CHILD_DISPATCH_LIMIT = 500;
|
|
12
|
+
export const SANDBOX_REAPER_V2_WORKFLOW_ID = "opengeni-sandbox-lease-reaper-v2";
|
|
13
|
+
export const SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID = "opengeni-sandbox-lease-maintenance-v1";
|
|
14
|
+
|
|
15
|
+
// A protocol-specific queue is the rolling-deploy compatibility boundary for
|
|
16
|
+
// the split reaper activities. An old control worker polls only the base queue,
|
|
17
|
+
// so it can replay already-started legacy workflows but can never receive a new
|
|
18
|
+
// activity type that its binary does not register.
|
|
19
|
+
export const SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX = "-sandbox-lifecycle-v1";
|
|
20
|
+
|
|
21
|
+
export function sandboxLifecycleTaskQueue(baseTaskQueue: string): string {
|
|
22
|
+
return baseTaskQueue.endsWith(SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX)
|
|
23
|
+
? baseTaskQueue
|
|
24
|
+
: `${baseTaskQueue}${SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX}`;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export type SandboxDrainTimeoutClass = "fast" | "extended";
|
|
28
|
+
|
|
29
|
+
export type SandboxDrainTarget = {
|
|
30
|
+
workspaceId: string;
|
|
31
|
+
sandboxGroupId: string;
|
|
32
|
+
instanceId: string | null;
|
|
33
|
+
leaseEpoch: number;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export type SandboxDrainActivityInput = {
|
|
37
|
+
target: SandboxDrainTarget;
|
|
38
|
+
timeoutClass: SandboxDrainTimeoutClass;
|
|
39
|
+
/** Provider operation budget frozen when the durable child is created. A
|
|
40
|
+
* rolling config change cannot make an existing fast child retry forever. */
|
|
41
|
+
snapshotTimeoutMs: number;
|
|
42
|
+
captureTimeoutMs: number;
|
|
43
|
+
/** Stable logical teardown operation; every Temporal activity retry keeps it
|
|
44
|
+
* while receiving a distinct exact capture-attempt id. */
|
|
45
|
+
operationId: string;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
export type SandboxDrainWorkflowInput = Omit<SandboxDrainActivityInput, "operationId">;
|
|
49
|
+
|
|
50
|
+
export type SandboxLeaseSweepMaintenanceInput = {
|
|
51
|
+
examined: number;
|
|
52
|
+
started: number;
|
|
53
|
+
alreadyRunning: number;
|
|
54
|
+
startFailed: number;
|
|
55
|
+
rotationsRequested: number;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
export function sandboxDrainTimeoutClass(captureTimeoutMs: number): SandboxDrainTimeoutClass {
|
|
59
|
+
return captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS <
|
|
60
|
+
SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS
|
|
61
|
+
? "fast"
|
|
62
|
+
: "extended";
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function sandboxDrainActivityTimeoutMs(timeoutClass: SandboxDrainTimeoutClass): number {
|
|
66
|
+
return timeoutClass === "fast"
|
|
67
|
+
? SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS
|
|
68
|
+
: SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function sandboxDrainWorkflowId(target: SandboxDrainTarget): string {
|
|
72
|
+
return `sandbox-drain:${target.workspaceId}:${target.sandboxGroupId}:${target.leaseEpoch}`;
|
|
73
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import {
|
|
2
|
+
SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,
|
|
3
|
+
SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS,
|
|
4
|
+
sandboxArchiveCaptureTimeoutMs,
|
|
5
|
+
type Settings,
|
|
6
|
+
} from "@opengeni/config";
|
|
7
|
+
import {
|
|
8
|
+
SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS,
|
|
9
|
+
sandboxDrainActivityTimeoutMs,
|
|
10
|
+
sandboxDrainTimeoutClass,
|
|
11
|
+
type SandboxDrainTimeoutClass,
|
|
12
|
+
} from "./sandbox-reaper-contract";
|
|
13
|
+
|
|
14
|
+
const STRICT_TIMEOUT_FENCE_MS = 1;
|
|
15
|
+
|
|
16
|
+
export function sandboxDrainTiming(settings: Pick<Settings, "sandboxSnapshotTimeoutMs">): {
|
|
17
|
+
snapshotTimeoutMs: number;
|
|
18
|
+
captureTimeoutMs: number;
|
|
19
|
+
timeoutClass: SandboxDrainTimeoutClass;
|
|
20
|
+
activityTimeoutMs: number;
|
|
21
|
+
} {
|
|
22
|
+
const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
|
|
23
|
+
const timeoutClass = sandboxDrainTimeoutClass(captureTimeoutMs);
|
|
24
|
+
return {
|
|
25
|
+
snapshotTimeoutMs: settings.sandboxSnapshotTimeoutMs,
|
|
26
|
+
captureTimeoutMs,
|
|
27
|
+
timeoutClass,
|
|
28
|
+
activityTimeoutMs: sandboxDrainActivityTimeoutMs(timeoutClass),
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function assertSandboxReaperActivityTimeout(
|
|
33
|
+
settings: Pick<Settings, "sandboxSnapshotTimeoutMs">,
|
|
34
|
+
): void {
|
|
35
|
+
const timing = sandboxDrainTiming(settings);
|
|
36
|
+
if (timing.snapshotTimeoutMs >= timing.captureTimeoutMs) {
|
|
37
|
+
throw new Error(
|
|
38
|
+
`Sandbox provider snapshot timeout (${timing.snapshotTimeoutMs}) must be strictly less than ` +
|
|
39
|
+
`its durable capture claim (${timing.captureTimeoutMs}).`,
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
if (
|
|
43
|
+
timing.captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS + STRICT_TIMEOUT_FENCE_MS >=
|
|
44
|
+
timing.activityTimeoutMs
|
|
45
|
+
) {
|
|
46
|
+
throw new Error(
|
|
47
|
+
`Sandbox ${timing.timeoutClass} drain activity timeout (${timing.activityTimeoutMs}) must ` +
|
|
48
|
+
`strictly exceed one durable archive capture and settlement margin ` +
|
|
49
|
+
`(${timing.captureTimeoutMs} + ${SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS} milliseconds).`,
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function assertSandboxDrainInputTiming(input: {
|
|
55
|
+
timeoutClass: SandboxDrainTimeoutClass;
|
|
56
|
+
snapshotTimeoutMs: number;
|
|
57
|
+
captureTimeoutMs: number;
|
|
58
|
+
}): void {
|
|
59
|
+
if (
|
|
60
|
+
!Number.isSafeInteger(input.snapshotTimeoutMs) ||
|
|
61
|
+
input.snapshotTimeoutMs <= 0 ||
|
|
62
|
+
input.snapshotTimeoutMs > SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS ||
|
|
63
|
+
!Number.isSafeInteger(input.captureTimeoutMs) ||
|
|
64
|
+
input.captureTimeoutMs <= 0 ||
|
|
65
|
+
input.captureTimeoutMs > SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS
|
|
66
|
+
) {
|
|
67
|
+
throw new Error("Sandbox drain timeout values are invalid");
|
|
68
|
+
}
|
|
69
|
+
const inputActivityTimeoutMs = sandboxDrainActivityTimeoutMs(input.timeoutClass);
|
|
70
|
+
if (
|
|
71
|
+
sandboxDrainTimeoutClass(input.captureTimeoutMs) !== input.timeoutClass ||
|
|
72
|
+
input.snapshotTimeoutMs >= input.captureTimeoutMs ||
|
|
73
|
+
input.captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS >= inputActivityTimeoutMs
|
|
74
|
+
) {
|
|
75
|
+
throw new Error(
|
|
76
|
+
`Sandbox drain timing is inconsistent with the ${input.timeoutClass} activity budget`,
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
}
|