@opengeni/worker-bundle 0.7.5 → 0.10.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/chunk-BJUW5LCM.js +325 -0
  2. package/dist/chunk-BJUW5LCM.js.map +1 -0
  3. package/dist/host-export.d.ts +70 -0
  4. package/dist/host-export.js +7 -0
  5. package/dist/host-export.js.map +1 -0
  6. package/dist/index.d.ts +98 -6
  7. package/dist/index.js +4097 -1271
  8. package/dist/index.js.map +1 -1
  9. package/dist/workflow-bundle.js +12360 -0
  10. package/package.json +18 -12
  11. package/src/activities/agent-turn.ts +2674 -828
  12. package/src/activities/capabilities.ts +11 -2
  13. package/src/activities/codex-capacity.ts +6 -11
  14. package/src/activities/codex-rotation.ts +75 -86
  15. package/src/activities/context-compaction.ts +104 -24
  16. package/src/activities/environment.ts +503 -97
  17. package/src/activities/expiring-material-renewal.ts +143 -0
  18. package/src/activities/goals.ts +4 -0
  19. package/src/activities/mcp-credentials.ts +38 -0
  20. package/src/activities/packs.ts +115 -2
  21. package/src/activities/rig-verification.ts +897 -244
  22. package/src/activities/run-credential-renewal.ts +70 -0
  23. package/src/activities/run-credentials.ts +132 -0
  24. package/src/activities/run-input.ts +23 -0
  25. package/src/activities/sandbox-lease.ts +33 -2
  26. package/src/activities/scheduled-tasks.ts +21 -0
  27. package/src/activities/session-state.ts +72 -0
  28. package/src/activities/streaming.ts +1 -0
  29. package/src/activities/toolspace-token-renewal.ts +57 -0
  30. package/src/activities/types.ts +44 -6
  31. package/src/activities/workspace-capture.ts +113 -93
  32. package/src/activities.ts +10 -2
  33. package/src/host-export-pump.ts +416 -0
  34. package/src/host-export.ts +22 -0
  35. package/src/http.ts +77 -28
  36. package/src/index.ts +360 -156
  37. package/src/observability-metrics.ts +83 -15
  38. package/src/sandbox-resume.ts +47 -1
  39. package/src/sandbox-routing.ts +1 -1
  40. package/src/worker-service-lifecycle.ts +104 -0
  41. package/src/workflows/activities.ts +13 -7
  42. package/src/workflows/rig-verification.ts +3 -1
  43. package/src/workflows/session.ts +219 -23
  44. package/src/workflows.ts +1 -0
@@ -5,6 +5,7 @@ import type { EventLogger } from "@opengeni/events";
5
5
  import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
6
6
  import {
7
7
  SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
8
+ modelUsageTokenCountOrNull,
8
9
  type RuntimeMetricsHooks,
9
10
  type SelfhostedOpObservation,
10
11
  type SelfhostedOpObserver,
@@ -17,6 +18,13 @@ export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
17
18
 
18
19
  const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
19
20
  const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
21
+ const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
22
+
23
+ // A worker process reaching one trillion tokens in one provider/cache counter is
24
+ // already far outside an ordinary scrape lifetime. Refuse further increments
25
+ // before floating-point precision can degrade; a restart resets both the
26
+ // process-local Prometheus registry and this guard together.
27
+ const MAX_MODEL_CACHE_COUNTER_TOTAL = 1_000_000_000_000;
20
28
 
21
29
  export function observabilityEventLogger(observability: Observability): EventLogger {
22
30
  return {
@@ -553,7 +561,8 @@ export function recordModelInputTokens(
553
561
  provider: string,
554
562
  inputTokens: number,
555
563
  ): void {
556
- if (!(inputTokens > 0)) {
564
+ const normalizedInputTokens = modelUsageTokenCountOrNull(inputTokens);
565
+ if (normalizedInputTokens === null || normalizedInputTokens === 0) {
557
566
  return;
558
567
  }
559
568
  observability.observeHistogram({
@@ -561,7 +570,7 @@ export function recordModelInputTokens(
561
570
  help: "Observed input (context) tokens per model response, by provider.",
562
571
  buckets: MODEL_INPUT_TOKENS_BUCKETS,
563
572
  labels: { provider },
564
- value: inputTokens,
573
+ value: normalizedInputTokens,
565
574
  });
566
575
  }
567
576
 
@@ -598,29 +607,56 @@ const MODEL_CACHE_HIT_RATIO_BUCKETS = [
598
607
  * - `opengeni_model_cached_tokens_total{provider}` — cumulative prompt tokens the
599
608
  * provider served from cache. Advanced only when >0, so a provider that never
600
609
  * reports cached tokens contributes nothing rather than phantom zero-increments.
610
+ * - `opengeni_model_cache_write_tokens_total{provider}` — cumulative tokens a
611
+ * reporting provider wrote into its prompt cache. Absent telemetry contributes
612
+ * nothing; in particular, Codex subscription usage does not currently expose it.
601
613
  * - `opengeni_model_cache_hit_ratio{provider}` — cached/prompt for the call,
602
- * observed whenever prompt tokens are known (>0). A call whose provider does NOT
603
- * report cached_tokens records a real 0 here — "we saw a call and the cache did
604
- * nothing" is exactly the signal the alert watches, so it must not be swallowed.
605
- * Absent/zero/non-finite `cachedTokens` (providers that don't report it) is safe: no
606
- * counter increment, ratio 0. A call with no prompt tokens has no ratio (skipped).
614
+ * observed only when prompt tokens are known (>0) AND cached tokens are reported.
615
+ * A reported cached-token zero records a real 0%; absent/null telemetry remains
616
+ * unknown and must not fabricate a low-cache signal.
617
+ * - `opengeni_model_cache_read_telemetry_total{provider,status}` — one bounded
618
+ * availability observation per authoritative call (`reported` or `missing`).
619
+ * Invalid, fractional, unsafe, or over-contract token telemetry is rejected. A call
620
+ * with no prompt tokens has no ratio (skipped).
607
621
  */
608
622
  export function recordModelCacheTokens(
609
623
  observability: Observability,
610
624
  provider: string,
611
- input: { cachedTokens: number | null | undefined; promptTokens: number | null | undefined },
625
+ input: {
626
+ cachedTokens: number | null | undefined;
627
+ cacheWriteTokens?: number | null | undefined;
628
+ promptTokens: number | null | undefined;
629
+ },
612
630
  ): void {
613
- const cached = nonNegativeTokenCount(input.cachedTokens);
614
- const prompt = nonNegativeTokenCount(input.promptTokens);
615
- if (cached > 0) {
616
- observability.incrementCounter({
631
+ const cached = modelUsageTokenCountOrNull(input.cachedTokens);
632
+ const cacheWrite = modelUsageTokenCountOrNull(input.cacheWriteTokens);
633
+ const prompt = modelUsageTokenCountOrNull(input.promptTokens);
634
+ incrementBoundedModelCacheCounter(observability, {
635
+ name: "opengeni_model_cache_read_telemetry_total",
636
+ help: "Authoritative model calls with reported or missing cache-read telemetry, by provider.",
637
+ provider,
638
+ labels: { provider, status: cached === null ? "missing" : "reported" },
639
+ amount: 1,
640
+ });
641
+ if (cached !== null && cached > 0) {
642
+ incrementBoundedModelCacheCounter(observability, {
617
643
  name: "opengeni_model_cached_tokens_total",
618
644
  help: "Total prompt tokens served from the provider's prompt cache, by provider.",
645
+ provider,
619
646
  labels: { provider },
620
647
  amount: cached,
621
648
  });
622
649
  }
623
- if (prompt > 0) {
650
+ if (cacheWrite !== null && cacheWrite > 0) {
651
+ incrementBoundedModelCacheCounter(observability, {
652
+ name: "opengeni_model_cache_write_tokens_total",
653
+ help: "Total prompt tokens written to the provider's prompt cache, by provider.",
654
+ provider,
655
+ labels: { provider },
656
+ amount: cacheWrite,
657
+ });
658
+ }
659
+ if (cached !== null && prompt !== null && prompt > 0) {
624
660
  observability.observeHistogram({
625
661
  name: "opengeni_model_cache_hit_ratio",
626
662
  help: "Per-call prompt-cache hit ratio (cached/prompt tokens) by provider.",
@@ -633,8 +669,40 @@ export function recordModelCacheTokens(
633
669
  }
634
670
  }
635
671
 
636
- function nonNegativeTokenCount(value: number | null | undefined): number {
637
- return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
672
+ function incrementBoundedModelCacheCounter(
673
+ observability: Observability,
674
+ input: {
675
+ name: string;
676
+ help: string;
677
+ provider: string;
678
+ labels: Record<string, string>;
679
+ amount: number;
680
+ },
681
+ ): void {
682
+ const totals = modelCacheCounterTotals.get(observability) ?? new Map<string, number>();
683
+ if (!modelCacheCounterTotals.has(observability)) {
684
+ modelCacheCounterTotals.set(observability, totals);
685
+ }
686
+ const labelKey = Object.entries(input.labels)
687
+ .sort(([left], [right]) => left.localeCompare(right))
688
+ .map(([key, value]) => `${key}=${value}`)
689
+ .join(",");
690
+ const key = `${input.name}:${labelKey}`;
691
+ const current = totals.get(key) ?? 0;
692
+ if (input.amount > MAX_MODEL_CACHE_COUNTER_TOTAL - current) {
693
+ observability.warn("model cache metric cumulative limit reached", {
694
+ provider: input.provider,
695
+ metric: input.name,
696
+ });
697
+ return;
698
+ }
699
+ observability.incrementCounter({
700
+ name: input.name,
701
+ help: input.help,
702
+ labels: input.labels,
703
+ amount: input.amount,
704
+ });
705
+ totals.set(key, current + input.amount);
638
706
  }
639
707
 
640
708
  /**
@@ -231,11 +231,24 @@ class SnapshotTimeoutError extends Error {
231
231
  export async function waitForWarmSnapshot(
232
232
  snapshot: Promise<unknown>,
233
233
  timeoutMs: number,
234
+ signal?: AbortSignal,
234
235
  ): Promise<boolean> {
236
+ if (signal?.aborted) return false;
235
237
  let timeout: ReturnType<typeof setTimeout> | undefined;
238
+ let cancelListener: (() => void) | undefined;
236
239
  try {
237
240
  await Promise.race([
238
241
  snapshot,
242
+ ...(signal
243
+ ? [
244
+ new Promise<never>((_resolve, reject) => {
245
+ cancelListener = () =>
246
+ reject(signal.reason ?? new Error("workspace snapshot wait cancelled"));
247
+ signal.addEventListener("abort", cancelListener, { once: true });
248
+ if (signal.aborted) cancelListener();
249
+ }),
250
+ ]
251
+ : []),
239
252
  new Promise<never>((_, reject) => {
240
253
  timeout = setTimeout(() => reject(new SnapshotTimeoutError(timeoutMs)), timeoutMs);
241
254
  if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
@@ -245,6 +258,7 @@ export async function waitForWarmSnapshot(
245
258
  ]);
246
259
  return true;
247
260
  } catch (error) {
261
+ if (signal?.aborted) return false;
248
262
  if (error instanceof SnapshotTimeoutError) {
249
263
  console.error("mid-session workspace snapshot wait timed out (turn unaffected)", error);
250
264
  return false;
@@ -254,6 +268,7 @@ export async function waitForWarmSnapshot(
254
268
  if (timeout) {
255
269
  clearTimeout(timeout);
256
270
  }
271
+ if (cancelListener) signal?.removeEventListener("abort", cancelListener);
257
272
  }
258
273
  }
259
274
 
@@ -389,9 +404,17 @@ function preserveWorkspaceArchivesOnResumeState(
389
404
  */
390
405
  export async function maybePersistWarmWorkspaceSnapshot(
391
406
  services: SandboxResumeServices,
392
- ids: { accountId: string; workspaceId: string; sandboxGroupId: string },
407
+ ids: {
408
+ accountId: string;
409
+ workspaceId: string;
410
+ sessionId: string;
411
+ turnId: string;
412
+ attemptId: string;
413
+ sandboxGroupId: string;
414
+ },
393
415
  session: unknown,
394
416
  leaseEpoch: number,
417
+ signal?: AbortSignal,
395
418
  ): Promise<boolean> {
396
419
  const { db, settings } = services;
397
420
  const intervalMs = settings.sandboxSnapshotIntervalMs;
@@ -404,6 +427,9 @@ export async function maybePersistWarmWorkspaceSnapshot(
404
427
  if (typeof persistable.persistWorkspace !== "function") {
405
428
  return false;
406
429
  }
430
+ if (signal?.aborted) {
431
+ return false;
432
+ }
407
433
  try {
408
434
  // Cheap throttle pre-check before the (potentially slow) capture;
409
435
  // persistWarmSnapshot re-checks atomically under the row lock, so this is
@@ -412,6 +438,9 @@ export async function maybePersistWarmWorkspaceSnapshot(
412
438
  if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm") {
413
439
  return false;
414
440
  }
441
+ if (signal?.aborted) {
442
+ return false;
443
+ }
415
444
  const sessionState =
416
445
  lease.resumeState && typeof lease.resumeState === "object"
417
446
  ? (lease.resumeState as { sessionState?: Record<string, unknown> }).sessionState
@@ -437,8 +466,18 @@ export async function maybePersistWarmWorkspaceSnapshot(
437
466
  // guard this timeout exists to provide.
438
467
  const capture = persistable.persistWorkspace();
439
468
  capture.catch(() => undefined);
469
+ let cancelListener: (() => void) | undefined;
440
470
  const bytes = await Promise.race([
441
471
  capture,
472
+ ...(signal
473
+ ? [
474
+ new Promise<undefined>((resolve) => {
475
+ cancelListener = () => resolve(undefined);
476
+ signal.addEventListener("abort", cancelListener, { once: true });
477
+ if (signal.aborted) cancelListener();
478
+ }),
479
+ ]
480
+ : []),
442
481
  new Promise<undefined>((resolve) => {
443
482
  timeout = setTimeout(() => resolve(undefined), settings.sandboxSnapshotTimeoutMs);
444
483
  if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
@@ -449,13 +488,20 @@ export async function maybePersistWarmWorkspaceSnapshot(
449
488
  if (timeout) {
450
489
  clearTimeout(timeout);
451
490
  }
491
+ if (cancelListener) signal?.removeEventListener("abort", cancelListener);
452
492
  });
453
493
  if (!bytes || bytes.length === 0) {
454
494
  return false;
455
495
  }
496
+ if (signal?.aborted) {
497
+ return false;
498
+ }
456
499
  const { wrote, priorArchiveForGc } = await persistWarmSnapshot(db, {
457
500
  accountId: ids.accountId,
458
501
  workspaceId: ids.workspaceId,
502
+ sessionId: ids.sessionId,
503
+ turnId: ids.turnId,
504
+ attemptId: ids.attemptId,
459
505
  sandboxGroupId: ids.sandboxGroupId,
460
506
  expectedEpoch: leaseEpoch,
461
507
  workspaceArchive: Buffer.from(bytes).toString("base64"),
@@ -4,7 +4,7 @@
4
4
  //
5
5
  // The turn resumes its group box by id (resumeBoxForTurn) and injects it
6
6
  // NON-OWNED into the run. With hot-swap, the injected `session` must be the
7
- // STABLE routing proxy (dossier §10.3): the SDK binds to it ONCE and calls its
7
+ // STABLE routing proxy: the SDK binds to it ONCE and calls its
8
8
  // methods per tool call; the proxy re-reads `(active_sandbox_id, active_epoch)`
9
9
  // per op and dispatches to the currently-active backend (the group Modal box by
10
10
  // default, or a swap target — a sibling Modal box or a selfhosted machine).
@@ -0,0 +1,104 @@
1
+ import type { Observability } from "@opengeni/observability";
2
+ import type { WorkerLifecycleState } from "./http";
3
+
4
+ export async function constructWithOwnedConnection<Connection, Result>(
5
+ connect: () => Promise<Connection>,
6
+ construct: (connection: Connection) => Promise<Result>,
7
+ close: (connection: Connection) => Promise<void>,
8
+ ): Promise<Result> {
9
+ const connection = await connect();
10
+ try {
11
+ return await construct(connection);
12
+ } catch (error) {
13
+ await close(connection).catch(() => undefined);
14
+ throw error;
15
+ }
16
+ }
17
+
18
+ export type WorkerRunTarget = {
19
+ run(): Promise<void>;
20
+ shutdown(): void;
21
+ };
22
+
23
+ export type WorkerServiceLifecycle = {
24
+ state(): WorkerLifecycleState;
25
+ run(): Promise<void>;
26
+ drain(reason?: string): void;
27
+ close(): Promise<void>;
28
+ };
29
+
30
+ export function createWorkerServiceLifecycle(input: {
31
+ role: "control" | "turn";
32
+ worker: WorkerRunTarget;
33
+ observability: Observability;
34
+ closeOwnedResources: () => Promise<void>;
35
+ onReady?: () => void;
36
+ }): WorkerServiceLifecycle {
37
+ let state: WorkerLifecycleState = "starting";
38
+ let runPromise: Promise<void> | undefined;
39
+ let resourcesClosed: Promise<void> | undefined;
40
+ const closeOwnedResources = () => {
41
+ resourcesClosed ??= input.closeOwnedResources();
42
+ return resourcesClosed;
43
+ };
44
+
45
+ const lifecycle: WorkerServiceLifecycle = {
46
+ state: () => state,
47
+ run: () => {
48
+ if (!runPromise && state === "draining") {
49
+ runPromise = (async () => {
50
+ state = "stopped";
51
+ await closeOwnedResources();
52
+ })();
53
+ return runPromise;
54
+ }
55
+ if (!runPromise && (state === "stopped" || state === "failed")) {
56
+ return Promise.reject(new Error(`cannot run a worker service that is ${state}`));
57
+ }
58
+ runPromise ??= (async () => {
59
+ if (state === "starting") {
60
+ state = "ready";
61
+ }
62
+ input.onReady?.();
63
+ try {
64
+ await input.worker.run();
65
+ state = "stopped";
66
+ } catch (error) {
67
+ state = "failed";
68
+ throw error;
69
+ } finally {
70
+ await closeOwnedResources();
71
+ }
72
+ })();
73
+ return runPromise;
74
+ },
75
+ drain: (reason = "host request") => {
76
+ if (state === "draining" || state === "stopped" || state === "failed") {
77
+ return;
78
+ }
79
+ state = "draining";
80
+ input.observability.info("OpenGeni worker draining (graceful shutdown)", {
81
+ role: input.role,
82
+ reason,
83
+ });
84
+ try {
85
+ input.worker.shutdown();
86
+ } catch (error) {
87
+ input.observability.warn("worker shutdown request failed", {
88
+ error: error instanceof Error ? error.message : String(error),
89
+ });
90
+ }
91
+ },
92
+ close: async () => {
93
+ lifecycle.drain("service close");
94
+ if (runPromise) {
95
+ await runPromise.catch(() => undefined);
96
+ } else {
97
+ state = "stopped";
98
+ await closeOwnedResources();
99
+ }
100
+ },
101
+ };
102
+
103
+ return lifecycle;
104
+ }
@@ -5,10 +5,12 @@ type WorkflowControlActivities = Pick<
5
5
  typeof activities,
6
6
  | "dispatchScheduledTaskRun"
7
7
  | "enqueueGoalRetryWake"
8
+ | "expireSessionHumanInput"
8
9
  | "failSessionAttempt"
9
10
  | "getCodexCapacityWait"
10
11
  | "markSessionIdle"
11
12
  | "peekSessionWork"
13
+ | "persistSessionAttemptQuiescence"
12
14
  | "reconcileCodexCapacityWait"
13
15
  | "recoverDispatch"
14
16
  | "settleSessionInterruptions"
@@ -49,7 +51,7 @@ export function turnTaskQueue(baseTaskQueue: string): string {
49
51
  return `${baseTaskQueue}-turns`;
50
52
  }
51
53
 
52
- export function turnActivityForTaskQueue(baseTaskQueue: string) {
54
+ export function turnActivityForTaskQueue(baseTaskQueue: string, receiptGatedCancellation = true) {
53
55
  return proxyActivities<Pick<typeof activities, "runAgentTurn">>({
54
56
  taskQueue: turnTaskQueue(baseTaskQueue),
55
57
  // Agent segments legitimately run for days. A started turn heartbeats;
@@ -57,12 +59,16 @@ export function turnActivityForTaskQueue(baseTaskQueue: string) {
57
59
  // accepts them and performs the atomic claim.
58
60
  startToCloseTimeout: "30 days",
59
61
  heartbeatTimeout: "2 minutes",
60
- // Pause/Steer may complete the durable control transition immediately,
61
- // but the session workflow must remain open until the old turn activity is
62
- // physically gone. Leaving this implicit uses Temporal's try-cancel wire
63
- // default: the activity promise rejects as soon as cancellation is
64
- // requested and the workflow can close while the worker keeps streaming.
65
- cancellationType: ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
62
+ // Pause/Steer first closes the exact attempt in Postgres, then asks
63
+ // Temporal to deliver cancellation. The workflow must not wait for the
64
+ // Temporal activity promise: a provider cleanup promise can outlive the
65
+ // fenced activity body, and Temporal terminalization is not proof that
66
+ // sandbox tools or processes are physically quiescent. The activity owns
67
+ // that proof and writes an exact receipt after its hard tool fence; the
68
+ // receipt transaction wakes the workflow to admit a replacement.
69
+ cancellationType: receiptGatedCancellation
70
+ ? ActivityCancellationType.TRY_CANCEL
71
+ : ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
66
72
  retry: { maximumAttempts: 1 },
67
73
  });
68
74
  }
@@ -1,4 +1,4 @@
1
- import { proxyActivities } from "@temporalio/workflow";
1
+ import { ActivityCancellationType, proxyActivities } from "@temporalio/workflow";
2
2
  import type * as activities from "../activities";
3
3
  import type { RigVerificationWorkflowInput } from "../activities/rig-verification";
4
4
 
@@ -6,6 +6,8 @@ const rigVerificationActivities = proxyActivities<
6
6
  Pick<typeof activities, "verifyRigChange" | "verifyRigVersion">
7
7
  >({
8
8
  startToCloseTimeout: "15 minutes",
9
+ heartbeatTimeout: "10 seconds",
10
+ cancellationType: ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
9
11
  retry: { maximumAttempts: 1 },
10
12
  });
11
13