@opengeni/worker-bundle 0.23.4-canary.0 → 0.24.2-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +7 -0
  2. package/dist/activities/agent-turn/model-usage.d.ts +5 -0
  3. package/dist/activities/agent-turn/quiescence.d.ts +2 -0
  4. package/dist/activities/agent-turn/run.d.ts +9 -0
  5. package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
  6. package/dist/activities/agent-turn/session-title.d.ts +21 -0
  7. package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
  8. package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
  9. package/dist/activities/agent-turn/turn-context.d.ts +2 -1
  10. package/dist/activities/agent-turn.d.ts +1 -1
  11. package/dist/activities/capabilities.d.ts +1 -0
  12. package/dist/activities/context-compaction.d.ts +2 -0
  13. package/dist/{activities-control-SGU2WEVB.js → activities-control-ZWH7J4KZ.js} +102 -94
  14. package/dist/activities-control-ZWH7J4KZ.js.map +1 -0
  15. package/dist/{activities-turn-7EAQXW6Q.js → activities-turn-I4OPLQLS.js} +613 -284
  16. package/dist/activities-turn-I4OPLQLS.js.map +1 -0
  17. package/dist/{chunk-TJIPCHMP.js → chunk-65JEWP2Q.js} +105 -6
  18. package/dist/chunk-65JEWP2Q.js.map +1 -0
  19. package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
  20. package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
  21. package/dist/{chunk-S2QVRVJW.js → chunk-RQNH6CGR.js} +28 -41
  22. package/dist/chunk-RQNH6CGR.js.map +1 -0
  23. package/dist/index.js +16 -3
  24. package/dist/index.js.map +1 -1
  25. package/dist/observability-metrics.d.ts +33 -7
  26. package/dist/sandbox-reaper-timeout.d.ts +2 -2
  27. package/dist/sandbox-resume.d.ts +6 -1
  28. package/package.json +17 -17
  29. package/src/activities/agent-turn/admission.ts +19 -0
  30. package/src/activities/agent-turn/claim.ts +9 -1
  31. package/src/activities/agent-turn/compaction-prep.ts +6 -1
  32. package/src/activities/agent-turn/finalization.ts +31 -17
  33. package/src/activities/agent-turn/model-usage.ts +33 -4
  34. package/src/activities/agent-turn/quiescence.ts +11 -2
  35. package/src/activities/agent-turn/run.ts +74 -1
  36. package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
  37. package/src/activities/agent-turn/session-title.ts +55 -2
  38. package/src/activities/agent-turn/stream-attempt.ts +258 -135
  39. package/src/activities/agent-turn/tool-environment.ts +41 -7
  40. package/src/activities/agent-turn/turn-context.ts +4 -2
  41. package/src/activities/agent-turn.ts +5 -1
  42. package/src/activities/api-integrations.ts +11 -1
  43. package/src/activities/capabilities.ts +57 -0
  44. package/src/activities/context-compaction.ts +3 -0
  45. package/src/activities/goals.ts +1 -0
  46. package/src/activities/mcp-credentials.ts +53 -0
  47. package/src/activities/rig-verification.ts +5 -3
  48. package/src/activities/scheduled-tasks.ts +111 -98
  49. package/src/index.ts +15 -0
  50. package/src/observability-metrics.ts +152 -14
  51. package/src/sandbox-reaper-timeout.ts +10 -4
  52. package/src/sandbox-resume.ts +58 -8
  53. package/dist/activities-control-SGU2WEVB.js.map +0 -1
  54. package/dist/activities-turn-7EAQXW6Q.js.map +0 -1
  55. package/dist/chunk-S2QVRVJW.js.map +0 -1
  56. package/dist/chunk-TJIPCHMP.js.map +0 -1
@@ -34,6 +34,10 @@ export type SessionRecoveryBacklog = {
34
34
  quiescence_missing: number;
35
35
  projection_stale: number;
36
36
  };
37
+ export type ContextCompactionPendingSummary = {
38
+ pendingCount: number;
39
+ oldestStartedAt: Date | null;
40
+ };
37
41
 
38
42
  export type TemporalTurnTaskQueueStats = {
39
43
  approximateBacklogCount?: unknown;
@@ -49,6 +53,7 @@ const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
49
53
  const modelRequestTrackers = new WeakMap<Observability, ModelRequestLifecycleMetrics>();
50
54
  const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
51
55
  const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
56
+ const initializedContextCompactionMetrics = new WeakSet<Observability>();
52
57
 
53
58
  // A worker process reaching one trillion tokens in one provider/cache counter is
54
59
  // already far outside an ordinary scrape lifetime. Refuse further increments
@@ -61,6 +66,18 @@ const WORKER_DEATH_RECOVERY_OUTCOMES: readonly WorkerDeathRecoveryOutcome[] = [
61
66
  "exhausted",
62
67
  ];
63
68
  const WORKER_DEATH_TIMEOUT_TYPES = ["heartbeat", "schedule_to_start"] as const;
69
+ const CONTEXT_COMPACTION_TRIGGERS = ["auto", "operator", "proactive", "overflow"] as const;
70
+
71
+ export type ContextCompactionTrigger = (typeof CONTEXT_COMPACTION_TRIGGERS)[number];
72
+
73
+ const CONTEXT_COMPACTION_STARTS_METRIC = {
74
+ name: "opengeni_context_compaction_starts_total",
75
+ help: "Total durable context compaction starts, by trigger.",
76
+ } as const;
77
+ const CONTEXT_COMPACTIONS_METRIC = {
78
+ name: "opengeni_context_compactions_total",
79
+ help: "Total completed context compactions, by trigger.",
80
+ } as const;
64
81
 
65
82
  export function observabilityEventLogger(observability: Observability): EventLogger {
66
83
  return {
@@ -346,6 +363,29 @@ export function initializeWorkerOutcomeMetrics(observability: Observability): vo
346
363
  }
347
364
  }
348
365
 
366
+ /**
367
+ * Publish every bounded compaction lifecycle series before the first event.
368
+ * The zero counters make the closed trigger catalog explicit before a scrape.
369
+ * Alerting uses the separate durable control-worker projection below; these
370
+ * process-local counters are lifecycle-rate diagnostics only.
371
+ */
372
+ export function initializeContextCompactionMetrics(observability: Observability): void {
373
+ if (initializedContextCompactionMetrics.has(observability)) return;
374
+ for (const trigger of CONTEXT_COMPACTION_TRIGGERS) {
375
+ observability.incrementCounter({
376
+ ...CONTEXT_COMPACTION_STARTS_METRIC,
377
+ labels: { trigger },
378
+ amount: 0,
379
+ });
380
+ observability.incrementCounter({
381
+ ...CONTEXT_COMPACTIONS_METRIC,
382
+ labels: { trigger },
383
+ amount: 0,
384
+ });
385
+ }
386
+ initializedContextCompactionMetrics.add(observability);
387
+ }
388
+
349
389
  export function recordWorkerDeathRecoveryMetrics(
350
390
  observability: Observability,
351
391
  input: {
@@ -830,6 +870,96 @@ export function startSessionRecoveryMonitor(input: {
830
870
  };
831
871
  }
832
872
 
873
+ export function recordContextCompactionPendingGauges(
874
+ observability: Observability,
875
+ summary: ContextCompactionPendingSummary,
876
+ nowMs = Date.now(),
877
+ ): void {
878
+ const oldestStartedAtMs = summary.oldestStartedAt?.getTime();
879
+ const oldestPendingAgeSeconds =
880
+ oldestStartedAtMs === undefined || !Number.isFinite(oldestStartedAtMs)
881
+ ? 0
882
+ : Math.max(0, (nowMs - oldestStartedAtMs) / 1_000);
883
+ observability.setGauge({
884
+ name: "opengeni_context_compaction_pending",
885
+ help: "Current exact active attempts whose latest automatic compaction landmark is still started.",
886
+ value: nonnegativeFinite(summary.pendingCount),
887
+ });
888
+ observability.setGauge({
889
+ name: "opengeni_context_compaction_oldest_pending_age_seconds",
890
+ help: "Age in seconds of the oldest durably pending automatic compaction, or zero when none are pending.",
891
+ value: oldestPendingAgeSeconds,
892
+ });
893
+ }
894
+
895
+ export function startContextCompactionPendingMonitor(input: {
896
+ observability: Observability;
897
+ read: () => Promise<ContextCompactionPendingSummary>;
898
+ intervalMs?: number;
899
+ now?: () => number;
900
+ }): { close: () => Promise<void> } {
901
+ const intervalMs = input.intervalMs ?? 60_000;
902
+ const now = input.now ?? Date.now;
903
+ const startedAt = now();
904
+ let lastSuccessAt: number | null = null;
905
+ let lastReadSucceeded = false;
906
+ let stopped = false;
907
+ let running: Promise<void> | null = null;
908
+ const recordStatus = () => {
909
+ const observedAt = now();
910
+ const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
911
+ const set = (name: string, help: string, value: number) =>
912
+ input.observability.setGauge({ name, help, value });
913
+ set(
914
+ "opengeni_context_compaction_monitor_last_read_success",
915
+ "Whether the latest durable context-compaction aggregate read completed successfully.",
916
+ lastReadSucceeded ? 1 : 0,
917
+ );
918
+ set(
919
+ "opengeni_context_compaction_monitor_last_success_timestamp_seconds",
920
+ "Unix timestamp of the latest successful durable context-compaction aggregate read, or zero before one succeeds.",
921
+ lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
922
+ );
923
+ set(
924
+ "opengeni_context_compaction_monitor_fresh",
925
+ "Whether durable context-compaction pending gauges have a successful read within three monitor intervals.",
926
+ lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
927
+ );
928
+ };
929
+ const refresh = () => {
930
+ recordStatus();
931
+ if (stopped || running) return;
932
+ running = input
933
+ .read()
934
+ .then((summary) => {
935
+ recordContextCompactionPendingGauges(input.observability, summary, now());
936
+ lastSuccessAt = now();
937
+ lastReadSucceeded = true;
938
+ recordStatus();
939
+ })
940
+ .catch((error) => {
941
+ lastReadSucceeded = false;
942
+ recordStatus();
943
+ input.observability.warn("context compaction monitor: durable aggregate read failed", {
944
+ error: error instanceof Error ? error.message : String(error),
945
+ });
946
+ })
947
+ .finally(() => {
948
+ running = null;
949
+ });
950
+ };
951
+ refresh();
952
+ const timer = setInterval(refresh, intervalMs);
953
+ timer.unref?.();
954
+ return {
955
+ close: async () => {
956
+ stopped = true;
957
+ clearInterval(timer);
958
+ await running;
959
+ },
960
+ };
961
+ }
962
+
833
963
  export function recordSandboxLeaseGauges(
834
964
  observability: Observability,
835
965
  counts: Partial<Record<SandboxLeaseLiveness, number>>,
@@ -1751,15 +1881,14 @@ export function recordSessionEventPublishLatency(
1751
1881
  });
1752
1882
  }
1753
1883
 
1754
- // Context tokens per response span a wide range; buckets track the pressure toward
1755
- // a model's window so "sessions are running hot but never compacting" is queryable.
1884
+ // Context tokens per response span a wide range; buckets retain context-pressure
1885
+ // diagnostics while compaction alerting follows the durable model-aware start.
1756
1886
  const MODEL_INPUT_TOKENS_BUCKETS = [
1757
1887
  1_000, 5_000, 10_000, 25_000, 50_000, 100_000, 150_000, 200_000, 300_000, 500_000, 1_000_000,
1758
1888
  ];
1759
1889
 
1760
- /** Observed input (context) tokens per model response — the context-pressure
1761
- * signal. Paired with `opengeni_context_compactions_total`, it makes the
1762
- * "compaction never firing while contexts run hot" failure mode expressible. */
1890
+ /** Observed input (context) tokens per model response for context-pressure
1891
+ * dashboards and provider diagnostics. */
1763
1892
  export function recordModelInputTokens(
1764
1893
  observability: Observability,
1765
1894
  provider: string,
@@ -1778,15 +1907,24 @@ export function recordModelInputTokens(
1778
1907
  });
1779
1908
  }
1780
1909
 
1781
- /** A context compaction actually fired, by trigger (operator | overflow | proactive
1782
- * | auto). The rate of this — against the input-tokens histogram above — is how an
1783
- * operator sees compaction working (or silently not). */
1784
- export function recordContextCompaction(observability: Observability, trigger: string): void {
1785
- observability.incrementCounter({
1786
- name: "opengeni_context_compactions_total",
1787
- help: "Total context compactions performed, by trigger.",
1788
- labels: { trigger },
1789
- });
1910
+ /** A context compaction successfully completed, by bounded trigger. */
1911
+ export function recordContextCompaction(
1912
+ observability: Observability,
1913
+ trigger: ContextCompactionTrigger,
1914
+ ): void {
1915
+ observability.incrementCounter({ ...CONTEXT_COMPACTIONS_METRIC, labels: { trigger } });
1916
+ }
1917
+
1918
+ /**
1919
+ * A durable context-compaction start, recorded only after the attempt-fenced
1920
+ * `compaction.started` transaction commits. This is the model-aware threshold
1921
+ * signal; it does not guess from a static token count shared by unlike models.
1922
+ */
1923
+ export function recordContextCompactionStarted(
1924
+ observability: Observability,
1925
+ trigger: ContextCompactionTrigger,
1926
+ ): void {
1927
+ observability.incrementCounter({ ...CONTEXT_COMPACTION_STARTS_METRIC, labels: { trigger } });
1790
1928
  }
1791
1929
 
1792
1930
  const MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS = [1, 8, 32, 128, 512, 2_048, 8_192];
@@ -1,6 +1,7 @@
1
1
  import {
2
2
  SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,
3
3
  SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS,
4
+ effectiveSandboxDrainSnapshotTimeoutMs,
4
5
  sandboxArchiveCaptureTimeoutMs,
5
6
  type Settings,
6
7
  } from "@opengeni/config";
@@ -13,16 +14,21 @@ import {
13
14
 
14
15
  const STRICT_TIMEOUT_FENCE_MS = 1;
15
16
 
16
- export function sandboxDrainTiming(settings: Pick<Settings, "sandboxSnapshotTimeoutMs">): {
17
+ export function sandboxDrainTiming(
18
+ settings: Pick<Settings, "sandboxSnapshotTimeoutMs" | "sandboxDrainSnapshotTimeoutMs">,
19
+ ): {
17
20
  snapshotTimeoutMs: number;
18
21
  captureTimeoutMs: number;
19
22
  timeoutClass: SandboxDrainTimeoutClass;
20
23
  activityTimeoutMs: number;
21
24
  } {
22
- const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
25
+ const snapshotTimeoutMs = effectiveSandboxDrainSnapshotTimeoutMs(settings);
26
+ const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs({
27
+ sandboxSnapshotTimeoutMs: snapshotTimeoutMs,
28
+ });
23
29
  const timeoutClass = sandboxDrainTimeoutClass(captureTimeoutMs);
24
30
  return {
25
- snapshotTimeoutMs: settings.sandboxSnapshotTimeoutMs,
31
+ snapshotTimeoutMs,
26
32
  captureTimeoutMs,
27
33
  timeoutClass,
28
34
  activityTimeoutMs: sandboxDrainActivityTimeoutMs(timeoutClass),
@@ -30,7 +36,7 @@ export function sandboxDrainTiming(settings: Pick<Settings, "sandboxSnapshotTime
30
36
  }
31
37
 
32
38
  export function assertSandboxReaperActivityTimeout(
33
- settings: Pick<Settings, "sandboxSnapshotTimeoutMs">,
39
+ settings: Pick<Settings, "sandboxSnapshotTimeoutMs" | "sandboxDrainSnapshotTimeoutMs">,
34
40
  ): void {
35
41
  const timing = sandboxDrainTiming(settings);
36
42
  if (timing.snapshotTimeoutMs >= timing.captureTimeoutMs) {
@@ -709,7 +709,13 @@ export async function persistSandboxDeadlineRotationCheckpoint(
709
709
  * the user's machine IS the persistence), or a snapshot newer than the
710
710
  * interval already rides the lease.
711
711
  */
712
- export async function maybePersistWarmWorkspaceSnapshot(
712
+ export type WarmWorkspaceSnapshotPromise = Promise<boolean> & {
713
+ /** Settles only after any provider capture continuation has published or
714
+ * failed and released its exact workspace-capture admission. */
715
+ readonly settled: Promise<void>;
716
+ };
717
+
718
+ export function maybePersistWarmWorkspaceSnapshot(
713
719
  services: SandboxResumeServices,
714
720
  ids: {
715
721
  accountId: string;
@@ -723,6 +729,50 @@ export async function maybePersistWarmWorkspaceSnapshot(
723
729
  leaseEpoch: number,
724
730
  signal?: AbortSignal,
725
731
  force = false,
732
+ ): WarmWorkspaceSnapshotPromise {
733
+ let continuationRegistered = false;
734
+ let resolvePhysicalSettlement: () => void = () => undefined;
735
+ const physicalSettlement = new Promise<void>((resolve) => {
736
+ resolvePhysicalSettlement = resolve;
737
+ });
738
+ const result = persistWarmWorkspaceSnapshot(
739
+ services,
740
+ ids,
741
+ session,
742
+ leaseEpoch,
743
+ signal,
744
+ force,
745
+ (continuation) => {
746
+ continuationRegistered = true;
747
+ void continuation.then(resolvePhysicalSettlement, resolvePhysicalSettlement);
748
+ },
749
+ );
750
+ void result.then(
751
+ () => {
752
+ if (!continuationRegistered) resolvePhysicalSettlement();
753
+ },
754
+ () => {
755
+ if (!continuationRegistered) resolvePhysicalSettlement();
756
+ },
757
+ );
758
+ return Object.assign(result, { settled: physicalSettlement });
759
+ }
760
+
761
+ async function persistWarmWorkspaceSnapshot(
762
+ services: SandboxResumeServices,
763
+ ids: {
764
+ accountId: string;
765
+ workspaceId: string;
766
+ sessionId: string;
767
+ turnId: string;
768
+ attemptId: string;
769
+ sandboxGroupId: string;
770
+ },
771
+ session: unknown,
772
+ leaseEpoch: number,
773
+ signal: AbortSignal | undefined,
774
+ force: boolean,
775
+ registerOwnedContinuation: (continuation: Promise<void>) => void,
726
776
  ): Promise<boolean> {
727
777
  const { db, settings } = services;
728
778
  const intervalMs = settings.sandboxSnapshotIntervalMs;
@@ -989,6 +1039,7 @@ export async function maybePersistWarmWorkspaceSnapshot(
989
1039
  return { kind: "settled" as const, persisted: false };
990
1040
  },
991
1041
  );
1042
+ registerOwnedContinuation(settled.then(() => undefined));
992
1043
  let cancelListener: (() => void) | undefined;
993
1044
  const outcome = await Promise.race([
994
1045
  settled,
@@ -1223,13 +1274,12 @@ export async function resumeBoxForTurn(
1223
1274
  // holder is gone / its attempt is no longer the active writer" from "the lease
1224
1275
  // could not be extended" (rotation requested, draining, another epoch). The
1225
1276
  // second is NOT a reason to release: once a provider-deadline rotation is
1226
- // requested the lease heartbeat is fenced by design while the still-running
1227
- // turn checkpoints, and the turn-side heartbeat owns that orderly
1228
- // SandboxDeadlineRotationError abort. Dropping the live holder here instead
1229
- // flips the lease to draining under the running turn, the reaper terminates
1230
- // the box, and the checkpoint then fails `attempt_fenced` (the staging
1231
- // ~61-minute box-age failure). Both loops therefore converge on
1232
- // "holder alive -> never release"; this loop keeps touching the holder.
1277
+ // requested the turn-side heartbeat immediately preempts the attempt, whose
1278
+ // finalizer retains this holder until every attempt-owned writer is physically
1279
+ // drained. Dropping it here would let the reaper capture or terminate while a
1280
+ // provider operation may still be mutating /workspace. Both loops therefore
1281
+ // converge on "holder alive -> never release"; finalization performs the
1282
+ // proof-bearing release.
1233
1283
  const releaseDeadHolder = (reason: string): void => {
1234
1284
  const message = "sandbox resume holder released: holder no longer live";
1235
1285
  const fields = {