@opengeni/worker-bundle 0.20.0 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -85,8 +85,54 @@ export type TurnInputOptions = {
85
85
  materializeSerializedRunState?: (serialized: string) => Promise<string>;
86
86
  projectModelHistory?: ModelHistoryAttachmentProjector;
87
87
  loadActiveHistory?: typeof getActiveSessionHistoryItemsPaged;
88
+ /** Bounded critical-path timings; telemetry failures never affect preparation. */
89
+ onPreparationPhase?: (measurement: HistoryPreparationPhaseMeasurement) => void;
88
90
  };
89
91
 
92
+ export type HistoryPreparationPhase =
93
+ | "system_update_load"
94
+ | "current_attachment_resolution"
95
+ | "durable_history_load"
96
+ | "sandbox_envelope_load"
97
+ | "canonical_projection"
98
+ | "provider_projection"
99
+ | "attachment_ref_projection"
100
+ | "screenshot_materialization"
101
+ | "model_attachment_projection"
102
+ | "runtime_input_assembly"
103
+ | "artifact_candidate_scan";
104
+
105
+ export type HistoryPreparationPhaseMeasurement = {
106
+ phase: HistoryPreparationPhase;
107
+ outcome: "completed" | "failed";
108
+ durationSeconds: number;
109
+ };
110
+
111
+ async function measureHistoryPreparationPhase<T>(
112
+ options: Pick<TurnInputOptions, "onPreparationPhase">,
113
+ phase: HistoryPreparationPhase,
114
+ operation: () => T | Promise<T>,
115
+ ): Promise<T> {
116
+ const startedAt = performance.now();
117
+ let outcome: HistoryPreparationPhaseMeasurement["outcome"] = "completed";
118
+ try {
119
+ return await operation();
120
+ } catch (error) {
121
+ outcome = "failed";
122
+ throw error;
123
+ } finally {
124
+ try {
125
+ options.onPreparationPhase?.({
126
+ phase,
127
+ outcome,
128
+ durationSeconds: (performance.now() - startedAt) / 1_000,
129
+ });
130
+ } catch {
131
+ // Diagnostics must not change durable history or provider input.
132
+ }
133
+ }
134
+ }
135
+
90
136
  export const MAX_INLINE_MODEL_ATTACHMENT_BYTES = 20 * 1024 * 1024;
91
137
 
92
138
  export type ModelAttachmentContent = {
@@ -378,11 +424,16 @@ export async function turnInput(
378
424
  if (!trigger) {
379
425
  throw new Error("Missing trigger event");
380
426
  }
381
- const updates = await listSessionSystemUpdatesForTurn(
382
- db,
383
- trigger.workspaceId,
384
- trigger.sessionId,
385
- options.turnId,
427
+ const updates = await measureHistoryPreparationPhase(
428
+ options,
429
+ "system_update_load",
430
+ async () =>
431
+ await listSessionSystemUpdatesForTurn(
432
+ db,
433
+ trigger.workspaceId,
434
+ trigger.sessionId,
435
+ options.turnId,
436
+ ),
386
437
  );
387
438
  if (updates.length > 0) {
388
439
  const historyItemIds = new Set(
@@ -413,10 +464,10 @@ export async function turnInput(
413
464
  throw new Error("user.message payload is missing text and annotations");
414
465
  }
415
466
  const resources = Array.isArray(payload.resources) ? (payload.resources as ResourceRef[]) : [];
416
- const fileAttachments = await resolveUserMessageFileAttachments(
417
- db,
418
- trigger.workspaceId,
419
- resources,
467
+ const fileAttachments = await measureHistoryPreparationPhase(
468
+ options,
469
+ "current_attachment_resolution",
470
+ async () => await resolveUserMessageFileAttachments(db, trigger.workspaceId, resources),
420
471
  );
421
472
  const attachmentContext = userMessageAttachmentsContext(fileAttachments);
422
473
  return await messageInput(
@@ -432,6 +483,7 @@ export async function turnInput(
432
483
  options.materializeModelHistory,
433
484
  options.projectModelHistory,
434
485
  options.loadActiveHistory,
486
+ options,
435
487
  );
436
488
  }
437
489
  if (trigger.type === "system.update.delivered") {
@@ -451,6 +503,7 @@ export async function turnInput(
451
503
  options.materializeModelHistory,
452
504
  options.projectModelHistory,
453
505
  options.loadActiveHistory,
506
+ options,
454
507
  );
455
508
  }
456
509
  if (trigger.type === "user.approvalDecision") {
@@ -536,46 +589,85 @@ async function messageInput(
536
589
  materializeModelHistory?: ModelHistoryAttachmentProjector,
537
590
  projectModelHistory?: ModelHistoryAttachmentProjector,
538
591
  loadActiveHistory: typeof getActiveSessionHistoryItemsPaged = getActiveSessionHistoryItemsPaged,
592
+ preparationOptions: Pick<TurnInputOptions, "onPreparationPhase"> = {},
539
593
  ): Promise<PreparedTurnInput> {
540
- const stored = await loadActiveHistory(db, trigger.workspaceId, trigger.sessionId);
541
- const envelope = await getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId);
542
- const canonicalView = projectRejectedProviderArtifacts(stored);
543
- const canonicalProviderView = projectCanonicalHistory
544
- ? await projectCanonicalHistory(canonicalView)
545
- : canonicalView;
546
- const providerView = projectHistoryForProvider(canonicalProviderView, providerApi);
547
- const referencedHistory = withCurrentUserAttachmentRefs(providerView, currentAttachmentRefs);
548
- const materializedHistory = materializeModelHistory
549
- ? await materializeModelHistory(referencedHistory)
550
- : referencedHistory;
551
- const historyItems = projectModelHistory
552
- ? await projectModelHistory(materializedHistory)
553
- : materializedHistory;
554
- const prepared = await runtime.prepareInput(agent, {
555
- kind: "message",
556
- ...(text ? { text } : {}),
557
- ...(internalContext ? { internalContext } : {}),
558
- historyItems: historyItems as any,
559
- sandboxEnvelope: envelope,
560
- ...(projectModelHistory ? { modelInputAlreadyProjected: true } : {}),
561
- });
562
- const preparedItems = Array.isArray(prepared.input)
563
- ? new Set(prepared.input)
564
- : new Set<unknown>();
594
+ const [stored, envelope] = await Promise.all([
595
+ measureHistoryPreparationPhase(preparationOptions, "durable_history_load", async () =>
596
+ loadActiveHistory(db, trigger.workspaceId, trigger.sessionId),
597
+ ),
598
+ measureHistoryPreparationPhase(preparationOptions, "sandbox_envelope_load", async () =>
599
+ getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId),
600
+ ),
601
+ ]);
602
+ const canonicalView = await measureHistoryPreparationPhase(
603
+ preparationOptions,
604
+ "canonical_projection",
605
+ async () => {
606
+ const active = projectRejectedProviderArtifacts(stored);
607
+ return projectCanonicalHistory ? await projectCanonicalHistory(active) : active;
608
+ },
609
+ );
610
+ const providerView = await measureHistoryPreparationPhase(
611
+ preparationOptions,
612
+ "provider_projection",
613
+ () => projectHistoryForProvider(canonicalView, providerApi),
614
+ );
615
+ const referencedHistory = await measureHistoryPreparationPhase(
616
+ preparationOptions,
617
+ "attachment_ref_projection",
618
+ () => withCurrentUserAttachmentRefs(providerView, currentAttachmentRefs),
619
+ );
620
+ const materializedHistory = await measureHistoryPreparationPhase(
621
+ preparationOptions,
622
+ "screenshot_materialization",
623
+ async () =>
624
+ materializeModelHistory
625
+ ? await materializeModelHistory(referencedHistory)
626
+ : referencedHistory,
627
+ );
628
+ const historyItems = await measureHistoryPreparationPhase(
629
+ preparationOptions,
630
+ "model_attachment_projection",
631
+ async () =>
632
+ projectModelHistory ? await projectModelHistory(materializedHistory) : materializedHistory,
633
+ );
634
+ const prepared = await measureHistoryPreparationPhase(
635
+ preparationOptions,
636
+ "runtime_input_assembly",
637
+ async () =>
638
+ await runtime.prepareInput(agent, {
639
+ kind: "message",
640
+ ...(text ? { text } : {}),
641
+ ...(internalContext ? { internalContext } : {}),
642
+ historyItems: historyItems as any,
643
+ sandboxEnvelope: envelope,
644
+ ...(projectModelHistory ? { modelInputAlreadyProjected: true } : {}),
645
+ }),
646
+ );
647
+ const providerArtifactCandidates = await measureHistoryPreparationPhase(
648
+ preparationOptions,
649
+ "artifact_candidate_scan",
650
+ () => {
651
+ const preparedItems = Array.isArray(prepared.input)
652
+ ? new Set(prepared.input)
653
+ : new Set<unknown>();
654
+ return {
655
+ knownHistoryItemIds: stored.map((row) => row.id),
656
+ historyItemIds: stored
657
+ .filter(
658
+ (row) =>
659
+ row.providerArtifactInvalidatedAt === null &&
660
+ hasOpaqueProviderArtifact(row.item) &&
661
+ preparedItems.has(row.item),
662
+ )
663
+ .map((row) => row.id),
664
+ };
665
+ },
666
+ );
565
667
  return {
566
668
  input: prepared,
567
669
  persistedHistoryCount: prepared.persistedHistoryCount,
568
- providerArtifactCandidates: {
569
- knownHistoryItemIds: stored.map((row) => row.id),
570
- historyItemIds: stored
571
- .filter(
572
- (row) =>
573
- row.providerArtifactInvalidatedAt === null &&
574
- hasOpaqueProviderArtifact(row.item) &&
575
- preparedItems.has(row.item),
576
- )
577
- .map((row) => row.id),
578
- },
670
+ providerArtifactCandidates,
579
671
  };
580
672
  }
581
673
 
@@ -1,6 +1,8 @@
1
1
  import {
2
2
  settleSessionAttemptInterruptions,
3
3
  applySessionTurnSettlement,
4
+ enqueueSessionWorkflowWake,
5
+ failSessionWorkBeforeAttemptClaim,
4
6
  requestSessionTurnRecovery,
5
7
  recoverSessionDispatch,
6
8
  reconcileSessionAttemptQuiescence,
@@ -26,6 +28,7 @@ import type {
26
28
  ExpireSessionInteractionInterventionResult,
27
29
  PeekSessionWorkInput,
28
30
  FailSessionAttemptInput,
31
+ FailSessionAttemptResult,
29
32
  SettleSessionInterruptionsInput,
30
33
  MarkSessionIdleInput,
31
34
  PersistSessionAttemptQuiescenceInput,
@@ -40,6 +43,8 @@ import type {
40
43
  export type SessionStateActivityOverrides = Partial<{
41
44
  settleSessionAttemptInterruptions: typeof settleSessionAttemptInterruptions;
42
45
  applySessionTurnSettlement: typeof applySessionTurnSettlement;
46
+ enqueueSessionWorkflowWake: typeof enqueueSessionWorkflowWake;
47
+ failSessionWorkBeforeAttemptClaim: typeof failSessionWorkBeforeAttemptClaim;
43
48
  requestSessionTurnRecovery: typeof requestSessionTurnRecovery;
44
49
  recoverSessionDispatch: typeof recoverSessionDispatch;
45
50
  reconcileSessionAttemptQuiescence: typeof reconcileSessionAttemptQuiescence;
@@ -73,6 +78,10 @@ export function createSessionStateActivities(
73
78
  overrides.settleSessionAttemptInterruptions ?? settleSessionAttemptInterruptions;
74
79
  const applySessionTurnSettlementFn =
75
80
  overrides.applySessionTurnSettlement ?? applySessionTurnSettlement;
81
+ const enqueueSessionWorkflowWakeFn =
82
+ overrides.enqueueSessionWorkflowWake ?? enqueueSessionWorkflowWake;
83
+ const failSessionWorkBeforeAttemptClaimFn =
84
+ overrides.failSessionWorkBeforeAttemptClaim ?? failSessionWorkBeforeAttemptClaim;
76
85
  const requestSessionTurnRecoveryFn =
77
86
  overrides.requestSessionTurnRecovery ?? requestSessionTurnRecovery;
78
87
  const recoverSessionDispatchFn = overrides.recoverSessionDispatch ?? recoverSessionDispatch;
@@ -100,19 +109,73 @@ export function createSessionStateActivities(
100
109
  const notifyParentOfChildIdleFn = overrides.notifyParentOfChildIdle ?? notifyParentOfChildIdle;
101
110
  const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
102
111
 
103
- async function failSessionAttempt(input: FailSessionAttemptInput): Promise<void> {
112
+ async function failSessionAttempt(
113
+ input: FailSessionAttemptInput,
114
+ ): Promise<FailSessionAttemptResult> {
104
115
  const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
105
116
  const session = await requireSessionFn(db, input.workspaceId, input.sessionId);
106
- if (session.status === "failed") {
107
- return;
117
+ if (session.status === "failed" || session.status === "cancelled") {
118
+ // The activity may be retried after failure settlement committed but its
119
+ // response or event fanout was lost. Preserve terminal session truth so
120
+ // the workflow cannot interpret an idle peek as permission to synthesize
121
+ // an active-goal continuation.
122
+ return { action: "terminal" };
108
123
  }
124
+ const workflowId =
125
+ input.workflowId ?? session.temporalWorkflowId ?? `session-${input.sessionId}`;
109
126
  const turn = await getSessionTurnForAttemptFn(
110
127
  db,
111
128
  input.workspaceId,
112
129
  input.sessionId,
113
130
  input.attemptId,
114
131
  );
115
- if (!turn) return;
132
+ if (!turn) {
133
+ const attempt = await getSessionAttemptActivityRefFn(db, {
134
+ accountId: input.accountId,
135
+ workspaceId: input.workspaceId,
136
+ sessionId: input.sessionId,
137
+ attemptId: input.attemptId,
138
+ temporalWorkflowId: workflowId,
139
+ });
140
+ if (attempt) return { action: "stale" };
141
+
142
+ if (input.preClaimFailureDisposition === "permanent" && input.trigger) {
143
+ const failed = await failSessionWorkBeforeAttemptClaimFn(db, input.workspaceId, {
144
+ accountId: input.accountId,
145
+ sessionId: input.sessionId,
146
+ workflowId,
147
+ trigger: input.trigger,
148
+ error: input.error ?? "Agent turn admission failed before attempt claim.",
149
+ });
150
+ if (failed.action === "terminal") return { action: "terminal" };
151
+ if (failed.action === "stale") return { action: "stale" };
152
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, failed.events);
153
+ if (failed.turnId) {
154
+ await deliverFailedChildTurnToParentFn(
155
+ { db, bus, settings, observability, wakeSessionWorkflow },
156
+ input.workspaceId,
157
+ input.sessionId,
158
+ failed.turnId,
159
+ );
160
+ }
161
+ return { action: "failed" };
162
+ }
163
+
164
+ const requestedRetryDelayMs = input.retryDelayMs;
165
+ const retryDelayMs =
166
+ typeof requestedRetryDelayMs === "number" && Number.isFinite(requestedRetryDelayMs)
167
+ ? Math.max(1_000, Math.min(60_000, Math.trunc(requestedRetryDelayMs)))
168
+ : 60_000;
169
+ await enqueueSessionWorkflowWakeFn(db, {
170
+ accountId: input.accountId,
171
+ workspaceId: input.workspaceId,
172
+ sessionId: input.sessionId,
173
+ temporalWorkflowId: workflowId,
174
+ reason: "turn_activity_failed_before_attempt_claim",
175
+ notBefore: new Date(Date.now() + retryDelayMs),
176
+ });
177
+ return { action: "unclaimed" };
178
+ }
116
179
  const trigger = await getSessionEventFn(db, input.workspaceId, turn.triggerEventId);
117
180
  const result = await applySessionTurnSettlementFn(db, input.workspaceId, {
118
181
  sessionId: input.sessionId,
@@ -135,7 +198,7 @@ export function createSessionStateActivities(
135
198
  ],
136
199
  });
137
200
  if (result.action === "stale") {
138
- return;
201
+ return { action: "stale" };
139
202
  }
140
203
  await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
141
204
  await deliverFailedChildTurnToParentFn(
@@ -144,6 +207,7 @@ export function createSessionStateActivities(
144
207
  input.sessionId,
145
208
  turn.id,
146
209
  );
210
+ return { action: "failed" };
147
211
  }
148
212
 
149
213
  async function settleSessionInterruptions(
@@ -230,9 +230,36 @@ export type FailSessionAttemptInput = {
230
230
  workspaceId: string;
231
231
  sessionId: string;
232
232
  attemptId: string;
233
+ /** Added in v2; old Temporal histories derive the session's canonical id. */
234
+ workflowId?: string;
235
+ /** Added in v2; old Temporal histories use the bounded 60-second floor. */
236
+ retryDelayMs?: number;
237
+ /** Added in v3. Upgraded turn workers classify failures from the atomic
238
+ * admission transaction; omission is a rolling-deploy/legacy unknown and
239
+ * deliberately keeps the recoverable wake behavior. */
240
+ preClaimFailureDisposition?: PreClaimFailureDisposition;
241
+ /** The workflow admission trigger is required to re-evaluate the same
242
+ * durable admission branch before terminally settling a permanent failure. */
243
+ trigger?: RunAgentTurnInput["trigger"];
233
244
  error?: string;
234
245
  };
235
246
 
247
+ export type FailSessionAttemptResult =
248
+ | { action: "failed" }
249
+ | { action: "unclaimed" }
250
+ | { action: "terminal" }
251
+ | { action: "stale" };
252
+
253
+ export type PreClaimFailureDisposition = "retryable" | "permanent";
254
+
255
+ export type PreClaimFailureDetail = {
256
+ disposition: PreClaimFailureDisposition;
257
+ code: "db_deadlock" | "db_serialization_failure" | "db_failure" | "claim_invariant";
258
+ };
259
+
260
+ export const PRE_CLAIM_FAILURE_TYPE = "OpenGeniPreClaimFailure";
261
+ export const PRE_CLAIM_FAILURE_MESSAGE = "Agent turn admission failed before attempt claim.";
262
+
236
263
  export type RecoverDispatchInput = {
237
264
  accountId: string;
238
265
  workspaceId: string;
package/src/activities.ts CHANGED
@@ -38,6 +38,8 @@ export type {
38
38
  PersistSessionAttemptQuiescenceInput,
39
39
  ReconcileSessionAttemptQuiescenceInput,
40
40
  ReconcileSessionAttemptQuiescenceResult,
41
+ FailSessionAttemptInput,
42
+ FailSessionAttemptResult,
41
43
  RunKnowledgeSourceSyncBatchInput,
42
44
  RunKnowledgeSourceSyncBatchResult,
43
45
  RunAgentTurnInput,
@@ -176,7 +176,12 @@ export function selfhostedOpObserverForMetrics(hooks: RuntimeMetricsHooks): Self
176
176
  /** A session-scoped `machine.op.*` event mapped from a completed-op observation. */
177
177
  export type MachineOpSessionEvent = {
178
178
  type: "machine.op.failed" | "machine.op.recovered";
179
- payload: { op: string; faultClass: string; attempts: number; machineId?: string };
179
+ payload: {
180
+ op: string;
181
+ faultClass: string;
182
+ attempts: number;
183
+ machineId?: string;
184
+ };
180
185
  };
181
186
 
182
187
  /** Map an observation to a `machine.op.*` session event, or null if it is not
@@ -254,7 +259,10 @@ export class TurnLifecycleMetrics {
254
259
 
255
260
  constructor(
256
261
  private readonly observability: Observability,
257
- private readonly options: { now?: () => number; refreshIntervalMs?: number } = {},
262
+ private readonly options: {
263
+ now?: () => number;
264
+ refreshIntervalMs?: number;
265
+ } = {},
258
266
  ) {}
259
267
 
260
268
  start(turnId: string): void {
@@ -715,7 +723,11 @@ const RETAINED_PROCESS_OWNER_STATE_SET = new Set<string>(RETAINED_PROCESS_OWNER_
715
723
 
716
724
  export function recordRetainedProcessInventoryGauges(
717
725
  observability: Observability,
718
- counts: Array<{ ownerState: string; activeCount: number; terminalOwnerCount: number }>,
726
+ counts: Array<{
727
+ ownerState: string;
728
+ activeCount: number;
729
+ terminalOwnerCount: number;
730
+ }>,
719
731
  ): void {
720
732
  const normalized = new Map<string, { active: number; terminal: number }>();
721
733
  for (const ownerState of RETAINED_PROCESS_OWNER_STATES) {
@@ -861,6 +873,175 @@ export function recordModelRequestPhase(
861
873
  });
862
874
  }
863
875
 
876
+ export type TurnStartupPhase =
877
+ | "claim_and_policy"
878
+ | "turn_start_settlement"
879
+ | "credential_selection"
880
+ | "runtime_preparation"
881
+ | "sandbox_establish"
882
+ | "file_resolution"
883
+ | "tool_context_preparation"
884
+ | "tool_preparation"
885
+ | "tool_server_construction"
886
+ | "tool_required_connect"
887
+ | "tool_optional_connect"
888
+ | "tool_attempt_catalog_build"
889
+ | "tool_attempt_catalog_persist"
890
+ | "post_tool_preparation"
891
+ | "agent_construction"
892
+ | "post_agent_preparation"
893
+ | "file_materialization"
894
+ | "history_preparation"
895
+ | "history_system_update_load"
896
+ | "history_current_attachment_resolution"
897
+ | "history_durable_history_load"
898
+ | "history_sandbox_envelope_load"
899
+ | "history_canonical_projection"
900
+ | "history_provider_projection"
901
+ | "history_attachment_ref_projection"
902
+ | "history_screenshot_materialization"
903
+ | "history_model_attachment_projection"
904
+ | "history_runtime_input_assembly"
905
+ | "history_artifact_candidate_scan"
906
+ | "history_generated_image_materialization"
907
+ | "history_position_load"
908
+ | "owned_sandbox_setup"
909
+ | "provider_dispatch"
910
+ | "model_request_preparation"
911
+ | "model_sdk_serialization"
912
+ | "model_prepare_sandbox_agent_preparation"
913
+ | "model_prepare_sandbox_agent_manifest_inventory"
914
+ | "model_prepare_sandbox_session_manifest_inventory"
915
+ | "model_prepare_sandbox_manifest_apply"
916
+ | "model_prepare_sandbox_entry_materialization"
917
+ | "model_prepare_sandbox_running_check"
918
+ | "model_prepare_sandbox_start"
919
+ | "model_prepare_sandbox_client_create"
920
+ | "model_prepare_sandbox_client_resume"
921
+ | "model_prepare_sandbox_client_delete"
922
+ | "model_prepare_sandbox_client_state_serialize"
923
+ | "model_prepare_sandbox_client_reuse_check"
924
+ | "model_prepare_runner_before_mcp_tools"
925
+ | "model_prepare_mcp_tools_snapshot"
926
+ | "model_prepare_mcp_tools_before_input_filter"
927
+ | "model_prepare_input_filter_base"
928
+ | "model_prepare_input_filter_genesis"
929
+ | "model_prepare_input_filter_host"
930
+ | "model_prepare_input_filter_tool_output"
931
+ | "model_prepare_input_filter_modality"
932
+ | "model_prepare_input_filter_context"
933
+ | "model_prepare_responses_input_conversion"
934
+ | "model_prepare_responses_request_build"
935
+ | "model_credential_resolution"
936
+ | "model_wire_normalization"
937
+ | "model_request_audit"
938
+ | "stream_bootstrap";
939
+
940
+ export type TurnStartupOutcome = "completed" | "failed";
941
+ export type TurnStartupCache = "hit" | "miss" | "disabled" | "none";
942
+ export type TurnStartupCountBucket = "0" | "1" | "2-5" | "6-20" | "21+" | "unknown";
943
+ export type TurnSandboxEstablishPolicy = "eager" | "on-demand";
944
+ export type TurnSandboxEstablishReason =
945
+ | "eligible"
946
+ | "lazy_disabled"
947
+ | "machine_primary"
948
+ | "backend_none"
949
+ | "initial_run_credentials"
950
+ | "generated_video_files"
951
+ | "signed_file_resources";
952
+
953
+ const TURN_STARTUP_PHASE_BUCKETS = [
954
+ 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120,
955
+ ];
956
+
957
+ export function turnStartupCountBucket(count: number | null): TurnStartupCountBucket {
958
+ if (count === null || !Number.isFinite(count) || count < 0) return "unknown";
959
+ if (count === 0) return "0";
960
+ if (count === 1) return "1";
961
+ if (count <= 5) return "2-5";
962
+ if (count <= 20) return "6-20";
963
+ return "21+";
964
+ }
965
+
966
+ /**
967
+ * Measure the critical path from a durable turn start to provider dispatch.
968
+ * Every label is a closed or configuration-derived enum; high-cardinality turn,
969
+ * session, credential, connection, file, and model identifiers are forbidden.
970
+ */
971
+ export function recordTurnStartupPhase(
972
+ observability: Observability,
973
+ input: {
974
+ phase: TurnStartupPhase;
975
+ provider: string;
976
+ backend: string;
977
+ outcome: TurnStartupOutcome;
978
+ durationSeconds: number;
979
+ count?: number | null;
980
+ cache?: TurnStartupCache;
981
+ },
982
+ ): void {
983
+ observability.observeHistogram({
984
+ name: "opengeni_turn_startup_phase_duration_seconds",
985
+ help: "Turn startup phase duration before the model response stream begins.",
986
+ buckets: TURN_STARTUP_PHASE_BUCKETS,
987
+ labels: {
988
+ phase: input.phase,
989
+ provider: input.provider,
990
+ backend: input.backend,
991
+ outcome: input.outcome,
992
+ count_bucket: turnStartupCountBucket(input.count ?? null),
993
+ cache: input.cache ?? "none",
994
+ },
995
+ value: Math.max(0, input.durationSeconds),
996
+ });
997
+ }
998
+
999
+ /**
1000
+ * Explain why a turn did or did not keep provider sandbox work off the
1001
+ * pre-model critical path. The reason is a closed enum: never attach session,
1002
+ * sandbox, credential, or file identifiers here.
1003
+ */
1004
+ export function recordTurnSandboxEstablishPolicy(
1005
+ observability: Observability,
1006
+ input: {
1007
+ policy: TurnSandboxEstablishPolicy;
1008
+ reason: TurnSandboxEstablishReason;
1009
+ backend: string;
1010
+ },
1011
+ ): void {
1012
+ observability.incrementCounter({
1013
+ name: "opengeni_turn_sandbox_establish_policy_total",
1014
+ help: "Turn sandbox establish policy decisions by bounded reason.",
1015
+ labels: {
1016
+ policy: input.policy,
1017
+ reason: input.reason,
1018
+ backend: input.backend,
1019
+ },
1020
+ });
1021
+ }
1022
+
1023
+ export function recordTurnWorkerPreparationTotal(
1024
+ observability: Observability,
1025
+ input: {
1026
+ provider: string;
1027
+ backend: string;
1028
+ outcome: TurnStartupOutcome;
1029
+ durationSeconds: number;
1030
+ },
1031
+ ): void {
1032
+ observability.observeHistogram({
1033
+ name: "opengeni_turn_worker_preparation_duration_seconds",
1034
+ help: "Worker preparation from the durable turn-start boundary until entering the runtime; lazy SDK request preparation and model-request audit are separate phases.",
1035
+ buckets: TURN_STARTUP_PHASE_BUCKETS,
1036
+ labels: {
1037
+ provider: input.provider,
1038
+ backend: input.backend,
1039
+ outcome: input.outcome,
1040
+ },
1041
+ value: Math.max(0, input.durationSeconds),
1042
+ });
1043
+ }
1044
+
864
1045
  // ── Streaming SLIs ────────────────────────────────────────────────────────────
865
1046
  // Instruments the token-streaming pipeline so "streaming is sluggish" is a number,
866
1047
  // not a vibe. Split across the three attributable stages so an operator can tell