@opengeni/worker-bundle 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +24 -1
- package/dist/activities/run-input.d.ts +8 -0
- package/dist/activities/session-state.d.ts +5 -3
- package/dist/activities/types.d.ts +27 -0
- package/dist/{activities-control-XURE4VKN.js → activities-control-SMKF2OBE.js} +54 -7
- package/dist/{activities-control-XURE4VKN.js.map → activities-control-SMKF2OBE.js.map} +1 -1
- package/dist/activities-control.d.ts +2 -2
- package/dist/{activities-turn-6EBXGGLS.js → activities-turn-IDS3WEHA.js} +1471 -975
- package/dist/activities-turn-IDS3WEHA.js.map +1 -0
- package/dist/activities.d.ts +3 -3
- package/dist/{chunk-TYNPCG4U.js → chunk-N5TJIFCE.js} +69 -1
- package/dist/chunk-N5TJIFCE.js.map +1 -0
- package/dist/{chunk-5GRJIP5H.js → chunk-P6Z2SLAA.js} +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -2
- package/dist/observability-metrics.d.ts +37 -0
- package/dist/workflow-bundle.js +174 -4
- package/dist/workflows/session.d.ts +17 -1
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +1675 -1175
- package/src/activities/run-input.ts +137 -45
- package/src/activities/session-state.ts +69 -5
- package/src/activities/types.ts +27 -0
- package/src/activities.ts +2 -0
- package/src/observability-metrics.ts +184 -3
- package/src/workflows/session.ts +199 -8
- package/dist/activities-turn-6EBXGGLS.js.map +0 -1
- package/dist/chunk-TYNPCG4U.js.map +0 -1
- /package/dist/{chunk-5GRJIP5H.js.map → chunk-P6Z2SLAA.js.map} +0 -0
|
@@ -85,8 +85,54 @@ export type TurnInputOptions = {
|
|
|
85
85
|
materializeSerializedRunState?: (serialized: string) => Promise<string>;
|
|
86
86
|
projectModelHistory?: ModelHistoryAttachmentProjector;
|
|
87
87
|
loadActiveHistory?: typeof getActiveSessionHistoryItemsPaged;
|
|
88
|
+
/** Bounded critical-path timings; telemetry failures never affect preparation. */
|
|
89
|
+
onPreparationPhase?: (measurement: HistoryPreparationPhaseMeasurement) => void;
|
|
88
90
|
};
|
|
89
91
|
|
|
92
|
+
export type HistoryPreparationPhase =
|
|
93
|
+
| "system_update_load"
|
|
94
|
+
| "current_attachment_resolution"
|
|
95
|
+
| "durable_history_load"
|
|
96
|
+
| "sandbox_envelope_load"
|
|
97
|
+
| "canonical_projection"
|
|
98
|
+
| "provider_projection"
|
|
99
|
+
| "attachment_ref_projection"
|
|
100
|
+
| "screenshot_materialization"
|
|
101
|
+
| "model_attachment_projection"
|
|
102
|
+
| "runtime_input_assembly"
|
|
103
|
+
| "artifact_candidate_scan";
|
|
104
|
+
|
|
105
|
+
export type HistoryPreparationPhaseMeasurement = {
|
|
106
|
+
phase: HistoryPreparationPhase;
|
|
107
|
+
outcome: "completed" | "failed";
|
|
108
|
+
durationSeconds: number;
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
async function measureHistoryPreparationPhase<T>(
|
|
112
|
+
options: Pick<TurnInputOptions, "onPreparationPhase">,
|
|
113
|
+
phase: HistoryPreparationPhase,
|
|
114
|
+
operation: () => T | Promise<T>,
|
|
115
|
+
): Promise<T> {
|
|
116
|
+
const startedAt = performance.now();
|
|
117
|
+
let outcome: HistoryPreparationPhaseMeasurement["outcome"] = "completed";
|
|
118
|
+
try {
|
|
119
|
+
return await operation();
|
|
120
|
+
} catch (error) {
|
|
121
|
+
outcome = "failed";
|
|
122
|
+
throw error;
|
|
123
|
+
} finally {
|
|
124
|
+
try {
|
|
125
|
+
options.onPreparationPhase?.({
|
|
126
|
+
phase,
|
|
127
|
+
outcome,
|
|
128
|
+
durationSeconds: (performance.now() - startedAt) / 1_000,
|
|
129
|
+
});
|
|
130
|
+
} catch {
|
|
131
|
+
// Diagnostics must not change durable history or provider input.
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
90
136
|
export const MAX_INLINE_MODEL_ATTACHMENT_BYTES = 20 * 1024 * 1024;
|
|
91
137
|
|
|
92
138
|
export type ModelAttachmentContent = {
|
|
@@ -378,11 +424,16 @@ export async function turnInput(
|
|
|
378
424
|
if (!trigger) {
|
|
379
425
|
throw new Error("Missing trigger event");
|
|
380
426
|
}
|
|
381
|
-
const updates = await
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
427
|
+
const updates = await measureHistoryPreparationPhase(
|
|
428
|
+
options,
|
|
429
|
+
"system_update_load",
|
|
430
|
+
async () =>
|
|
431
|
+
await listSessionSystemUpdatesForTurn(
|
|
432
|
+
db,
|
|
433
|
+
trigger.workspaceId,
|
|
434
|
+
trigger.sessionId,
|
|
435
|
+
options.turnId,
|
|
436
|
+
),
|
|
386
437
|
);
|
|
387
438
|
if (updates.length > 0) {
|
|
388
439
|
const historyItemIds = new Set(
|
|
@@ -413,10 +464,10 @@ export async function turnInput(
|
|
|
413
464
|
throw new Error("user.message payload is missing text and annotations");
|
|
414
465
|
}
|
|
415
466
|
const resources = Array.isArray(payload.resources) ? (payload.resources as ResourceRef[]) : [];
|
|
416
|
-
const fileAttachments = await
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
resources,
|
|
467
|
+
const fileAttachments = await measureHistoryPreparationPhase(
|
|
468
|
+
options,
|
|
469
|
+
"current_attachment_resolution",
|
|
470
|
+
async () => await resolveUserMessageFileAttachments(db, trigger.workspaceId, resources),
|
|
420
471
|
);
|
|
421
472
|
const attachmentContext = userMessageAttachmentsContext(fileAttachments);
|
|
422
473
|
return await messageInput(
|
|
@@ -432,6 +483,7 @@ export async function turnInput(
|
|
|
432
483
|
options.materializeModelHistory,
|
|
433
484
|
options.projectModelHistory,
|
|
434
485
|
options.loadActiveHistory,
|
|
486
|
+
options,
|
|
435
487
|
);
|
|
436
488
|
}
|
|
437
489
|
if (trigger.type === "system.update.delivered") {
|
|
@@ -451,6 +503,7 @@ export async function turnInput(
|
|
|
451
503
|
options.materializeModelHistory,
|
|
452
504
|
options.projectModelHistory,
|
|
453
505
|
options.loadActiveHistory,
|
|
506
|
+
options,
|
|
454
507
|
);
|
|
455
508
|
}
|
|
456
509
|
if (trigger.type === "user.approvalDecision") {
|
|
@@ -536,46 +589,85 @@ async function messageInput(
|
|
|
536
589
|
materializeModelHistory?: ModelHistoryAttachmentProjector,
|
|
537
590
|
projectModelHistory?: ModelHistoryAttachmentProjector,
|
|
538
591
|
loadActiveHistory: typeof getActiveSessionHistoryItemsPaged = getActiveSessionHistoryItemsPaged,
|
|
592
|
+
preparationOptions: Pick<TurnInputOptions, "onPreparationPhase"> = {},
|
|
539
593
|
): Promise<PreparedTurnInput> {
|
|
540
|
-
const stored = await
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
const
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
594
|
+
const [stored, envelope] = await Promise.all([
|
|
595
|
+
measureHistoryPreparationPhase(preparationOptions, "durable_history_load", async () =>
|
|
596
|
+
loadActiveHistory(db, trigger.workspaceId, trigger.sessionId),
|
|
597
|
+
),
|
|
598
|
+
measureHistoryPreparationPhase(preparationOptions, "sandbox_envelope_load", async () =>
|
|
599
|
+
getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId),
|
|
600
|
+
),
|
|
601
|
+
]);
|
|
602
|
+
const canonicalView = await measureHistoryPreparationPhase(
|
|
603
|
+
preparationOptions,
|
|
604
|
+
"canonical_projection",
|
|
605
|
+
async () => {
|
|
606
|
+
const active = projectRejectedProviderArtifacts(stored);
|
|
607
|
+
return projectCanonicalHistory ? await projectCanonicalHistory(active) : active;
|
|
608
|
+
},
|
|
609
|
+
);
|
|
610
|
+
const providerView = await measureHistoryPreparationPhase(
|
|
611
|
+
preparationOptions,
|
|
612
|
+
"provider_projection",
|
|
613
|
+
() => projectHistoryForProvider(canonicalView, providerApi),
|
|
614
|
+
);
|
|
615
|
+
const referencedHistory = await measureHistoryPreparationPhase(
|
|
616
|
+
preparationOptions,
|
|
617
|
+
"attachment_ref_projection",
|
|
618
|
+
() => withCurrentUserAttachmentRefs(providerView, currentAttachmentRefs),
|
|
619
|
+
);
|
|
620
|
+
const materializedHistory = await measureHistoryPreparationPhase(
|
|
621
|
+
preparationOptions,
|
|
622
|
+
"screenshot_materialization",
|
|
623
|
+
async () =>
|
|
624
|
+
materializeModelHistory
|
|
625
|
+
? await materializeModelHistory(referencedHistory)
|
|
626
|
+
: referencedHistory,
|
|
627
|
+
);
|
|
628
|
+
const historyItems = await measureHistoryPreparationPhase(
|
|
629
|
+
preparationOptions,
|
|
630
|
+
"model_attachment_projection",
|
|
631
|
+
async () =>
|
|
632
|
+
projectModelHistory ? await projectModelHistory(materializedHistory) : materializedHistory,
|
|
633
|
+
);
|
|
634
|
+
const prepared = await measureHistoryPreparationPhase(
|
|
635
|
+
preparationOptions,
|
|
636
|
+
"runtime_input_assembly",
|
|
637
|
+
async () =>
|
|
638
|
+
await runtime.prepareInput(agent, {
|
|
639
|
+
kind: "message",
|
|
640
|
+
...(text ? { text } : {}),
|
|
641
|
+
...(internalContext ? { internalContext } : {}),
|
|
642
|
+
historyItems: historyItems as any,
|
|
643
|
+
sandboxEnvelope: envelope,
|
|
644
|
+
...(projectModelHistory ? { modelInputAlreadyProjected: true } : {}),
|
|
645
|
+
}),
|
|
646
|
+
);
|
|
647
|
+
const providerArtifactCandidates = await measureHistoryPreparationPhase(
|
|
648
|
+
preparationOptions,
|
|
649
|
+
"artifact_candidate_scan",
|
|
650
|
+
() => {
|
|
651
|
+
const preparedItems = Array.isArray(prepared.input)
|
|
652
|
+
? new Set(prepared.input)
|
|
653
|
+
: new Set<unknown>();
|
|
654
|
+
return {
|
|
655
|
+
knownHistoryItemIds: stored.map((row) => row.id),
|
|
656
|
+
historyItemIds: stored
|
|
657
|
+
.filter(
|
|
658
|
+
(row) =>
|
|
659
|
+
row.providerArtifactInvalidatedAt === null &&
|
|
660
|
+
hasOpaqueProviderArtifact(row.item) &&
|
|
661
|
+
preparedItems.has(row.item),
|
|
662
|
+
)
|
|
663
|
+
.map((row) => row.id),
|
|
664
|
+
};
|
|
665
|
+
},
|
|
666
|
+
);
|
|
565
667
|
return {
|
|
566
668
|
input: prepared,
|
|
567
669
|
persistedHistoryCount: prepared.persistedHistoryCount,
|
|
568
|
-
providerArtifactCandidates
|
|
569
|
-
knownHistoryItemIds: stored.map((row) => row.id),
|
|
570
|
-
historyItemIds: stored
|
|
571
|
-
.filter(
|
|
572
|
-
(row) =>
|
|
573
|
-
row.providerArtifactInvalidatedAt === null &&
|
|
574
|
-
hasOpaqueProviderArtifact(row.item) &&
|
|
575
|
-
preparedItems.has(row.item),
|
|
576
|
-
)
|
|
577
|
-
.map((row) => row.id),
|
|
578
|
-
},
|
|
670
|
+
providerArtifactCandidates,
|
|
579
671
|
};
|
|
580
672
|
}
|
|
581
673
|
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
settleSessionAttemptInterruptions,
|
|
3
3
|
applySessionTurnSettlement,
|
|
4
|
+
enqueueSessionWorkflowWake,
|
|
5
|
+
failSessionWorkBeforeAttemptClaim,
|
|
4
6
|
requestSessionTurnRecovery,
|
|
5
7
|
recoverSessionDispatch,
|
|
6
8
|
reconcileSessionAttemptQuiescence,
|
|
@@ -26,6 +28,7 @@ import type {
|
|
|
26
28
|
ExpireSessionInteractionInterventionResult,
|
|
27
29
|
PeekSessionWorkInput,
|
|
28
30
|
FailSessionAttemptInput,
|
|
31
|
+
FailSessionAttemptResult,
|
|
29
32
|
SettleSessionInterruptionsInput,
|
|
30
33
|
MarkSessionIdleInput,
|
|
31
34
|
PersistSessionAttemptQuiescenceInput,
|
|
@@ -40,6 +43,8 @@ import type {
|
|
|
40
43
|
export type SessionStateActivityOverrides = Partial<{
|
|
41
44
|
settleSessionAttemptInterruptions: typeof settleSessionAttemptInterruptions;
|
|
42
45
|
applySessionTurnSettlement: typeof applySessionTurnSettlement;
|
|
46
|
+
enqueueSessionWorkflowWake: typeof enqueueSessionWorkflowWake;
|
|
47
|
+
failSessionWorkBeforeAttemptClaim: typeof failSessionWorkBeforeAttemptClaim;
|
|
43
48
|
requestSessionTurnRecovery: typeof requestSessionTurnRecovery;
|
|
44
49
|
recoverSessionDispatch: typeof recoverSessionDispatch;
|
|
45
50
|
reconcileSessionAttemptQuiescence: typeof reconcileSessionAttemptQuiescence;
|
|
@@ -73,6 +78,10 @@ export function createSessionStateActivities(
|
|
|
73
78
|
overrides.settleSessionAttemptInterruptions ?? settleSessionAttemptInterruptions;
|
|
74
79
|
const applySessionTurnSettlementFn =
|
|
75
80
|
overrides.applySessionTurnSettlement ?? applySessionTurnSettlement;
|
|
81
|
+
const enqueueSessionWorkflowWakeFn =
|
|
82
|
+
overrides.enqueueSessionWorkflowWake ?? enqueueSessionWorkflowWake;
|
|
83
|
+
const failSessionWorkBeforeAttemptClaimFn =
|
|
84
|
+
overrides.failSessionWorkBeforeAttemptClaim ?? failSessionWorkBeforeAttemptClaim;
|
|
76
85
|
const requestSessionTurnRecoveryFn =
|
|
77
86
|
overrides.requestSessionTurnRecovery ?? requestSessionTurnRecovery;
|
|
78
87
|
const recoverSessionDispatchFn = overrides.recoverSessionDispatch ?? recoverSessionDispatch;
|
|
@@ -100,19 +109,73 @@ export function createSessionStateActivities(
|
|
|
100
109
|
const notifyParentOfChildIdleFn = overrides.notifyParentOfChildIdle ?? notifyParentOfChildIdle;
|
|
101
110
|
const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
|
|
102
111
|
|
|
103
|
-
async function failSessionAttempt(
|
|
112
|
+
async function failSessionAttempt(
|
|
113
|
+
input: FailSessionAttemptInput,
|
|
114
|
+
): Promise<FailSessionAttemptResult> {
|
|
104
115
|
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
105
116
|
const session = await requireSessionFn(db, input.workspaceId, input.sessionId);
|
|
106
|
-
if (session.status === "failed") {
|
|
107
|
-
|
|
117
|
+
if (session.status === "failed" || session.status === "cancelled") {
|
|
118
|
+
// The activity may be retried after failure settlement committed but its
|
|
119
|
+
// response or event fanout was lost. Preserve terminal session truth so
|
|
120
|
+
// the workflow cannot interpret an idle peek as permission to synthesize
|
|
121
|
+
// an active-goal continuation.
|
|
122
|
+
return { action: "terminal" };
|
|
108
123
|
}
|
|
124
|
+
const workflowId =
|
|
125
|
+
input.workflowId ?? session.temporalWorkflowId ?? `session-${input.sessionId}`;
|
|
109
126
|
const turn = await getSessionTurnForAttemptFn(
|
|
110
127
|
db,
|
|
111
128
|
input.workspaceId,
|
|
112
129
|
input.sessionId,
|
|
113
130
|
input.attemptId,
|
|
114
131
|
);
|
|
115
|
-
if (!turn)
|
|
132
|
+
if (!turn) {
|
|
133
|
+
const attempt = await getSessionAttemptActivityRefFn(db, {
|
|
134
|
+
accountId: input.accountId,
|
|
135
|
+
workspaceId: input.workspaceId,
|
|
136
|
+
sessionId: input.sessionId,
|
|
137
|
+
attemptId: input.attemptId,
|
|
138
|
+
temporalWorkflowId: workflowId,
|
|
139
|
+
});
|
|
140
|
+
if (attempt) return { action: "stale" };
|
|
141
|
+
|
|
142
|
+
if (input.preClaimFailureDisposition === "permanent" && input.trigger) {
|
|
143
|
+
const failed = await failSessionWorkBeforeAttemptClaimFn(db, input.workspaceId, {
|
|
144
|
+
accountId: input.accountId,
|
|
145
|
+
sessionId: input.sessionId,
|
|
146
|
+
workflowId,
|
|
147
|
+
trigger: input.trigger,
|
|
148
|
+
error: input.error ?? "Agent turn admission failed before attempt claim.",
|
|
149
|
+
});
|
|
150
|
+
if (failed.action === "terminal") return { action: "terminal" };
|
|
151
|
+
if (failed.action === "stale") return { action: "stale" };
|
|
152
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, failed.events);
|
|
153
|
+
if (failed.turnId) {
|
|
154
|
+
await deliverFailedChildTurnToParentFn(
|
|
155
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
156
|
+
input.workspaceId,
|
|
157
|
+
input.sessionId,
|
|
158
|
+
failed.turnId,
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
return { action: "failed" };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const requestedRetryDelayMs = input.retryDelayMs;
|
|
165
|
+
const retryDelayMs =
|
|
166
|
+
typeof requestedRetryDelayMs === "number" && Number.isFinite(requestedRetryDelayMs)
|
|
167
|
+
? Math.max(1_000, Math.min(60_000, Math.trunc(requestedRetryDelayMs)))
|
|
168
|
+
: 60_000;
|
|
169
|
+
await enqueueSessionWorkflowWakeFn(db, {
|
|
170
|
+
accountId: input.accountId,
|
|
171
|
+
workspaceId: input.workspaceId,
|
|
172
|
+
sessionId: input.sessionId,
|
|
173
|
+
temporalWorkflowId: workflowId,
|
|
174
|
+
reason: "turn_activity_failed_before_attempt_claim",
|
|
175
|
+
notBefore: new Date(Date.now() + retryDelayMs),
|
|
176
|
+
});
|
|
177
|
+
return { action: "unclaimed" };
|
|
178
|
+
}
|
|
116
179
|
const trigger = await getSessionEventFn(db, input.workspaceId, turn.triggerEventId);
|
|
117
180
|
const result = await applySessionTurnSettlementFn(db, input.workspaceId, {
|
|
118
181
|
sessionId: input.sessionId,
|
|
@@ -135,7 +198,7 @@ export function createSessionStateActivities(
|
|
|
135
198
|
],
|
|
136
199
|
});
|
|
137
200
|
if (result.action === "stale") {
|
|
138
|
-
return;
|
|
201
|
+
return { action: "stale" };
|
|
139
202
|
}
|
|
140
203
|
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
|
|
141
204
|
await deliverFailedChildTurnToParentFn(
|
|
@@ -144,6 +207,7 @@ export function createSessionStateActivities(
|
|
|
144
207
|
input.sessionId,
|
|
145
208
|
turn.id,
|
|
146
209
|
);
|
|
210
|
+
return { action: "failed" };
|
|
147
211
|
}
|
|
148
212
|
|
|
149
213
|
async function settleSessionInterruptions(
|
package/src/activities/types.ts
CHANGED
|
@@ -230,9 +230,36 @@ export type FailSessionAttemptInput = {
|
|
|
230
230
|
workspaceId: string;
|
|
231
231
|
sessionId: string;
|
|
232
232
|
attemptId: string;
|
|
233
|
+
/** Added in v2; old Temporal histories derive the session's canonical id. */
|
|
234
|
+
workflowId?: string;
|
|
235
|
+
/** Added in v2; old Temporal histories use the bounded 60-second floor. */
|
|
236
|
+
retryDelayMs?: number;
|
|
237
|
+
/** Added in v3. Upgraded turn workers classify failures from the atomic
|
|
238
|
+
* admission transaction; omission is a rolling-deploy/legacy unknown and
|
|
239
|
+
* deliberately keeps the recoverable wake behavior. */
|
|
240
|
+
preClaimFailureDisposition?: PreClaimFailureDisposition;
|
|
241
|
+
/** The workflow admission trigger is required to re-evaluate the same
|
|
242
|
+
* durable admission branch before terminally settling a permanent failure. */
|
|
243
|
+
trigger?: RunAgentTurnInput["trigger"];
|
|
233
244
|
error?: string;
|
|
234
245
|
};
|
|
235
246
|
|
|
247
|
+
export type FailSessionAttemptResult =
|
|
248
|
+
| { action: "failed" }
|
|
249
|
+
| { action: "unclaimed" }
|
|
250
|
+
| { action: "terminal" }
|
|
251
|
+
| { action: "stale" };
|
|
252
|
+
|
|
253
|
+
export type PreClaimFailureDisposition = "retryable" | "permanent";
|
|
254
|
+
|
|
255
|
+
export type PreClaimFailureDetail = {
|
|
256
|
+
disposition: PreClaimFailureDisposition;
|
|
257
|
+
code: "db_deadlock" | "db_serialization_failure" | "db_failure" | "claim_invariant";
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
export const PRE_CLAIM_FAILURE_TYPE = "OpenGeniPreClaimFailure";
|
|
261
|
+
export const PRE_CLAIM_FAILURE_MESSAGE = "Agent turn admission failed before attempt claim.";
|
|
262
|
+
|
|
236
263
|
export type RecoverDispatchInput = {
|
|
237
264
|
accountId: string;
|
|
238
265
|
workspaceId: string;
|
package/src/activities.ts
CHANGED
|
@@ -38,6 +38,8 @@ export type {
|
|
|
38
38
|
PersistSessionAttemptQuiescenceInput,
|
|
39
39
|
ReconcileSessionAttemptQuiescenceInput,
|
|
40
40
|
ReconcileSessionAttemptQuiescenceResult,
|
|
41
|
+
FailSessionAttemptInput,
|
|
42
|
+
FailSessionAttemptResult,
|
|
41
43
|
RunKnowledgeSourceSyncBatchInput,
|
|
42
44
|
RunKnowledgeSourceSyncBatchResult,
|
|
43
45
|
RunAgentTurnInput,
|
|
@@ -176,7 +176,12 @@ export function selfhostedOpObserverForMetrics(hooks: RuntimeMetricsHooks): Self
|
|
|
176
176
|
/** A session-scoped `machine.op.*` event mapped from a completed-op observation. */
|
|
177
177
|
export type MachineOpSessionEvent = {
|
|
178
178
|
type: "machine.op.failed" | "machine.op.recovered";
|
|
179
|
-
payload: {
|
|
179
|
+
payload: {
|
|
180
|
+
op: string;
|
|
181
|
+
faultClass: string;
|
|
182
|
+
attempts: number;
|
|
183
|
+
machineId?: string;
|
|
184
|
+
};
|
|
180
185
|
};
|
|
181
186
|
|
|
182
187
|
/** Map an observation to a `machine.op.*` session event, or null if it is not
|
|
@@ -254,7 +259,10 @@ export class TurnLifecycleMetrics {
|
|
|
254
259
|
|
|
255
260
|
constructor(
|
|
256
261
|
private readonly observability: Observability,
|
|
257
|
-
private readonly options: {
|
|
262
|
+
private readonly options: {
|
|
263
|
+
now?: () => number;
|
|
264
|
+
refreshIntervalMs?: number;
|
|
265
|
+
} = {},
|
|
258
266
|
) {}
|
|
259
267
|
|
|
260
268
|
start(turnId: string): void {
|
|
@@ -715,7 +723,11 @@ const RETAINED_PROCESS_OWNER_STATE_SET = new Set<string>(RETAINED_PROCESS_OWNER_
|
|
|
715
723
|
|
|
716
724
|
export function recordRetainedProcessInventoryGauges(
|
|
717
725
|
observability: Observability,
|
|
718
|
-
counts: Array<{
|
|
726
|
+
counts: Array<{
|
|
727
|
+
ownerState: string;
|
|
728
|
+
activeCount: number;
|
|
729
|
+
terminalOwnerCount: number;
|
|
730
|
+
}>,
|
|
719
731
|
): void {
|
|
720
732
|
const normalized = new Map<string, { active: number; terminal: number }>();
|
|
721
733
|
for (const ownerState of RETAINED_PROCESS_OWNER_STATES) {
|
|
@@ -861,6 +873,175 @@ export function recordModelRequestPhase(
|
|
|
861
873
|
});
|
|
862
874
|
}
|
|
863
875
|
|
|
876
|
+
export type TurnStartupPhase =
|
|
877
|
+
| "claim_and_policy"
|
|
878
|
+
| "turn_start_settlement"
|
|
879
|
+
| "credential_selection"
|
|
880
|
+
| "runtime_preparation"
|
|
881
|
+
| "sandbox_establish"
|
|
882
|
+
| "file_resolution"
|
|
883
|
+
| "tool_context_preparation"
|
|
884
|
+
| "tool_preparation"
|
|
885
|
+
| "tool_server_construction"
|
|
886
|
+
| "tool_required_connect"
|
|
887
|
+
| "tool_optional_connect"
|
|
888
|
+
| "tool_attempt_catalog_build"
|
|
889
|
+
| "tool_attempt_catalog_persist"
|
|
890
|
+
| "post_tool_preparation"
|
|
891
|
+
| "agent_construction"
|
|
892
|
+
| "post_agent_preparation"
|
|
893
|
+
| "file_materialization"
|
|
894
|
+
| "history_preparation"
|
|
895
|
+
| "history_system_update_load"
|
|
896
|
+
| "history_current_attachment_resolution"
|
|
897
|
+
| "history_durable_history_load"
|
|
898
|
+
| "history_sandbox_envelope_load"
|
|
899
|
+
| "history_canonical_projection"
|
|
900
|
+
| "history_provider_projection"
|
|
901
|
+
| "history_attachment_ref_projection"
|
|
902
|
+
| "history_screenshot_materialization"
|
|
903
|
+
| "history_model_attachment_projection"
|
|
904
|
+
| "history_runtime_input_assembly"
|
|
905
|
+
| "history_artifact_candidate_scan"
|
|
906
|
+
| "history_generated_image_materialization"
|
|
907
|
+
| "history_position_load"
|
|
908
|
+
| "owned_sandbox_setup"
|
|
909
|
+
| "provider_dispatch"
|
|
910
|
+
| "model_request_preparation"
|
|
911
|
+
| "model_sdk_serialization"
|
|
912
|
+
| "model_prepare_sandbox_agent_preparation"
|
|
913
|
+
| "model_prepare_sandbox_agent_manifest_inventory"
|
|
914
|
+
| "model_prepare_sandbox_session_manifest_inventory"
|
|
915
|
+
| "model_prepare_sandbox_manifest_apply"
|
|
916
|
+
| "model_prepare_sandbox_entry_materialization"
|
|
917
|
+
| "model_prepare_sandbox_running_check"
|
|
918
|
+
| "model_prepare_sandbox_start"
|
|
919
|
+
| "model_prepare_sandbox_client_create"
|
|
920
|
+
| "model_prepare_sandbox_client_resume"
|
|
921
|
+
| "model_prepare_sandbox_client_delete"
|
|
922
|
+
| "model_prepare_sandbox_client_state_serialize"
|
|
923
|
+
| "model_prepare_sandbox_client_reuse_check"
|
|
924
|
+
| "model_prepare_runner_before_mcp_tools"
|
|
925
|
+
| "model_prepare_mcp_tools_snapshot"
|
|
926
|
+
| "model_prepare_mcp_tools_before_input_filter"
|
|
927
|
+
| "model_prepare_input_filter_base"
|
|
928
|
+
| "model_prepare_input_filter_genesis"
|
|
929
|
+
| "model_prepare_input_filter_host"
|
|
930
|
+
| "model_prepare_input_filter_tool_output"
|
|
931
|
+
| "model_prepare_input_filter_modality"
|
|
932
|
+
| "model_prepare_input_filter_context"
|
|
933
|
+
| "model_prepare_responses_input_conversion"
|
|
934
|
+
| "model_prepare_responses_request_build"
|
|
935
|
+
| "model_credential_resolution"
|
|
936
|
+
| "model_wire_normalization"
|
|
937
|
+
| "model_request_audit"
|
|
938
|
+
| "stream_bootstrap";
|
|
939
|
+
|
|
940
|
+
export type TurnStartupOutcome = "completed" | "failed";
|
|
941
|
+
export type TurnStartupCache = "hit" | "miss" | "disabled" | "none";
|
|
942
|
+
export type TurnStartupCountBucket = "0" | "1" | "2-5" | "6-20" | "21+" | "unknown";
|
|
943
|
+
export type TurnSandboxEstablishPolicy = "eager" | "on-demand";
|
|
944
|
+
export type TurnSandboxEstablishReason =
|
|
945
|
+
| "eligible"
|
|
946
|
+
| "lazy_disabled"
|
|
947
|
+
| "machine_primary"
|
|
948
|
+
| "backend_none"
|
|
949
|
+
| "initial_run_credentials"
|
|
950
|
+
| "generated_video_files"
|
|
951
|
+
| "signed_file_resources";
|
|
952
|
+
|
|
953
|
+
const TURN_STARTUP_PHASE_BUCKETS = [
|
|
954
|
+
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120,
|
|
955
|
+
];
|
|
956
|
+
|
|
957
|
+
export function turnStartupCountBucket(count: number | null): TurnStartupCountBucket {
|
|
958
|
+
if (count === null || !Number.isFinite(count) || count < 0) return "unknown";
|
|
959
|
+
if (count === 0) return "0";
|
|
960
|
+
if (count === 1) return "1";
|
|
961
|
+
if (count <= 5) return "2-5";
|
|
962
|
+
if (count <= 20) return "6-20";
|
|
963
|
+
return "21+";
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
/**
|
|
967
|
+
* Measure the critical path from a durable turn start to provider dispatch.
|
|
968
|
+
* Every label is a closed or configuration-derived enum; high-cardinality turn,
|
|
969
|
+
* session, credential, connection, file, and model identifiers are forbidden.
|
|
970
|
+
*/
|
|
971
|
+
export function recordTurnStartupPhase(
|
|
972
|
+
observability: Observability,
|
|
973
|
+
input: {
|
|
974
|
+
phase: TurnStartupPhase;
|
|
975
|
+
provider: string;
|
|
976
|
+
backend: string;
|
|
977
|
+
outcome: TurnStartupOutcome;
|
|
978
|
+
durationSeconds: number;
|
|
979
|
+
count?: number | null;
|
|
980
|
+
cache?: TurnStartupCache;
|
|
981
|
+
},
|
|
982
|
+
): void {
|
|
983
|
+
observability.observeHistogram({
|
|
984
|
+
name: "opengeni_turn_startup_phase_duration_seconds",
|
|
985
|
+
help: "Turn startup phase duration before the model response stream begins.",
|
|
986
|
+
buckets: TURN_STARTUP_PHASE_BUCKETS,
|
|
987
|
+
labels: {
|
|
988
|
+
phase: input.phase,
|
|
989
|
+
provider: input.provider,
|
|
990
|
+
backend: input.backend,
|
|
991
|
+
outcome: input.outcome,
|
|
992
|
+
count_bucket: turnStartupCountBucket(input.count ?? null),
|
|
993
|
+
cache: input.cache ?? "none",
|
|
994
|
+
},
|
|
995
|
+
value: Math.max(0, input.durationSeconds),
|
|
996
|
+
});
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
/**
|
|
1000
|
+
* Explain why a turn did or did not keep provider sandbox work off the
|
|
1001
|
+
* pre-model critical path. The reason is a closed enum: never attach session,
|
|
1002
|
+
* sandbox, credential, or file identifiers here.
|
|
1003
|
+
*/
|
|
1004
|
+
export function recordTurnSandboxEstablishPolicy(
|
|
1005
|
+
observability: Observability,
|
|
1006
|
+
input: {
|
|
1007
|
+
policy: TurnSandboxEstablishPolicy;
|
|
1008
|
+
reason: TurnSandboxEstablishReason;
|
|
1009
|
+
backend: string;
|
|
1010
|
+
},
|
|
1011
|
+
): void {
|
|
1012
|
+
observability.incrementCounter({
|
|
1013
|
+
name: "opengeni_turn_sandbox_establish_policy_total",
|
|
1014
|
+
help: "Turn sandbox establish policy decisions by bounded reason.",
|
|
1015
|
+
labels: {
|
|
1016
|
+
policy: input.policy,
|
|
1017
|
+
reason: input.reason,
|
|
1018
|
+
backend: input.backend,
|
|
1019
|
+
},
|
|
1020
|
+
});
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
export function recordTurnWorkerPreparationTotal(
|
|
1024
|
+
observability: Observability,
|
|
1025
|
+
input: {
|
|
1026
|
+
provider: string;
|
|
1027
|
+
backend: string;
|
|
1028
|
+
outcome: TurnStartupOutcome;
|
|
1029
|
+
durationSeconds: number;
|
|
1030
|
+
},
|
|
1031
|
+
): void {
|
|
1032
|
+
observability.observeHistogram({
|
|
1033
|
+
name: "opengeni_turn_worker_preparation_duration_seconds",
|
|
1034
|
+
help: "Worker preparation from the durable turn-start boundary until entering the runtime; lazy SDK request preparation and model-request audit are separate phases.",
|
|
1035
|
+
buckets: TURN_STARTUP_PHASE_BUCKETS,
|
|
1036
|
+
labels: {
|
|
1037
|
+
provider: input.provider,
|
|
1038
|
+
backend: input.backend,
|
|
1039
|
+
outcome: input.outcome,
|
|
1040
|
+
},
|
|
1041
|
+
value: Math.max(0, input.durationSeconds),
|
|
1042
|
+
});
|
|
1043
|
+
}
|
|
1044
|
+
|
|
864
1045
|
// ── Streaming SLIs ────────────────────────────────────────────────────────────
|
|
865
1046
|
// Instruments the token-streaming pipeline so "streaming is sluggish" is a number,
|
|
866
1047
|
// not a vibe. Split across the three attributable stages so an operator can tell
|