@narumitw/pi-otel 0.0.0-stage → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +170 -0
- package/LICENSE +21 -0
- package/README.md +371 -2
- package/dist/chunks/chunk-IXR5MVGN.ts +156 -0
- package/dist/chunks/chunk-IXR5MVGN.ts.map +7 -0
- package/dist/chunks/runtime-KWIFBNQ2.ts +404 -0
- package/dist/chunks/runtime-KWIFBNQ2.ts.map +7 -0
- package/dist/config.d.ts +36 -0
- package/dist/durable-projection.d.ts +9 -0
- package/dist/durable-recorder.d.ts +57 -0
- package/dist/durable-state.d.ts +43 -0
- package/dist/durable.d.ts +25 -0
- package/dist/durable.js +897 -0
- package/dist/durable.js.map +7 -0
- package/dist/git.d.ts +5 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1736 -0
- package/dist/index.js.map +7 -0
- package/dist/index.ts +1878 -0
- package/dist/index.ts.map +7 -0
- package/dist/library-chunks/chunk-BBL32ZV5.js +84 -0
- package/dist/library-chunks/chunk-BBL32ZV5.js.map +7 -0
- package/dist/library-chunks/chunk-ROM6FIM7.js +78 -0
- package/dist/library-chunks/chunk-ROM6FIM7.js.map +7 -0
- package/dist/library-chunks/chunk-ZCEZBO2T.js +158 -0
- package/dist/library-chunks/chunk-ZCEZBO2T.js.map +7 -0
- package/dist/library-chunks/runtime-7TSAKORH.js +406 -0
- package/dist/library-chunks/runtime-7TSAKORH.js.map +7 -0
- package/dist/otel.d.ts +13 -0
- package/dist/otlp-settings.d.ts +9 -0
- package/dist/pi-session.d.ts +43 -0
- package/dist/public-api.d.ts +2 -0
- package/dist/runtime-api.d.ts +6 -0
- package/dist/runtime-core.d.ts +26 -0
- package/dist/runtime-defaults.d.ts +1 -0
- package/dist/runtime.d.ts +26 -0
- package/dist/sanitizer.d.ts +2 -0
- package/dist/tracing.d.ts +199 -0
- package/docs/durable.md +128 -0
- package/package.json +87 -4
- package/src/config.ts +330 -0
- package/src/durable-projection.ts +112 -0
- package/src/durable-recorder.ts +539 -0
- package/src/durable-state.ts +135 -0
- package/src/durable.ts +364 -0
- package/src/git.ts +64 -0
- package/src/index.ts +12 -0
- package/src/otel.ts +397 -0
- package/src/otlp-settings.ts +80 -0
- package/src/pi-session.ts +570 -0
- package/src/public-api.ts +13 -0
- package/src/runtime-api.ts +16 -0
- package/src/runtime-core.ts +114 -0
- package/src/runtime-defaults.ts +1 -0
- package/src/runtime.ts +509 -0
- package/src/sanitizer.ts +157 -0
- package/src/tracing.ts +946 -0
|
@@ -0,0 +1,539 @@
|
|
|
1
|
+
import type { EntryId, EntryRecord, SubmissionId, SubmissionRecord } from "@earendil-works/pi-durable";
|
|
2
|
+
import { durableAttributes, durableContent, durableDiagnosticCount, durableUsage } from "./durable-projection.js";
|
|
3
|
+
import type { DurableCommit, DurableLive, DurableSnapshot, DurableTask } from "./durable-state.js";
|
|
4
|
+
import { sanitizeTraceValue } from "./sanitizer.js";
|
|
5
|
+
import type { Observation, ObservationAttributes, ObservationType, TraceBackend } from "./tracing.js";
|
|
6
|
+
|
|
7
|
+
export interface DurableCorrelation {
|
|
8
|
+
requestId?: string;
|
|
9
|
+
userId?: string;
|
|
10
|
+
applicationId?: string;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface DurableRecorderOptions extends DurableCorrelation {
|
|
14
|
+
conversationId: number;
|
|
15
|
+
sessionId?: string;
|
|
16
|
+
traceName?: string;
|
|
17
|
+
captureContent?: boolean;
|
|
18
|
+
metadata?: Readonly<Record<string, unknown>>;
|
|
19
|
+
submissionCorrelation?: (record: SubmissionRecord) => DurableCorrelation;
|
|
20
|
+
onTraceId?: (traceId: string) => void | Promise<void>;
|
|
21
|
+
onError?: (stage: string) => void | Promise<void>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const SAFE_UNANSWERED_REASONS = new Set([
|
|
25
|
+
"aborted",
|
|
26
|
+
"model_error",
|
|
27
|
+
"no_model",
|
|
28
|
+
"reset",
|
|
29
|
+
"stale",
|
|
30
|
+
"faulted",
|
|
31
|
+
"missing_task",
|
|
32
|
+
"task_too_old",
|
|
33
|
+
"migration_failed",
|
|
34
|
+
"orphaned",
|
|
35
|
+
"interrupted",
|
|
36
|
+
"restart_limit",
|
|
37
|
+
"nothing_to_answer",
|
|
38
|
+
"turn_limit",
|
|
39
|
+
"run_failed",
|
|
40
|
+
]);
|
|
41
|
+
|
|
42
|
+
interface Span {
|
|
43
|
+
observation?: Observation;
|
|
44
|
+
ended: boolean;
|
|
45
|
+
}
|
|
46
|
+
interface SubmissionSpan {
|
|
47
|
+
span: Span;
|
|
48
|
+
correlation: DurableCorrelation;
|
|
49
|
+
terminal: boolean;
|
|
50
|
+
}
|
|
51
|
+
interface RunSpan {
|
|
52
|
+
span: Span;
|
|
53
|
+
primary?: SubmissionId;
|
|
54
|
+
identity: string;
|
|
55
|
+
inputs: readonly SubmissionId[];
|
|
56
|
+
correlation: DurableCorrelation;
|
|
57
|
+
round: number;
|
|
58
|
+
generation?: Span;
|
|
59
|
+
generationKey?: string;
|
|
60
|
+
tools: Map<string, Span>;
|
|
61
|
+
completedTools: Set<string>;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Durable state machine using the same trace backend, ownership and capture core as the Pi adapter. */
|
|
65
|
+
export class DurableRecorder {
|
|
66
|
+
private readonly root: Span;
|
|
67
|
+
private run?: RunSpan;
|
|
68
|
+
private readonly submissions = new Map<SubmissionId, SubmissionSpan>();
|
|
69
|
+
private readonly pendingSubmissions = new Set<SubmissionId>();
|
|
70
|
+
private readonly entries = new Set<number>();
|
|
71
|
+
private readonly completedRuns = new Set<string>();
|
|
72
|
+
private readonly completedGenerations = new Set<string>();
|
|
73
|
+
private live: DurableLive = { tools: [] };
|
|
74
|
+
private baselineEntry = 0;
|
|
75
|
+
private disposed = false;
|
|
76
|
+
private gaps = 0;
|
|
77
|
+
|
|
78
|
+
constructor(
|
|
79
|
+
private readonly backend: TraceBackend,
|
|
80
|
+
private readonly options: DurableRecorderOptions,
|
|
81
|
+
) {
|
|
82
|
+
this.root = this.start(options.traceName ?? "pi.durable.conversation", "agent", undefined, {
|
|
83
|
+
metadata: {
|
|
84
|
+
...this.customMetadata(),
|
|
85
|
+
"pi.runtime": "durable",
|
|
86
|
+
"pi.durable.schema_version": "1",
|
|
87
|
+
"pi.durable.conversation_id": options.conversationId,
|
|
88
|
+
...(options.applicationId ? { "pi.application.id": options.applicationId } : {}),
|
|
89
|
+
},
|
|
90
|
+
});
|
|
91
|
+
this.safe(() => {
|
|
92
|
+
this.root.observation?.updateTrace?.(
|
|
93
|
+
durableAttributes({
|
|
94
|
+
name: options.traceName ?? "pi.durable.conversation",
|
|
95
|
+
sessionId: options.sessionId ?? String(options.conversationId),
|
|
96
|
+
...(options.userId ? { userId: options.userId } : {}),
|
|
97
|
+
metadata: this.customMetadata(),
|
|
98
|
+
version: "durable-1",
|
|
99
|
+
}),
|
|
100
|
+
);
|
|
101
|
+
});
|
|
102
|
+
const traceId = this.root.observation?.traceId;
|
|
103
|
+
if (traceId) this.callback("trace-id callback", () => options.onTraceId?.(traceId));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
get submissionIds(): readonly SubmissionId[] {
|
|
107
|
+
return [...this.pendingSubmissions];
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
report(stage: string): void {
|
|
111
|
+
this.callback("error callback", () => this.options.onError?.(stage), false);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
private callback(stage: string, action: () => void | Promise<void>, report = true): void {
|
|
115
|
+
try {
|
|
116
|
+
Promise.resolve(action()).catch(() => {
|
|
117
|
+
if (report) this.report(stage);
|
|
118
|
+
});
|
|
119
|
+
} catch {
|
|
120
|
+
if (report) this.report(stage);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
private customMetadata(): Record<string, unknown> {
|
|
125
|
+
const value = sanitizeTraceValue(this.options.metadata, true);
|
|
126
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
|
|
127
|
+
return Object.fromEntries(Object.entries(value).filter(([key]) => !key.startsWith("pi.")));
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
private safe(action: () => void): void {
|
|
131
|
+
try {
|
|
132
|
+
action();
|
|
133
|
+
} catch {
|
|
134
|
+
this.report("exporter observation");
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
private start(name: string, type: ObservationType, parent?: Span, attributes: ObservationAttributes = {}): Span {
|
|
139
|
+
if (this.disposed || (parent && (parent.ended || !parent.observation))) return { ended: true };
|
|
140
|
+
const span: Span = { ended: false };
|
|
141
|
+
const inherited = parent === this.run?.span ? this.correlationAttributes(this.run?.correlation ?? {}) : {};
|
|
142
|
+
this.safe(() => {
|
|
143
|
+
span.observation = this.backend.start(
|
|
144
|
+
String(sanitizeTraceValue(name, true)),
|
|
145
|
+
durableAttributes({
|
|
146
|
+
version: "durable-1",
|
|
147
|
+
sessionId: this.options.sessionId ?? String(this.options.conversationId),
|
|
148
|
+
...(this.options.userId ? { userId: this.options.userId } : {}),
|
|
149
|
+
...inherited,
|
|
150
|
+
...attributes,
|
|
151
|
+
metadata: { ...inherited.metadata, ...attributes.metadata },
|
|
152
|
+
}),
|
|
153
|
+
{ asType: type, parent: parent?.observation },
|
|
154
|
+
);
|
|
155
|
+
});
|
|
156
|
+
// A missing parent handle must never become a new exporter root for descendants.
|
|
157
|
+
if (!span.observation) span.ended = true;
|
|
158
|
+
if (this.disposed) this.interrupted(span, "Observer disposed during exporter start.");
|
|
159
|
+
return span;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
private update(span: Span, attributes: ObservationAttributes): void {
|
|
163
|
+
if (!span.ended)
|
|
164
|
+
this.safe(() => {
|
|
165
|
+
span.observation?.update(durableAttributes(attributes));
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
private end(span: Span | undefined, attributes: ObservationAttributes = {}): void {
|
|
170
|
+
if (!span || span.ended) return;
|
|
171
|
+
span.ended = true;
|
|
172
|
+
const observation = span.observation;
|
|
173
|
+
// Release ended SDK handles/payloads, and claim the end before any callback can reenter disposal.
|
|
174
|
+
span.observation = undefined;
|
|
175
|
+
this.safe(() => {
|
|
176
|
+
observation?.update(durableAttributes(attributes));
|
|
177
|
+
});
|
|
178
|
+
this.safe(() => {
|
|
179
|
+
observation?.end();
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
private interrupted(span: Span | undefined, reason: string): void {
|
|
184
|
+
this.end(span, {
|
|
185
|
+
level: "WARNING",
|
|
186
|
+
statusMessage: reason,
|
|
187
|
+
metadata: { "pi.durable.outcome": "observation_interrupted" },
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private correlationAttributes(correlation: DurableCorrelation): ObservationAttributes {
|
|
192
|
+
return {
|
|
193
|
+
metadata: {
|
|
194
|
+
...(correlation.userId ? { "pi.user.id": correlation.userId } : {}),
|
|
195
|
+
...(correlation.requestId ? { "pi.request.id": correlation.requestId } : {}),
|
|
196
|
+
...(correlation.applicationId ? { "pi.application.id": correlation.applicationId } : {}),
|
|
197
|
+
},
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
async submission(
|
|
202
|
+
record: SubmissionRecord,
|
|
203
|
+
readEntry: (id: EntryId) => Promise<EntryRecord | undefined>,
|
|
204
|
+
correlation?: DurableCorrelation,
|
|
205
|
+
snapshot = false,
|
|
206
|
+
): Promise<void> {
|
|
207
|
+
if (this.disposed || record.conversationId !== this.options.conversationId) return;
|
|
208
|
+
let state = this.submissions.get(record.id);
|
|
209
|
+
if (!state) {
|
|
210
|
+
let host: DurableCorrelation = {};
|
|
211
|
+
try {
|
|
212
|
+
host = this.options.submissionCorrelation?.(record) ?? {};
|
|
213
|
+
} catch {
|
|
214
|
+
this.report("submission correlation");
|
|
215
|
+
}
|
|
216
|
+
if (this.disposed) return;
|
|
217
|
+
const ids = sanitizeTraceValue(
|
|
218
|
+
{
|
|
219
|
+
requestId: record.requestId,
|
|
220
|
+
userId: this.options.userId,
|
|
221
|
+
applicationId: this.options.applicationId,
|
|
222
|
+
...host,
|
|
223
|
+
...correlation,
|
|
224
|
+
},
|
|
225
|
+
true,
|
|
226
|
+
) as DurableCorrelation;
|
|
227
|
+
state = {
|
|
228
|
+
span: this.start("pi.durable.submission", "span", this.root, {
|
|
229
|
+
...this.correlationAttributes(ids),
|
|
230
|
+
metadata: {
|
|
231
|
+
...this.correlationAttributes(ids).metadata,
|
|
232
|
+
"pi.durable.submission_id": record.id,
|
|
233
|
+
"pi.durable.submission_type": record.type,
|
|
234
|
+
"pi.durable.snapshot_observation": snapshot,
|
|
235
|
+
},
|
|
236
|
+
}),
|
|
237
|
+
correlation: ids,
|
|
238
|
+
terminal: false,
|
|
239
|
+
};
|
|
240
|
+
if (this.disposed) return;
|
|
241
|
+
this.submissions.set(record.id, state);
|
|
242
|
+
this.pendingSubmissions.add(record.id);
|
|
243
|
+
}
|
|
244
|
+
if (state.terminal) return;
|
|
245
|
+
if (correlation) {
|
|
246
|
+
state.correlation = sanitizeTraceValue({ ...state.correlation, ...correlation }, true) as DurableCorrelation;
|
|
247
|
+
this.update(state.span, this.correlationAttributes(state.correlation));
|
|
248
|
+
}
|
|
249
|
+
const run = this.run;
|
|
250
|
+
if (run?.primary === record.id) {
|
|
251
|
+
run.correlation = state.correlation;
|
|
252
|
+
const attributes = this.correlationAttributes(state.correlation);
|
|
253
|
+
this.update(run.span, attributes);
|
|
254
|
+
if (run.generation) this.update(run.generation, attributes);
|
|
255
|
+
for (const tool of run.tools.values()) this.update(tool, attributes);
|
|
256
|
+
}
|
|
257
|
+
this.update(state.span, { metadata: { "pi.durable.submission_status": record.status } });
|
|
258
|
+
if (record.entry !== undefined && record.type === "input") {
|
|
259
|
+
const entry = await readEntry(record.entry);
|
|
260
|
+
if (this.disposed || state.terminal) return;
|
|
261
|
+
this.update(state.span, { input: durableContent(entry?.model, this.options.captureContent ?? true) });
|
|
262
|
+
}
|
|
263
|
+
if (record.status === "done" || record.status === "unanswered") {
|
|
264
|
+
let output: unknown;
|
|
265
|
+
if (record.status === "done" && record.type !== "write") {
|
|
266
|
+
const entry = await readEntry(record.answer);
|
|
267
|
+
if (this.disposed || state.terminal) return;
|
|
268
|
+
// The answer ID is authoritative; no streamed fragments or last intermediate assistant inference.
|
|
269
|
+
output = durableContent(entry?.model, this.options.captureContent ?? true);
|
|
270
|
+
}
|
|
271
|
+
const reason =
|
|
272
|
+
record.status === "unanswered" && SAFE_UNANSWERED_REASONS.has(record.reason) ? record.reason : undefined;
|
|
273
|
+
state.terminal = true;
|
|
274
|
+
this.pendingSubmissions.delete(record.id);
|
|
275
|
+
this.end(state.span, {
|
|
276
|
+
...(output !== undefined ? { output } : {}),
|
|
277
|
+
level:
|
|
278
|
+
record.status === "done"
|
|
279
|
+
? "DEFAULT"
|
|
280
|
+
: reason === "model_error" || reason === "no_model" || reason === "faulted"
|
|
281
|
+
? "ERROR"
|
|
282
|
+
: "WARNING",
|
|
283
|
+
// Only known structural reason codes are safe; arbitrary reason/detail can contain provider secrets.
|
|
284
|
+
metadata: {
|
|
285
|
+
"pi.durable.submission_status": record.status,
|
|
286
|
+
"pi.durable.answer_entry_id": record.answer,
|
|
287
|
+
...(record.status === "unanswered" ? { "pi.durable.unanswered_reason": reason ?? "unknown" } : {}),
|
|
288
|
+
},
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
snapshot(snapshot: DurableSnapshot, initial = false): void {
|
|
294
|
+
if (this.disposed) return;
|
|
295
|
+
for (const entry of snapshot.entries) this.baselineEntry = Math.max(this.baselineEntry, entry.id);
|
|
296
|
+
if (!initial) {
|
|
297
|
+
this.gaps += 1;
|
|
298
|
+
this.update(this.root, {
|
|
299
|
+
metadata: { "pi.durable.resync_count": this.gaps, "pi.durable.history_complete": false },
|
|
300
|
+
});
|
|
301
|
+
} else
|
|
302
|
+
this.update(this.root, {
|
|
303
|
+
metadata: { "pi.durable.history_complete": false, "pi.durable.attachment": "snapshot" },
|
|
304
|
+
});
|
|
305
|
+
if (this.run && !this.sameRun(snapshot.live.run))
|
|
306
|
+
this.closeRun("Run boundary not observed; snapshot replaced the observation.");
|
|
307
|
+
if (this.run) this.closeRound("Snapshot replaced generation/tool observation; activity may have been missed.");
|
|
308
|
+
this.live = snapshot.live;
|
|
309
|
+
if (snapshot.live.run) {
|
|
310
|
+
this.beginRun(snapshot.live.run, true);
|
|
311
|
+
// A snapshot never turns transcript, partial messages or running slots into historical starts.
|
|
312
|
+
if (this.run) this.update(this.run.span, { metadata: { "pi.durable.observation_incomplete": true } });
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
private sameRun(run: DurableLive["run"]): boolean {
|
|
317
|
+
if (!run || !this.run) return false;
|
|
318
|
+
// afterRun settles the inputs before its tasks finish; an empty inputs list is not a run boundary.
|
|
319
|
+
return (
|
|
320
|
+
(this.run.primary !== undefined && run.inputs[0] === this.run.primary) ||
|
|
321
|
+
(run.afterRun && this.live.run?.taskId === run.taskId)
|
|
322
|
+
);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
private beginRun(live: NonNullable<DurableLive["run"]>, snapshot = false): void {
|
|
326
|
+
if (this.disposed) return;
|
|
327
|
+
const primary = live.inputs[0];
|
|
328
|
+
if (this.sameRun(live)) {
|
|
329
|
+
if (!this.run) return;
|
|
330
|
+
this.run.inputs = [...live.inputs];
|
|
331
|
+
this.update(this.run.span, { metadata: { "pi.durable.inputs": this.run.inputs } });
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
const identity = primary === undefined ? `task:${live.taskId}` : `submission:${primary}`;
|
|
335
|
+
if (this.completedRuns.has(identity)) return;
|
|
336
|
+
if (this.run) this.closeRun("Unexpected run replacement.");
|
|
337
|
+
const correlation = (primary === undefined ? undefined : this.submissions.get(primary)?.correlation) ?? {};
|
|
338
|
+
this.run = {
|
|
339
|
+
span: this.start("pi.durable.run", "agent", this.root, {
|
|
340
|
+
...this.correlationAttributes(correlation),
|
|
341
|
+
metadata: {
|
|
342
|
+
...this.correlationAttributes(correlation).metadata,
|
|
343
|
+
...(primary === undefined ? {} : { "pi.durable.primary_submission_id": primary }),
|
|
344
|
+
"pi.durable.inputs": [...live.inputs],
|
|
345
|
+
"pi.durable.snapshot_observation": snapshot,
|
|
346
|
+
},
|
|
347
|
+
}),
|
|
348
|
+
primary,
|
|
349
|
+
identity,
|
|
350
|
+
inputs: [...live.inputs],
|
|
351
|
+
correlation,
|
|
352
|
+
round: 0,
|
|
353
|
+
tools: new Map(),
|
|
354
|
+
completedTools: new Set(),
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
private closeRound(reason: string): void {
|
|
359
|
+
const run = this.run;
|
|
360
|
+
if (!run) return;
|
|
361
|
+
this.interrupted(run.generation, reason);
|
|
362
|
+
run.generation = undefined;
|
|
363
|
+
for (const tool of run.tools.values()) this.interrupted(tool, reason);
|
|
364
|
+
run.tools.clear();
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
private closeRun(reason?: string): void {
|
|
368
|
+
const run = this.run;
|
|
369
|
+
if (!run) return;
|
|
370
|
+
this.closeRound(reason ?? "Run ended without an observed descendant completion.");
|
|
371
|
+
this.end(
|
|
372
|
+
run.span,
|
|
373
|
+
reason
|
|
374
|
+
? { level: "WARNING", statusMessage: reason, metadata: { "pi.durable.observation_incomplete": true } }
|
|
375
|
+
: {},
|
|
376
|
+
);
|
|
377
|
+
this.completedRuns.add(run.identity);
|
|
378
|
+
this.run = undefined;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
private generation(key?: string): Span | undefined {
|
|
382
|
+
const run = this.run;
|
|
383
|
+
if (this.disposed || !run || (key && this.completedGenerations.has(key))) return undefined;
|
|
384
|
+
if (run.generation && key && run.generationKey !== key) {
|
|
385
|
+
this.interrupted(run.generation, "Generation boundary was not observed.");
|
|
386
|
+
run.generation = undefined;
|
|
387
|
+
}
|
|
388
|
+
if (!run.generation) {
|
|
389
|
+
run.round += 1;
|
|
390
|
+
run.generationKey = key;
|
|
391
|
+
run.generation = this.start("pi.durable.llm", "generation", run.span, {
|
|
392
|
+
metadata: { "pi.durable.observed_generation_index": run.round },
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
return run.generation;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
private entry(entry: EntryRecord, retryAttempt?: number): void {
|
|
399
|
+
if (this.disposed || entry.id <= this.baselineEntry || this.entries.has(entry.id)) return;
|
|
400
|
+
this.entries.add(entry.id);
|
|
401
|
+
const message = entry.model?.[0];
|
|
402
|
+
const run = this.run;
|
|
403
|
+
if (!run) return;
|
|
404
|
+
if (message?.role === "assistant") {
|
|
405
|
+
const generation = this.generation();
|
|
406
|
+
if (!generation) return;
|
|
407
|
+
const usage = durableUsage(message);
|
|
408
|
+
this.end(generation, {
|
|
409
|
+
output: durableContent(entry.model, this.options.captureContent ?? true),
|
|
410
|
+
model: message.model,
|
|
411
|
+
...usage,
|
|
412
|
+
level: message.stopReason === "error" ? "ERROR" : message.stopReason === "aborted" ? "WARNING" : "DEFAULT",
|
|
413
|
+
metadata: {
|
|
414
|
+
...usage.metadata,
|
|
415
|
+
"pi.durable.entry_id": entry.id,
|
|
416
|
+
"pi.durable.task_id": entry.byTaskId,
|
|
417
|
+
"pi.durable.stop_reason": message.stopReason,
|
|
418
|
+
"pi.durable.provider": message.provider,
|
|
419
|
+
...(retryAttempt === undefined ? {} : { "pi.durable.retry_attempt": retryAttempt }),
|
|
420
|
+
},
|
|
421
|
+
});
|
|
422
|
+
if (run.generationKey) this.completedGenerations.add(run.generationKey);
|
|
423
|
+
run.generation = undefined;
|
|
424
|
+
} else if (message?.role === "toolResult") {
|
|
425
|
+
const key = entry.byTaskId === undefined ? `entry:${entry.id}` : `task:${entry.byTaskId}`;
|
|
426
|
+
this.finishTool(key, message.toolCallId, entry);
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
private finishTool(key: string, callId: string, entry?: EntryRecord): void {
|
|
431
|
+
const run = this.run;
|
|
432
|
+
if (!run || run.completedTools.has(key)) return;
|
|
433
|
+
run.completedTools.add(key);
|
|
434
|
+
const tool =
|
|
435
|
+
run.tools.get(key) ??
|
|
436
|
+
this.start("pi.durable.tool", "tool", run.span, {
|
|
437
|
+
metadata: { "pi.durable.start_not_observed": true, "pi.durable.tool_call_id": callId },
|
|
438
|
+
});
|
|
439
|
+
const message = entry?.model?.[0];
|
|
440
|
+
const usage = message ? durableUsage(message) : {};
|
|
441
|
+
this.end(tool, {
|
|
442
|
+
...(entry
|
|
443
|
+
? { output: durableContent(entry.model, this.options.captureContent ?? true, durableDiagnosticCount(entry)) }
|
|
444
|
+
: {}),
|
|
445
|
+
...usage,
|
|
446
|
+
level: message?.role === "toolResult" && message.isError ? "ERROR" : entry ? "DEFAULT" : "WARNING",
|
|
447
|
+
metadata: {
|
|
448
|
+
...usage.metadata,
|
|
449
|
+
"pi.durable.result_entry_id": entry?.id,
|
|
450
|
+
"pi.durable.result_missing": !entry,
|
|
451
|
+
},
|
|
452
|
+
});
|
|
453
|
+
run.tools.delete(key);
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
private task(task: DurableTask, live: DurableLive): void {
|
|
457
|
+
const run = this.run;
|
|
458
|
+
if (this.disposed || !run) return;
|
|
459
|
+
if (task.failed) this.update(run.span, { level: "ERROR", metadata: { "pi.durable.failed_task_id": task.id } });
|
|
460
|
+
if (task.kind === "pi.generation") {
|
|
461
|
+
if (task.phase === "request" || task.phase === "poll")
|
|
462
|
+
// Overflow compaction can repeat an attempt with a newer context cutoff; repeated polls keep that cutoff.
|
|
463
|
+
this.generation(`${task.id}:${task.attempt ?? 0}:${task.cutoff ?? "unknown"}`);
|
|
464
|
+
else if (task.phase === "retry") {
|
|
465
|
+
this.end(run.generation, {
|
|
466
|
+
level: "ERROR",
|
|
467
|
+
statusMessage: "Provider attempt will retry.",
|
|
468
|
+
metadata: { "pi.durable.retry_attempt": task.attempt },
|
|
469
|
+
});
|
|
470
|
+
if (run.generationKey) this.completedGenerations.add(run.generationKey);
|
|
471
|
+
run.generation = undefined;
|
|
472
|
+
}
|
|
473
|
+
return;
|
|
474
|
+
}
|
|
475
|
+
if (task.kind !== "pi.tool" || task.callId === undefined) return;
|
|
476
|
+
const key = `task:${task.id}`;
|
|
477
|
+
if (run.completedTools.has(key)) return;
|
|
478
|
+
if (task.arguments !== undefined && !run.tools.has(key)) {
|
|
479
|
+
const slot = live.tools.find((slot) => slot.taskId === task.id);
|
|
480
|
+
run.tools.set(
|
|
481
|
+
key,
|
|
482
|
+
this.start("pi.durable.tool", "tool", run.span, {
|
|
483
|
+
input:
|
|
484
|
+
this.options.captureContent === false
|
|
485
|
+
? "[content capture disabled]"
|
|
486
|
+
: sanitizeTraceValue(task.arguments, true),
|
|
487
|
+
metadata: {
|
|
488
|
+
"pi.durable.tool_call_id": task.callId,
|
|
489
|
+
"pi.durable.tool_name": slot?.name ?? task.name,
|
|
490
|
+
"pi.durable.task_id": task.id,
|
|
491
|
+
"pi.durable.observed_generation_index": run.round,
|
|
492
|
+
},
|
|
493
|
+
}),
|
|
494
|
+
);
|
|
495
|
+
}
|
|
496
|
+
if (task.terminal) this.finishTool(key, task.callId);
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
async commit(commit: DurableCommit, readEntry: (id: EntryId) => Promise<EntryRecord | undefined>): Promise<void> {
|
|
500
|
+
if (this.disposed) return;
|
|
501
|
+
// Resolve correlation before creating run children. One failed read cannot discard the rest of a publication.
|
|
502
|
+
for (const record of [...commit.submissions].sort((a, b) => a.id - b.id)) {
|
|
503
|
+
try {
|
|
504
|
+
await this.submission(record, readEntry);
|
|
505
|
+
} catch {
|
|
506
|
+
if (!this.disposed) this.report("observer delivery");
|
|
507
|
+
}
|
|
508
|
+
if (this.disposed) return;
|
|
509
|
+
}
|
|
510
|
+
const next = commit.live ?? this.live;
|
|
511
|
+
if (!this.run && next.run) this.beginRun(next.run);
|
|
512
|
+
// Starts precede completions, but a terminal task cannot hide its result entry in the same publication.
|
|
513
|
+
for (const task of commit.tasks) if (!task.terminal && task.phase !== "retry") this.task(task, next);
|
|
514
|
+
for (const entry of [...commit.entries].sort((a, b) => a.id - b.id)) {
|
|
515
|
+
const retry = commit.tasks.find((task) => task.id === entry.byTaskId && task.phase === "retry");
|
|
516
|
+
this.entry(entry, retry?.attempt);
|
|
517
|
+
}
|
|
518
|
+
for (const task of commit.tasks) if (task.terminal || task.phase === "retry") this.task(task, next);
|
|
519
|
+
if (this.disposed) return;
|
|
520
|
+
if (commit.live) {
|
|
521
|
+
if (this.run && !this.sameRun(next.run)) this.closeRun();
|
|
522
|
+
if (next.run) this.beginRun(next.run);
|
|
523
|
+
this.live = next;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
dispose(reason = "Tracing observer disposed; execution outcome is unknown."): void {
|
|
528
|
+
if (this.disposed) return;
|
|
529
|
+
this.disposed = true;
|
|
530
|
+
this.closeRun(reason);
|
|
531
|
+
for (const state of this.submissions.values()) if (!state.terminal) this.interrupted(state.span, reason);
|
|
532
|
+
this.end(this.root, { metadata: { "pi.durable.resync_count": this.gaps } });
|
|
533
|
+
this.submissions.clear();
|
|
534
|
+
this.pendingSubmissions.clear();
|
|
535
|
+
this.entries.clear();
|
|
536
|
+
this.completedRuns.clear();
|
|
537
|
+
this.completedGenerations.clear();
|
|
538
|
+
}
|
|
539
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type CommitPublication,
|
|
3
|
+
type ConversationId,
|
|
4
|
+
type ConversationView,
|
|
5
|
+
type EntryRecord,
|
|
6
|
+
InboxDoc,
|
|
7
|
+
type InboxState,
|
|
8
|
+
type JsonObject,
|
|
9
|
+
LiveDoc,
|
|
10
|
+
type LiveState,
|
|
11
|
+
type SubmissionId,
|
|
12
|
+
type SubmissionRecord,
|
|
13
|
+
type TaskId,
|
|
14
|
+
type ToolTaskCheckpoint,
|
|
15
|
+
type ToolTaskInput,
|
|
16
|
+
} from "@earendil-works/pi-durable";
|
|
17
|
+
|
|
18
|
+
export interface DurableLive {
|
|
19
|
+
run?: { taskId: TaskId; inputs: readonly SubmissionId[]; afterRun: boolean };
|
|
20
|
+
tools: readonly { taskId?: TaskId; callId: string; name: string; status: "pending" | "running" | "done" }[];
|
|
21
|
+
}
|
|
22
|
+
export interface DurableSnapshot {
|
|
23
|
+
entries: readonly EntryRecord[];
|
|
24
|
+
live: DurableLive;
|
|
25
|
+
inbox: readonly SubmissionId[];
|
|
26
|
+
}
|
|
27
|
+
export interface DurableTask {
|
|
28
|
+
id: TaskId;
|
|
29
|
+
kind: string;
|
|
30
|
+
failed: boolean;
|
|
31
|
+
terminal: boolean;
|
|
32
|
+
phase?: string;
|
|
33
|
+
attempt?: number;
|
|
34
|
+
cutoff?: number;
|
|
35
|
+
callId?: string;
|
|
36
|
+
name?: string;
|
|
37
|
+
arguments?: JsonObject;
|
|
38
|
+
}
|
|
39
|
+
export interface DurableCommit {
|
|
40
|
+
entries: readonly EntryRecord[];
|
|
41
|
+
submissions: readonly SubmissionRecord[];
|
|
42
|
+
tasks: readonly DurableTask[];
|
|
43
|
+
live?: DurableLive;
|
|
44
|
+
inbox?: readonly SubmissionId[];
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function liveProjection(live: LiveState = {}): DurableLive {
|
|
48
|
+
return {
|
|
49
|
+
...(live.run
|
|
50
|
+
? { run: { taskId: live.run.taskId, inputs: live.run.inputs, afterRun: live.run.afterRun !== undefined } }
|
|
51
|
+
: {}),
|
|
52
|
+
tools: [...(live.tools ?? []), ...(live.nestedTools ?? [])].map(({ taskId, callId, name, status }) => ({
|
|
53
|
+
taskId,
|
|
54
|
+
callId,
|
|
55
|
+
name,
|
|
56
|
+
status,
|
|
57
|
+
})),
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The view is a baseline only: never retain partial messages, running output, provider state or diagnostics. */
|
|
62
|
+
export function durableSnapshot(view: ConversationView): DurableSnapshot {
|
|
63
|
+
return {
|
|
64
|
+
entries: view.entries,
|
|
65
|
+
live: liveProjection(view.docs[LiveDoc.definition.kind] as LiveState | undefined),
|
|
66
|
+
inbox: ((view.docs[InboxDoc.definition.kind] as InboxState | undefined)?.items ?? []).map(({ id }) => id),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Record only the public committed fields tracing needs; never inspect request options or raw failure bodies. */
|
|
71
|
+
export function durableCommit(publication: CommitPublication, conversationId: ConversationId): DurableCommit {
|
|
72
|
+
const entries: EntryRecord[] = [];
|
|
73
|
+
const submissions: SubmissionRecord[] = [];
|
|
74
|
+
const tasks: DurableTask[] = [];
|
|
75
|
+
let live: DurableLive | undefined;
|
|
76
|
+
let inbox: readonly SubmissionId[] | undefined;
|
|
77
|
+
for (const change of publication.changes) {
|
|
78
|
+
if (change.type === "document") {
|
|
79
|
+
if (change.conversationId !== conversationId || change.record.key !== undefined) continue;
|
|
80
|
+
if (change.record.kind === LiveDoc.definition.kind)
|
|
81
|
+
live = liveProjection(change.kind === "change" ? (change.value as LiveState) : undefined);
|
|
82
|
+
else if (change.record.kind === InboxDoc.definition.kind)
|
|
83
|
+
inbox = change.kind === "change" ? (change.value as InboxState).items.map(({ id }) => id) : [];
|
|
84
|
+
} else if (
|
|
85
|
+
change.type !== "document.copy" &&
|
|
86
|
+
change.type !== "conversation" &&
|
|
87
|
+
change.value.conversationId === conversationId
|
|
88
|
+
) {
|
|
89
|
+
if (change.type === "entry") entries.push(change.value);
|
|
90
|
+
else if (change.type === "submission") submissions.push(change.value);
|
|
91
|
+
else if (change.type === "task") {
|
|
92
|
+
const task = change.value;
|
|
93
|
+
const terminal = task.state.status === "terminal" || task.state.status === "completing";
|
|
94
|
+
const failed =
|
|
95
|
+
terminal && (task.state.outcome?.status === "faulted" || task.state.outcome?.status === "orphaned");
|
|
96
|
+
if (task.kind !== "pi.tool" && task.kind !== "pi.generation") {
|
|
97
|
+
if (failed) tasks.push({ id: task.id, kind: "other", terminal, failed });
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
const checkpoint = task.state.checkpoint;
|
|
101
|
+
const record: DurableTask = {
|
|
102
|
+
id: task.id,
|
|
103
|
+
kind: task.kind,
|
|
104
|
+
terminal,
|
|
105
|
+
failed,
|
|
106
|
+
...(checkpoint && typeof checkpoint === "object" && "phase" in checkpoint
|
|
107
|
+
? { phase: String(checkpoint.phase) }
|
|
108
|
+
: {}),
|
|
109
|
+
};
|
|
110
|
+
if (task.kind === "pi.tool") {
|
|
111
|
+
const input = task.input as ToolTaskInput;
|
|
112
|
+
record.callId = input.kind === "nested" ? input.call.id : input.callId;
|
|
113
|
+
if (input.kind === "nested") record.name = input.call.name;
|
|
114
|
+
const intent = checkpoint as ToolTaskCheckpoint | undefined;
|
|
115
|
+
if (intent?.phase === "execute") record.arguments = intent.arguments;
|
|
116
|
+
} else if (checkpoint && typeof checkpoint === "object" && "attempt" in checkpoint) {
|
|
117
|
+
const attempt = checkpoint.attempt;
|
|
118
|
+
if (typeof attempt === "number" && Number.isFinite(attempt)) record.attempt = attempt;
|
|
119
|
+
if ("cutoff" in checkpoint && typeof checkpoint.cutoff === "number") record.cutoff = checkpoint.cutoff;
|
|
120
|
+
}
|
|
121
|
+
tasks.push(record);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return { entries, submissions, tasks, ...(live ? { live } : {}), ...(inbox ? { inbox } : {}) };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function advanceDurableSnapshot(snapshot: DurableSnapshot, commit: DurableCommit): DurableSnapshot {
|
|
129
|
+
return {
|
|
130
|
+
// This is only the bounded overflow baseline, not a second transcript or a historical replay buffer.
|
|
131
|
+
entries: [...snapshot.entries, ...commit.entries].slice(-100),
|
|
132
|
+
live: commit.live ?? snapshot.live,
|
|
133
|
+
inbox: commit.inbox ?? snapshot.inbox,
|
|
134
|
+
};
|
|
135
|
+
}
|