@shipfox/api-logs 9.3.0 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +25 -0
- package/dist/core/append-logs.d.ts.map +1 -1
- package/dist/core/append-logs.js +100 -12
- package/dist/core/append-logs.js.map +1 -1
- package/dist/core/close-stream.d.ts +6 -5
- package/dist/core/close-stream.d.ts.map +1 -1
- package/dist/core/close-stream.js +25 -7
- package/dist/core/close-stream.js.map +1 -1
- package/dist/core/entities/attempt-stream.d.ts +5 -0
- package/dist/core/entities/attempt-stream.d.ts.map +1 -1
- package/dist/core/entities/attempt-stream.js +1 -1
- package/dist/core/entities/attempt-stream.js.map +1 -1
- package/dist/core/session/claude/rows.d.ts +4 -2
- package/dist/core/session/claude/rows.d.ts.map +1 -1
- package/dist/core/session/claude/rows.js +44 -9
- package/dist/core/session/claude/rows.js.map +1 -1
- package/dist/core/session/claude-parser.d.ts +8 -1
- package/dist/core/session/claude-parser.d.ts.map +1 -1
- package/dist/core/session/claude-parser.js +30 -4
- package/dist/core/session/claude-parser.js.map +1 -1
- package/dist/core/session/parse-session.d.ts +6 -1
- package/dist/core/session/parse-session.d.ts.map +1 -1
- package/dist/core/session/parse-session.js +2 -2
- package/dist/core/session/parse-session.js.map +1 -1
- package/dist/db/db.d.ts +94 -0
- package/dist/db/db.d.ts.map +1 -1
- package/dist/db/schema/attempt-streams.d.ts +94 -0
- package/dist/db/schema/attempt-streams.d.ts.map +1 -1
- package/dist/db/schema/attempt-streams.js +9 -1
- package/dist/db/schema/attempt-streams.js.map +1 -1
- package/dist/db/streams.d.ts +17 -6
- package/dist/db/streams.d.ts.map +1 -1
- package/dist/db/streams.js +21 -6
- package/dist/db/streams.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/drizzle/0001_strong_major_mapleleaf.sql +4 -0
- package/drizzle/meta/0001_snapshot.json +555 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +5 -5
- package/src/core/append-logs.test.ts +144 -4
- package/src/core/append-logs.ts +158 -6
- package/src/core/close-stream.ts +26 -7
- package/src/core/entities/attempt-stream.ts +6 -0
- package/src/core/finalize-attempt-stream.test.ts +57 -0
- package/src/core/session/claude/rows.ts +57 -7
- package/src/core/session/claude-parser.test.ts +118 -1
- package/src/core/session/claude-parser.ts +55 -4
- package/src/core/session/parse-session.ts +10 -1
- package/src/db/schema/attempt-streams.ts +10 -0
- package/src/db/streams.ts +43 -7
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -213,10 +213,9 @@ describe('appendLogs', () => {
|
|
|
213
213
|
await appendLogs({...ctx, attempt: 1, offset: 0, body}, workflows);
|
|
214
214
|
|
|
215
215
|
const stream = await findStream({...ctx, attempt: 1});
|
|
216
|
-
const rows = recordsFromChunks(await listChunks(stream?.id as string)).
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
});
|
|
216
|
+
const rows = recordsFromChunks(await listChunks(stream?.id as string)).flatMap((record) =>
|
|
217
|
+
record.type === 'agent_session' ? [record.row] : [],
|
|
218
|
+
);
|
|
220
219
|
expect(rows).toEqual([
|
|
221
220
|
{
|
|
222
221
|
kind: 'tool-call',
|
|
@@ -228,6 +227,147 @@ describe('appendLogs', () => {
|
|
|
228
227
|
]);
|
|
229
228
|
});
|
|
230
229
|
|
|
230
|
+
it('reports Claude re-prompts as turns and labels the re-prompt as platform input', async () => {
|
|
231
|
+
const ctx = newCtx();
|
|
232
|
+
await jobAccountingFactory.create({
|
|
233
|
+
jobId: ctx.jobId,
|
|
234
|
+
workspaceId: ctx.workspaceId,
|
|
235
|
+
startedAt: new Date(Date.now() - 60 * 60_000),
|
|
236
|
+
});
|
|
237
|
+
const workflows = createFakeInterModuleClients({
|
|
238
|
+
workflows: defineInterModulePresentation(workflowsInterModuleContract, {
|
|
239
|
+
startRunFromTrigger: vi.fn(),
|
|
240
|
+
deliverEventToJobListener: vi.fn(),
|
|
241
|
+
getStepLogContext: () => ({harness: 'claude' as const}),
|
|
242
|
+
getLeasedAgentToolContext: vi.fn(),
|
|
243
|
+
}),
|
|
244
|
+
}).workflows;
|
|
245
|
+
const init = sessionLine(
|
|
246
|
+
JSON.stringify({type: 'system', subtype: 'init', session_id: 'session-1'}),
|
|
247
|
+
);
|
|
248
|
+
const result = (text: string) =>
|
|
249
|
+
sessionLine(JSON.stringify({type: 'result', subtype: 'success', result: text}));
|
|
250
|
+
const reprompt = sessionLine(
|
|
251
|
+
JSON.stringify({
|
|
252
|
+
type: 'user',
|
|
253
|
+
message: {
|
|
254
|
+
role: 'user',
|
|
255
|
+
content:
|
|
256
|
+
'The previous turn ended without setting required workflow outputs: answer. ' +
|
|
257
|
+
'Call set_output for each missing key, then provide your final response.',
|
|
258
|
+
},
|
|
259
|
+
}),
|
|
260
|
+
);
|
|
261
|
+
const body = ndjsonBody(
|
|
262
|
+
init,
|
|
263
|
+
result('first response'),
|
|
264
|
+
reprompt,
|
|
265
|
+
init,
|
|
266
|
+
result('second response'),
|
|
267
|
+
reprompt,
|
|
268
|
+
init,
|
|
269
|
+
result('final response'),
|
|
270
|
+
endLine(0),
|
|
271
|
+
);
|
|
272
|
+
|
|
273
|
+
await appendLogs({...ctx, attempt: 1, offset: 0, body}, workflows);
|
|
274
|
+
|
|
275
|
+
const stream = await findStream({...ctx, attempt: 1});
|
|
276
|
+
const rows = recordsFromChunks(await listChunks(stream?.id as string)).flatMap((record) =>
|
|
277
|
+
record.type === 'agent_session' ? [record.row] : [],
|
|
278
|
+
);
|
|
279
|
+
const lifecycleRows = rows.filter(
|
|
280
|
+
(row): row is Extract<NonNullable<(typeof rows)[number]>, {kind: 'lifecycle'}> =>
|
|
281
|
+
row?.kind === 'lifecycle',
|
|
282
|
+
);
|
|
283
|
+
const messageRows = rows.filter(
|
|
284
|
+
(row): row is Extract<NonNullable<(typeof rows)[number]>, {kind: 'message'}> =>
|
|
285
|
+
row?.kind === 'message',
|
|
286
|
+
);
|
|
287
|
+
|
|
288
|
+
expect(lifecycleRows.map((row) => row.label)).toEqual([
|
|
289
|
+
'Session started',
|
|
290
|
+
'Turn 1 completed',
|
|
291
|
+
'Turn 2 started',
|
|
292
|
+
'Turn 2 completed',
|
|
293
|
+
'Turn 3 started',
|
|
294
|
+
'Session completed',
|
|
295
|
+
]);
|
|
296
|
+
expect(
|
|
297
|
+
lifecycleRows.filter((row) => row.label.startsWith('Turn ')).map((row) => row.meta),
|
|
298
|
+
).toEqual([
|
|
299
|
+
[{label: 'turn', value: '1'}],
|
|
300
|
+
[{label: 'turn', value: '2'}],
|
|
301
|
+
[{label: 'turn', value: '2'}],
|
|
302
|
+
[{label: 'turn', value: '3'}],
|
|
303
|
+
]);
|
|
304
|
+
expect(messageRows.map((row) => ({role: row.role, label: row.label}))).toEqual([
|
|
305
|
+
{role: 'platform', label: 'platform'},
|
|
306
|
+
{role: 'platform', label: 'platform'},
|
|
307
|
+
]);
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
it('continues Claude turn context across append requests', async () => {
|
|
311
|
+
const ctx = newCtx();
|
|
312
|
+
await jobAccountingFactory.create({
|
|
313
|
+
jobId: ctx.jobId,
|
|
314
|
+
workspaceId: ctx.workspaceId,
|
|
315
|
+
startedAt: new Date(Date.now() - 60 * 60_000),
|
|
316
|
+
});
|
|
317
|
+
const workflows = createFakeInterModuleClients({
|
|
318
|
+
workflows: defineInterModulePresentation(workflowsInterModuleContract, {
|
|
319
|
+
startRunFromTrigger: vi.fn(),
|
|
320
|
+
deliverEventToJobListener: vi.fn(),
|
|
321
|
+
getStepLogContext: () => ({harness: 'claude' as const}),
|
|
322
|
+
getLeasedAgentToolContext: vi.fn(),
|
|
323
|
+
}),
|
|
324
|
+
}).workflows;
|
|
325
|
+
const init = sessionLine(
|
|
326
|
+
JSON.stringify({type: 'system', subtype: 'init', session_id: 'session-1'}),
|
|
327
|
+
);
|
|
328
|
+
const result = sessionLine(
|
|
329
|
+
JSON.stringify({type: 'result', subtype: 'success', result: 'response'}),
|
|
330
|
+
);
|
|
331
|
+
const reprompt = sessionLine(
|
|
332
|
+
JSON.stringify({
|
|
333
|
+
type: 'user',
|
|
334
|
+
message: {
|
|
335
|
+
role: 'user',
|
|
336
|
+
content:
|
|
337
|
+
'The previous turn ended without setting required workflow outputs: answer. ' +
|
|
338
|
+
'Call set_output for each missing key, then provide your final response.',
|
|
339
|
+
},
|
|
340
|
+
}),
|
|
341
|
+
);
|
|
342
|
+
const first = ndjsonBody(init, result);
|
|
343
|
+
const second = ndjsonBody(reprompt, init, result, endLine(0));
|
|
344
|
+
|
|
345
|
+
await appendLogs({...ctx, attempt: 1, offset: 0, body: first}, workflows);
|
|
346
|
+
await appendLogs({...ctx, attempt: 1, offset: first.length, body: second}, workflows);
|
|
347
|
+
|
|
348
|
+
const stream = await findStream({...ctx, attempt: 1});
|
|
349
|
+
const rows = recordsFromChunks(await listChunks(stream?.id as string)).flatMap((record) =>
|
|
350
|
+
record.type === 'agent_session' ? [record.row] : [],
|
|
351
|
+
);
|
|
352
|
+
const lifecycleRows = rows.filter(
|
|
353
|
+
(row): row is Extract<NonNullable<(typeof rows)[number]>, {kind: 'lifecycle'}> =>
|
|
354
|
+
row?.kind === 'lifecycle',
|
|
355
|
+
);
|
|
356
|
+
|
|
357
|
+
expect(lifecycleRows.map((row) => row.label)).toEqual([
|
|
358
|
+
'Session started',
|
|
359
|
+
'Turn 1 completed',
|
|
360
|
+
'Turn 2 started',
|
|
361
|
+
'Session completed',
|
|
362
|
+
]);
|
|
363
|
+
expect(stream).toMatchObject({
|
|
364
|
+
claudeHasInit: true,
|
|
365
|
+
claudeSessionId: 'session-1',
|
|
366
|
+
claudeTurn: 2,
|
|
367
|
+
claudePendingResult: null,
|
|
368
|
+
});
|
|
369
|
+
});
|
|
370
|
+
|
|
231
371
|
it('does not duplicate parsed rows on a retried append', async () => {
|
|
232
372
|
const ctx = newCtx();
|
|
233
373
|
await allowLargeLogBudget(ctx);
|
package/src/core/append-logs.ts
CHANGED
|
@@ -4,6 +4,8 @@ import {
|
|
|
4
4
|
parseLogRecordLine,
|
|
5
5
|
parseRawLogRecordLine,
|
|
6
6
|
type RawLogRecord,
|
|
7
|
+
type SessionViewLifecycleRow,
|
|
8
|
+
type SessionViewRow,
|
|
7
9
|
} from '@shipfox/api-logs-dto';
|
|
8
10
|
import type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';
|
|
9
11
|
import {logger} from '@shipfox/node-opentelemetry';
|
|
@@ -16,6 +18,7 @@ import {
|
|
|
16
18
|
casExtendCommittedLength,
|
|
17
19
|
getAttemptStream,
|
|
18
20
|
getOrCreateAttemptStreamWithStatus,
|
|
21
|
+
setClaudeParseContext,
|
|
19
22
|
setDeclaredTotalBytes,
|
|
20
23
|
} from '#db/streams.js';
|
|
21
24
|
import {
|
|
@@ -27,6 +30,12 @@ import {
|
|
|
27
30
|
import {allowedBudget} from './budget.js';
|
|
28
31
|
import {closeStream, controlTombstone} from './close-stream.js';
|
|
29
32
|
import {MalformedLogChunkError, OffsetGapError} from './errors.js';
|
|
33
|
+
import {
|
|
34
|
+
type ClaudeParseContext,
|
|
35
|
+
claudeInitSessionId,
|
|
36
|
+
createClaudeParseContext,
|
|
37
|
+
} from './session/claude-parser.js';
|
|
38
|
+
import type {SessionParseContext} from './session/parse-session.js';
|
|
30
39
|
import {parseSessionRecord} from './session/parse-session.js';
|
|
31
40
|
import type {AgentSessionRecord} from './session/session-record.js';
|
|
32
41
|
|
|
@@ -166,6 +175,13 @@ interface StoreChunkResult extends AppendLogsResult {
|
|
|
166
175
|
recordCounts: Partial<Record<LogRecord['type'], number>>;
|
|
167
176
|
}
|
|
168
177
|
|
|
178
|
+
interface StoredBody {
|
|
179
|
+
body: Buffer;
|
|
180
|
+
recordCounts: Partial<Record<LogRecord['type'], number>>;
|
|
181
|
+
claudeParseContext: ClaudeParseContext | undefined;
|
|
182
|
+
claudePendingResult: SessionViewLifecycleRow | null | undefined;
|
|
183
|
+
}
|
|
184
|
+
|
|
169
185
|
/**
|
|
170
186
|
* Accrues the stored bytes, persists the chunk, and trips the per-job cap when
|
|
171
187
|
* this append crosses the budget. Runs only after the offset-CAS extended
|
|
@@ -228,16 +244,62 @@ async function storeChunk(
|
|
|
228
244
|
function buildStoredBody(
|
|
229
245
|
records: readonly RawLogRecord[],
|
|
230
246
|
harness: Harness,
|
|
231
|
-
|
|
247
|
+
initialClaudeParseContext?: ClaudeParseContext,
|
|
248
|
+
initialClaudePendingResult?: SessionViewLifecycleRow | null,
|
|
249
|
+
isStreamFinal = false,
|
|
250
|
+
): StoredBody {
|
|
232
251
|
const storedRecords: LogRecord[] = [];
|
|
233
|
-
|
|
252
|
+
const parseContext: SessionParseContext | undefined =
|
|
253
|
+
harness === 'claude'
|
|
254
|
+
? {
|
|
255
|
+
claude: initialClaudeParseContext ?? createClaudeParseContext(),
|
|
256
|
+
isFinalResult: true,
|
|
257
|
+
}
|
|
258
|
+
: undefined;
|
|
259
|
+
const latestClaudeInitIndicesBySessionId =
|
|
260
|
+
harness === 'claude' ? latestClaudeInitIndices(records) : undefined;
|
|
261
|
+
let pendingResult = initialClaudePendingResult ?? null;
|
|
262
|
+
|
|
263
|
+
if (parseContext?.claude !== undefined && pendingResult !== null) {
|
|
264
|
+
const firstInit = firstClaudeInit(records);
|
|
265
|
+
if (isStreamFinal || firstInit.hasInit) {
|
|
266
|
+
storedRecords.push(
|
|
267
|
+
storedAgentSessionRow(
|
|
268
|
+
finalizePendingResult(
|
|
269
|
+
pendingResult,
|
|
270
|
+
firstInit.sessionId !== undefined &&
|
|
271
|
+
firstInit.sessionId === parseContext.claude.sessionId,
|
|
272
|
+
parseContext.claude.turn,
|
|
273
|
+
),
|
|
274
|
+
),
|
|
275
|
+
);
|
|
276
|
+
pendingResult = null;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
for (const [index, record] of records.entries()) {
|
|
234
281
|
if (record.type !== 'agent_session') {
|
|
235
282
|
storedRecords.push(record);
|
|
236
283
|
continue;
|
|
237
284
|
}
|
|
238
285
|
|
|
239
|
-
|
|
240
|
-
|
|
286
|
+
if (parseContext?.claude !== undefined && latestClaudeInitIndicesBySessionId !== undefined) {
|
|
287
|
+
parseContext.isFinalResult = !hasFutureClaudeInit(
|
|
288
|
+
latestClaudeInitIndicesBySessionId,
|
|
289
|
+
index,
|
|
290
|
+
parseContext.claude.sessionId,
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
for (const row of parseSessionRecord(agentSessionRecord(record), harness, parseContext)) {
|
|
295
|
+
if (parseContext?.claude !== undefined && !isStreamFinal && isClaudeFinalResultRow(row)) {
|
|
296
|
+
if (pendingResult !== null) {
|
|
297
|
+
storedRecords.push(storedAgentSessionRow(pendingResult));
|
|
298
|
+
}
|
|
299
|
+
pendingResult = row;
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
storedRecords.push(storedAgentSessionRow(row));
|
|
241
303
|
}
|
|
242
304
|
}
|
|
243
305
|
|
|
@@ -247,7 +309,71 @@ function buildStoredBody(
|
|
|
247
309
|
recordCounts[record.type] = (recordCounts[record.type] ?? 0) + 1;
|
|
248
310
|
}
|
|
249
311
|
|
|
250
|
-
return {
|
|
312
|
+
return {
|
|
313
|
+
body,
|
|
314
|
+
recordCounts,
|
|
315
|
+
claudeParseContext: parseContext?.claude,
|
|
316
|
+
claudePendingResult: parseContext?.claude === undefined ? undefined : pendingResult,
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
function storedAgentSessionRow(row: SessionViewRow): LogRecord {
|
|
321
|
+
return {v: 1, ts: row.timestamp, type: 'agent_session', row};
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function isClaudeFinalResultRow(row: SessionViewRow): row is SessionViewLifecycleRow {
|
|
325
|
+
return row.kind === 'lifecycle' && row.label === 'Session completed';
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
function finalizePendingResult(
|
|
329
|
+
row: SessionViewLifecycleRow,
|
|
330
|
+
sameSession: boolean,
|
|
331
|
+
turn: number,
|
|
332
|
+
): SessionViewLifecycleRow {
|
|
333
|
+
if (!sameSession) return row;
|
|
334
|
+
|
|
335
|
+
return {
|
|
336
|
+
...row,
|
|
337
|
+
label: `Turn ${turn} completed`,
|
|
338
|
+
meta: [{label: 'turn', value: String(turn)}, ...row.meta],
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function firstClaudeInit(records: readonly RawLogRecord[]): {
|
|
343
|
+
hasInit: boolean;
|
|
344
|
+
sessionId: string | undefined;
|
|
345
|
+
} {
|
|
346
|
+
for (const record of records) {
|
|
347
|
+
if (record.type !== 'agent_session') continue;
|
|
348
|
+
const sessionRecord = agentSessionRecord(record);
|
|
349
|
+
const sessionId = claudeInitSessionId(sessionRecord);
|
|
350
|
+
if (sessionId !== undefined) return {hasInit: true, sessionId};
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
return {hasInit: false, sessionId: undefined};
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function latestClaudeInitIndices(records: readonly RawLogRecord[]): Map<string, number> {
|
|
357
|
+
const latestIndices = new Map<string, number>();
|
|
358
|
+
|
|
359
|
+
for (const [index, record] of records.entries()) {
|
|
360
|
+
if (record.type !== 'agent_session') continue;
|
|
361
|
+
|
|
362
|
+
const sessionId = claudeInitSessionId(agentSessionRecord(record));
|
|
363
|
+
if (sessionId !== undefined) latestIndices.set(sessionId, index);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
return latestIndices;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
function hasFutureClaudeInit(
|
|
370
|
+
latestClaudeInitIndicesBySessionId: ReadonlyMap<string, number>,
|
|
371
|
+
currentIndex: number,
|
|
372
|
+
sessionId: string | null,
|
|
373
|
+
): boolean {
|
|
374
|
+
if (sessionId === null) return false;
|
|
375
|
+
|
|
376
|
+
return (latestClaudeInitIndicesBySessionId.get(sessionId) ?? -1) > currentIndex;
|
|
251
377
|
}
|
|
252
378
|
|
|
253
379
|
function agentSessionRecord(
|
|
@@ -287,7 +413,6 @@ export async function appendLogs(
|
|
|
287
413
|
const sessionHarness = parsed.hasAgentSessionRecord
|
|
288
414
|
? await getSessionHarness(workflows, params.stepId)
|
|
289
415
|
: DEFAULT_HARNESS;
|
|
290
|
-
const stored = buildStoredBody(parsed.records, sessionHarness);
|
|
291
416
|
const commitByteLen = params.body.length;
|
|
292
417
|
const metrics = {
|
|
293
418
|
recordCounts: {} as Partial<Record<LogRecordMetricKind, number>>,
|
|
@@ -325,6 +450,24 @@ export async function appendLogs(
|
|
|
325
450
|
return {committedLength: cas.committedLength, capped: await isJobCapped(tx, params.jobId)};
|
|
326
451
|
}
|
|
327
452
|
|
|
453
|
+
const parseHarness =
|
|
454
|
+
sessionHarness === 'claude' || stream.claudePendingResult !== null
|
|
455
|
+
? 'claude'
|
|
456
|
+
: sessionHarness;
|
|
457
|
+
const stored = buildStoredBody(
|
|
458
|
+
parsed.records,
|
|
459
|
+
parseHarness,
|
|
460
|
+
parseHarness === 'claude'
|
|
461
|
+
? {
|
|
462
|
+
hasInit: stream.claudeHasInit,
|
|
463
|
+
sessionId: stream.claudeSessionId,
|
|
464
|
+
turn: stream.claudeTurn,
|
|
465
|
+
}
|
|
466
|
+
: undefined,
|
|
467
|
+
parseHarness === 'claude' ? stream.claudePendingResult : undefined,
|
|
468
|
+
declaredTotalBytes !== undefined,
|
|
469
|
+
);
|
|
470
|
+
|
|
328
471
|
const {
|
|
329
472
|
recordCounts,
|
|
330
473
|
stored: chunkStored,
|
|
@@ -338,6 +481,15 @@ export async function appendLogs(
|
|
|
338
481
|
});
|
|
339
482
|
if (chunkStored) {
|
|
340
483
|
addRecordCounts(metrics.recordCounts, stored.recordCounts);
|
|
484
|
+
if (stored.claudeParseContext !== undefined) {
|
|
485
|
+
await setClaudeParseContext(tx, {
|
|
486
|
+
streamId: stream.id,
|
|
487
|
+
hasInit: stored.claudeParseContext.hasInit,
|
|
488
|
+
sessionId: stored.claudeParseContext.sessionId,
|
|
489
|
+
turn: stored.claudeParseContext.turn,
|
|
490
|
+
pendingResult: stored.claudePendingResult ?? null,
|
|
491
|
+
});
|
|
492
|
+
}
|
|
341
493
|
}
|
|
342
494
|
addRecordCounts(metrics.recordCounts, recordCounts);
|
|
343
495
|
|
package/src/core/close-stream.ts
CHANGED
|
@@ -26,12 +26,13 @@ export interface CloseStreamParams {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
/**
|
|
29
|
-
* The guarded UPDATE (`WHERE state='open'`) in `markStreamClosed`
|
|
30
|
-
* the idempotency gate: a stream already closed by the other path (append-time
|
|
29
|
+
* The row lock plus guarded UPDATE (`WHERE state='open'`) in `markStreamClosed` provide
|
|
30
|
+
* the lock and idempotency gate: a stream already closed by the other path (append-time
|
|
31
31
|
* declared close vs the job-terminated timeout sweep) returns null, so no duplicate
|
|
32
|
-
* `LOG_STREAM_CLOSED` is written and no second tombstone lands. A
|
|
33
|
-
*
|
|
34
|
-
* if any, was injected earlier at
|
|
32
|
+
* `LOG_STREAM_CLOSED` is written and no second tombstone lands. A pending Claude result
|
|
33
|
+
* is materialized before the close event, then a timeout close sets `truncated` and injects
|
|
34
|
+
* a `runner_lost` tombstone in-band (a `capped` tombstone, if any, was injected earlier at
|
|
35
|
+
* the append that tripped the cap).
|
|
35
36
|
*
|
|
36
37
|
* The event drives compaction; it is written in the same transaction as the flip.
|
|
37
38
|
*/
|
|
@@ -39,12 +40,30 @@ export async function closeStream(
|
|
|
39
40
|
tx: Transaction,
|
|
40
41
|
params: CloseStreamParams,
|
|
41
42
|
): Promise<AttemptStream | null> {
|
|
42
|
-
const
|
|
43
|
+
const closedResult = await markStreamClosed(tx, {
|
|
43
44
|
streamId: params.streamId,
|
|
44
45
|
reason: params.reason,
|
|
45
46
|
markTruncated: params.reason === 'timeout',
|
|
46
47
|
});
|
|
47
|
-
if (!
|
|
48
|
+
if (!closedResult) return null;
|
|
49
|
+
|
|
50
|
+
const {stream: closed, pendingClaudeResult} = closedResult;
|
|
51
|
+
if (pendingClaudeResult !== null) {
|
|
52
|
+
const record: LogRecord = {
|
|
53
|
+
v: 1,
|
|
54
|
+
ts: pendingClaudeResult.timestamp,
|
|
55
|
+
type: 'agent_session',
|
|
56
|
+
row: pendingClaudeResult,
|
|
57
|
+
};
|
|
58
|
+
const data = Buffer.from(`${JSON.stringify(record)}\n`, 'utf8');
|
|
59
|
+
await insertChunk(tx, {
|
|
60
|
+
streamId: closed.id,
|
|
61
|
+
streamOffset: closed.committedLength,
|
|
62
|
+
byteLen: data.length,
|
|
63
|
+
data,
|
|
64
|
+
origin: 'control',
|
|
65
|
+
});
|
|
66
|
+
}
|
|
48
67
|
|
|
49
68
|
if (params.reason === 'timeout') {
|
|
50
69
|
const tombstone = controlTombstone('runner_lost');
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type {SessionViewLifecycleRow} from '@shipfox/api-logs-dto';
|
|
2
|
+
|
|
1
3
|
/** Lifecycle state of an attempt's log stream. */
|
|
2
4
|
export type StreamState = 'open' | 'closed';
|
|
3
5
|
|
|
@@ -29,6 +31,10 @@ export interface AttemptStream {
|
|
|
29
31
|
state: StreamState;
|
|
30
32
|
closeReason: StreamCloseReason | null;
|
|
31
33
|
declaredTotalBytes: number | null;
|
|
34
|
+
claudeHasInit: boolean;
|
|
35
|
+
claudeSessionId: string | null;
|
|
36
|
+
claudeTurn: number;
|
|
37
|
+
claudePendingResult: SessionViewLifecycleRow | null;
|
|
32
38
|
truncated: boolean;
|
|
33
39
|
objectKey: string | null;
|
|
34
40
|
createdAt: Date;
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import {parseLogRecordLine} from '@shipfox/api-logs-dto';
|
|
2
|
+
import {workflowsInterModuleContract} from '@shipfox/api-workflows-dto/inter-module';
|
|
3
|
+
import {defineInterModulePresentation} from '@shipfox/inter-module';
|
|
4
|
+
import {createFakeInterModuleClients} from '@shipfox/node-module/inter-module/testing';
|
|
5
|
+
import {appendLogs} from '#core/append-logs.js';
|
|
2
6
|
import {db} from '#db/db.js';
|
|
3
7
|
import {getOrCreateAttemptStream} from '#db/streams.js';
|
|
8
|
+
import {jobAccountingFactory} from '#test/factories/job-accounting.js';
|
|
9
|
+
import {ndjsonBody, sessionLine} from '#test/fixtures/ndjson.js';
|
|
4
10
|
import {listChunks, listStreamClosedEvents} from '#test/queries.js';
|
|
5
11
|
import {
|
|
6
12
|
createFinalizeAttemptLogStream,
|
|
@@ -74,6 +80,57 @@ describe('finalizeAttemptLogStream', () => {
|
|
|
74
80
|
expect(metrics.recordAppendedAdd).toHaveBeenCalledWith(1, {kind: 'runner_lost'});
|
|
75
81
|
});
|
|
76
82
|
|
|
83
|
+
it.each([
|
|
84
|
+
{logOutcome: 'drained' as const, expectedOrigins: ['runner', 'control']},
|
|
85
|
+
{logOutcome: 'abandoned' as const, expectedOrigins: ['runner', 'control', 'control']},
|
|
86
|
+
])('flushes a pending Claude result before a $logOutcome close', async ({
|
|
87
|
+
logOutcome,
|
|
88
|
+
expectedOrigins,
|
|
89
|
+
}) => {
|
|
90
|
+
const identity = newIdentity({logOutcome});
|
|
91
|
+
await jobAccountingFactory.create({
|
|
92
|
+
jobId: identity.jobId,
|
|
93
|
+
workspaceId: identity.workspaceId,
|
|
94
|
+
startedAt: new Date(Date.now() - 60 * 60_000),
|
|
95
|
+
});
|
|
96
|
+
const workflows = createFakeInterModuleClients({
|
|
97
|
+
workflows: defineInterModulePresentation(workflowsInterModuleContract, {
|
|
98
|
+
startRunFromTrigger: vi.fn(),
|
|
99
|
+
deliverEventToJobListener: vi.fn(),
|
|
100
|
+
getStepLogContext: () => ({harness: 'claude' as const}),
|
|
101
|
+
getLeasedAgentToolContext: vi.fn(),
|
|
102
|
+
}),
|
|
103
|
+
}).workflows;
|
|
104
|
+
const body = ndjsonBody(
|
|
105
|
+
sessionLine(JSON.stringify({type: 'system', subtype: 'init', session_id: 'session-1'})),
|
|
106
|
+
sessionLine(JSON.stringify({type: 'result', subtype: 'success', result: 'response'})),
|
|
107
|
+
);
|
|
108
|
+
|
|
109
|
+
await appendLogs({...identity, offset: 0, body}, workflows);
|
|
110
|
+
const stream = await finalizeAttemptLogStream(identity);
|
|
111
|
+
const chunks = await listChunks(stream.id);
|
|
112
|
+
const records = chunks.flatMap((chunk) =>
|
|
113
|
+
chunk.data.toString('utf8').split('\n').filter(Boolean).map(parseLogRecordLine),
|
|
114
|
+
);
|
|
115
|
+
|
|
116
|
+
expect(chunks.map((chunk) => chunk.origin)).toEqual(expectedOrigins);
|
|
117
|
+
expect(
|
|
118
|
+
records.flatMap((record) =>
|
|
119
|
+
record.type === 'agent_session' && record.row.kind === 'lifecycle'
|
|
120
|
+
? [record.row.label]
|
|
121
|
+
: [],
|
|
122
|
+
),
|
|
123
|
+
).toEqual(['Session started', 'Session completed']);
|
|
124
|
+
if (logOutcome === 'abandoned') {
|
|
125
|
+
expect(records.map((record) => record.type)).toEqual([
|
|
126
|
+
'agent_session',
|
|
127
|
+
'agent_session',
|
|
128
|
+
'runner_lost',
|
|
129
|
+
]);
|
|
130
|
+
}
|
|
131
|
+
expect(stream.claudePendingResult).toBeNull();
|
|
132
|
+
});
|
|
133
|
+
|
|
77
134
|
it('does not emit another tombstone or close event when finalized again', async () => {
|
|
78
135
|
const identity = newIdentity({logOutcome: 'abandoned'});
|
|
79
136
|
|
|
@@ -5,6 +5,7 @@ import type {
|
|
|
5
5
|
SessionViewToolCallRow,
|
|
6
6
|
SessionViewToolResultRow,
|
|
7
7
|
} from '@shipfox/api-logs-dto';
|
|
8
|
+
import type {ClaudeParseContext} from '../claude-parser.js';
|
|
8
9
|
import {asLooseObject} from '../entry-schema.js';
|
|
9
10
|
import {
|
|
10
11
|
booleanField,
|
|
@@ -18,14 +19,30 @@ import {
|
|
|
18
19
|
} from '../object.js';
|
|
19
20
|
import {lifecycleRow, messageRow, thinkingRow} from '../rows.js';
|
|
20
21
|
|
|
22
|
+
export const PURE_PROGRESS_CLAUDE_SYSTEM_SUBTYPES = new Set<string>([
|
|
23
|
+
'thinking_tokens',
|
|
24
|
+
'status',
|
|
25
|
+
'session_state_changed',
|
|
26
|
+
'task_progress',
|
|
27
|
+
'hook_progress',
|
|
28
|
+
'hook_started',
|
|
29
|
+
'commands_changed',
|
|
30
|
+
'files_persisted',
|
|
31
|
+
'memory_recall',
|
|
32
|
+
'local_command_output',
|
|
33
|
+
'plugin_install',
|
|
34
|
+
]);
|
|
35
|
+
const OUTPUT_REPROMPT_PREFIX = 'The previous turn ended without setting required workflow outputs:';
|
|
36
|
+
|
|
21
37
|
export function systemRow(
|
|
22
38
|
timestamp: number,
|
|
23
39
|
message: Record<string, unknown>,
|
|
40
|
+
context: ClaudeParseContext,
|
|
24
41
|
): SessionViewLifecycleRow {
|
|
25
42
|
const subtype = stringField(message, 'subtype');
|
|
26
|
-
const
|
|
27
|
-
const
|
|
28
|
-
const
|
|
43
|
+
const sessionId = stringField(message, 'session_id') ?? stringField(message, 'sessionId');
|
|
44
|
+
const isInit = subtype === 'init' || message.type === 'init';
|
|
45
|
+
const baseMeta = [
|
|
29
46
|
metaItem('cwd', stringField(message, 'cwd'), false),
|
|
30
47
|
metaItem('model', stringField(message, 'model')),
|
|
31
48
|
metaItem(
|
|
@@ -34,7 +51,21 @@ export function systemRow(
|
|
|
34
51
|
),
|
|
35
52
|
].filter(isMeta);
|
|
36
53
|
|
|
37
|
-
|
|
54
|
+
if (!isInit)
|
|
55
|
+
return lifecycleRow(timestamp, 'Session event', sessionId ?? null, 'default', false, baseMeta);
|
|
56
|
+
|
|
57
|
+
const isNewSession =
|
|
58
|
+
!context.hasInit || sessionId === undefined || sessionId !== context.sessionId;
|
|
59
|
+
context.hasInit = true;
|
|
60
|
+
context.sessionId = sessionId ?? null;
|
|
61
|
+
context.turn = isNewSession ? 1 : context.turn + 1;
|
|
62
|
+
|
|
63
|
+
const label = isNewSession ? 'Session started' : `Turn ${context.turn} started`;
|
|
64
|
+
const meta = [metaItem('turn', isNewSession ? null : String(context.turn)), ...baseMeta].filter(
|
|
65
|
+
isMeta,
|
|
66
|
+
);
|
|
67
|
+
|
|
68
|
+
return lifecycleRow(timestamp, label, sessionId ?? null, 'default', false, meta);
|
|
38
69
|
}
|
|
39
70
|
|
|
40
71
|
export function assistantRows(
|
|
@@ -92,11 +123,12 @@ export function userRows(
|
|
|
92
123
|
message: Record<string, unknown>,
|
|
93
124
|
): readonly SessionViewRow[] {
|
|
94
125
|
const sdkMessage = asLooseObject(message.message) ?? message;
|
|
126
|
+
const role = isPlatformMessage(sdkMessage) ? 'platform' : 'user';
|
|
95
127
|
const rows: SessionViewRow[] = [];
|
|
96
128
|
const textParts: string[] = [];
|
|
97
129
|
const pushText = () => {
|
|
98
130
|
if (textParts.length === 0) return;
|
|
99
|
-
rows.push(messageRow(timestamp,
|
|
131
|
+
rows.push(messageRow(timestamp, role, role, textParts.join('\n\n'), false));
|
|
100
132
|
textParts.length = 0;
|
|
101
133
|
};
|
|
102
134
|
|
|
@@ -117,12 +149,14 @@ export function userRows(
|
|
|
117
149
|
if (rows.length > 0) return rows;
|
|
118
150
|
|
|
119
151
|
const content = stringField(sdkMessage, 'content');
|
|
120
|
-
return [messageRow(timestamp,
|
|
152
|
+
return [messageRow(timestamp, role, role, content ?? toJson(message), false)];
|
|
121
153
|
}
|
|
122
154
|
|
|
123
155
|
export function resultRow(
|
|
124
156
|
timestamp: number,
|
|
125
157
|
message: Record<string, unknown>,
|
|
158
|
+
turn: number,
|
|
159
|
+
isFinalResult: boolean,
|
|
126
160
|
): SessionViewLifecycleRow {
|
|
127
161
|
const isError = booleanField(message, 'is_error') || booleanField(message, 'isError');
|
|
128
162
|
const subtype = stringField(message, 'subtype');
|
|
@@ -133,6 +167,7 @@ export function resultRow(
|
|
|
133
167
|
stringField(message, 'message') ??
|
|
134
168
|
null;
|
|
135
169
|
const meta = [
|
|
170
|
+
metaItem('turn', !terminalFailure && !isFinalResult && turn > 0 ? String(turn) : null),
|
|
136
171
|
numberMeta(message, 'duration_ms', 'duration', 'ms'),
|
|
137
172
|
numberMeta(message, 'duration_api_ms', 'api duration', 'ms'),
|
|
138
173
|
numberMeta(message, 'num_turns', 'turns'),
|
|
@@ -141,7 +176,11 @@ export function resultRow(
|
|
|
141
176
|
|
|
142
177
|
return lifecycleRow(
|
|
143
178
|
timestamp,
|
|
144
|
-
terminalFailure
|
|
179
|
+
terminalFailure
|
|
180
|
+
? 'Session failed'
|
|
181
|
+
: isFinalResult || turn === 0
|
|
182
|
+
? 'Session completed'
|
|
183
|
+
: `Turn ${turn} completed`,
|
|
145
184
|
detail,
|
|
146
185
|
terminalFailure ? 'error' : 'default',
|
|
147
186
|
terminalFailure,
|
|
@@ -149,6 +188,17 @@ export function resultRow(
|
|
|
149
188
|
);
|
|
150
189
|
}
|
|
151
190
|
|
|
191
|
+
function isPlatformMessage(message: Record<string, unknown>): boolean {
|
|
192
|
+
const content = field(message, 'content');
|
|
193
|
+
if (typeof content === 'string') return content.startsWith(OUTPUT_REPROMPT_PREFIX);
|
|
194
|
+
if (!Array.isArray(content)) return false;
|
|
195
|
+
|
|
196
|
+
return content.some((block) => {
|
|
197
|
+
const object = asLooseObject(block);
|
|
198
|
+
return stringField(object, 'text')?.startsWith(OUTPUT_REPROMPT_PREFIX) === true;
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
|
|
152
202
|
function contentBlocks(message: Record<string, unknown>): Record<string, unknown>[] {
|
|
153
203
|
const content = message.content;
|
|
154
204
|
if (!Array.isArray(content)) return [];
|