@shipfox/api-logs 9.3.0 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +25 -0
- package/dist/core/append-logs.d.ts.map +1 -1
- package/dist/core/append-logs.js +100 -12
- package/dist/core/append-logs.js.map +1 -1
- package/dist/core/close-stream.d.ts +6 -5
- package/dist/core/close-stream.d.ts.map +1 -1
- package/dist/core/close-stream.js +25 -7
- package/dist/core/close-stream.js.map +1 -1
- package/dist/core/entities/attempt-stream.d.ts +5 -0
- package/dist/core/entities/attempt-stream.d.ts.map +1 -1
- package/dist/core/entities/attempt-stream.js +1 -1
- package/dist/core/entities/attempt-stream.js.map +1 -1
- package/dist/core/session/claude/rows.d.ts +4 -2
- package/dist/core/session/claude/rows.d.ts.map +1 -1
- package/dist/core/session/claude/rows.js +44 -9
- package/dist/core/session/claude/rows.js.map +1 -1
- package/dist/core/session/claude-parser.d.ts +8 -1
- package/dist/core/session/claude-parser.d.ts.map +1 -1
- package/dist/core/session/claude-parser.js +30 -4
- package/dist/core/session/claude-parser.js.map +1 -1
- package/dist/core/session/parse-session.d.ts +6 -1
- package/dist/core/session/parse-session.d.ts.map +1 -1
- package/dist/core/session/parse-session.js +2 -2
- package/dist/core/session/parse-session.js.map +1 -1
- package/dist/db/db.d.ts +94 -0
- package/dist/db/db.d.ts.map +1 -1
- package/dist/db/schema/attempt-streams.d.ts +94 -0
- package/dist/db/schema/attempt-streams.d.ts.map +1 -1
- package/dist/db/schema/attempt-streams.js +9 -1
- package/dist/db/schema/attempt-streams.js.map +1 -1
- package/dist/db/streams.d.ts +17 -6
- package/dist/db/streams.d.ts.map +1 -1
- package/dist/db/streams.js +21 -6
- package/dist/db/streams.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/drizzle/0001_strong_major_mapleleaf.sql +4 -0
- package/drizzle/meta/0001_snapshot.json +555 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +5 -5
- package/src/core/append-logs.test.ts +144 -4
- package/src/core/append-logs.ts +158 -6
- package/src/core/close-stream.ts +26 -7
- package/src/core/entities/attempt-stream.ts +6 -0
- package/src/core/finalize-attempt-stream.test.ts +57 -0
- package/src/core/session/claude/rows.ts +57 -7
- package/src/core/session/claude-parser.test.ts +118 -1
- package/src/core/session/claude-parser.ts +55 -4
- package/src/core/session/parse-session.ts +10 -1
- package/src/db/schema/attempt-streams.ts +10 -0
- package/src/db/streams.ts +43 -7
- package/tsconfig.build.tsbuildinfo +1 -1
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
$ shipfox-swc && shipfox-temporal-bundle dist/temporal/workflows/index.js
|
|
2
|
-
Successfully compiled: 59 files with swc (
|
|
3
|
-
2026-07-
|
|
2
|
+
Successfully compiled: 59 files with swc (440.92ms)
|
|
3
|
+
2026-07-28T07:35:36.197Z [INFO] asset workflow-bundle-17839eee3b21fbae5b8c.js 3.18 MiB [emitted] [immutable] (name: main)
|
|
4
4
|
orphan modules 33.5 KiB [orphan] 21 modules
|
|
5
5
|
runtime modules 1.13 KiB 5 modules
|
|
6
6
|
modules by path ../../../node_modules/.pnpm/ 970 KiB 196 modules
|
|
@@ -16,6 +16,6 @@ optional modules 30 bytes [optional]
|
|
|
16
16
|
__temporal_custom_payload_converter (ignored) 15 bytes [optional] [built] [code generated]
|
|
17
17
|
__temporal_custom_failure_converter (ignored) 15 bytes [optional] [built] [code generated]
|
|
18
18
|
../../shared/node/temporal/dist/workflow-error-interceptor.js 1.21 KiB [built] [code generated]
|
|
19
|
-
webpack 5.107.2 compiled successfully in
|
|
20
|
-
2026-07-
|
|
19
|
+
webpack 5.107.2 compiled successfully in 6770 ms
|
|
20
|
+
2026-07-28T07:35:36.211Z [INFO] Workflow bundle created { size: '3.18MB' }
|
|
21
21
|
/home/runner/work/shipfox/shipfox/libs/api/logs/dist/temporal/workflows/index.js: 3335222 bytes
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# @shipfox/api-logs
|
|
2
2
|
|
|
3
|
+
## 10.0.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 930e3e7: Report repeated Claude turns as turn boundaries and distinguish workflow-output re-prompts from user messages.
|
|
8
|
+
- dcce9cb: Drop pure-progress Claude session events before storing normalized log rows.
|
|
9
|
+
|
|
10
|
+
### Patch Changes
|
|
11
|
+
|
|
12
|
+
- Updated dependencies [74f9e31]
|
|
13
|
+
- Updated dependencies [22bf8a2]
|
|
14
|
+
- @shipfox/node-fastify@0.4.0
|
|
15
|
+
- @shipfox/api-workflows-dto@10.0.0
|
|
16
|
+
- @shipfox/api-auth-context@10.0.0
|
|
17
|
+
- @shipfox/node-module@1.0.4
|
|
18
|
+
- @shipfox/api-logs-dto@9.0.2
|
|
19
|
+
- @shipfox/config@1.2.4
|
|
20
|
+
- @shipfox/node-drizzle@0.3.4
|
|
21
|
+
- @shipfox/node-error-monitoring@0.3.0
|
|
22
|
+
- @shipfox/node-opentelemetry@0.6.3
|
|
23
|
+
- @shipfox/node-outbox@0.2.6
|
|
24
|
+
- @shipfox/node-postgres@0.4.4
|
|
25
|
+
- @shipfox/node-temporal@0.4.4
|
|
26
|
+
- @shipfox/workflow-document@2.1.3
|
|
27
|
+
|
|
3
28
|
## 9.3.0
|
|
4
29
|
|
|
5
30
|
### Patch Changes
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"append-logs.d.ts","sourceRoot":"","sources":["../../src/core/append-logs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"append-logs.d.ts","sourceRoot":"","sources":["../../src/core/append-logs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AASnC,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,yCAAyC,CAAC;AAgCnF,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACjB;AAyUD;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,gBAAgB,EACxB,SAAS,CAAC,EAAE,qBAAqB,GAChC,OAAO,CAAC,gBAAgB,CAAC,CA6H3B"}
|
package/dist/core/append-logs.js
CHANGED
|
@@ -6,11 +6,12 @@ import { config } from '#config.js';
|
|
|
6
6
|
import { accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped } from '#db/accounting.js';
|
|
7
7
|
import { insertChunk } from '#db/chunks.js';
|
|
8
8
|
import { db } from '#db/db.js';
|
|
9
|
-
import { casExtendCommittedLength, getAttemptStream, getOrCreateAttemptStreamWithStatus, setDeclaredTotalBytes } from '#db/streams.js';
|
|
9
|
+
import { casExtendCommittedLength, getAttemptStream, getOrCreateAttemptStreamWithStatus, setClaudeParseContext, setDeclaredTotalBytes } from '#db/streams.js';
|
|
10
10
|
import { recordAppendedCount, streamClosedCount, streamOpenedCount } from '#metrics/instance.js';
|
|
11
11
|
import { allowedBudget } from './budget.js';
|
|
12
12
|
import { closeStream, controlTombstone } from './close-stream.js';
|
|
13
13
|
import { MalformedLogChunkError, OffsetGapError } from './errors.js';
|
|
14
|
+
import { claudeInitSessionId, createClaudeParseContext } from './session/claude-parser.js';
|
|
14
15
|
import { parseSessionRecord } from './session/parse-session.js';
|
|
15
16
|
/**
|
|
16
17
|
* Pure pre-transaction parse. Requires whole, newline-terminated lines so
|
|
@@ -168,20 +169,38 @@ async function getSessionHarness(workflows, stepId) {
|
|
|
168
169
|
} : {}
|
|
169
170
|
};
|
|
170
171
|
}
|
|
171
|
-
function buildStoredBody(records, harness) {
|
|
172
|
+
function buildStoredBody(records, harness, initialClaudeParseContext, initialClaudePendingResult, isStreamFinal = false) {
|
|
172
173
|
const storedRecords = [];
|
|
173
|
-
|
|
174
|
+
const parseContext = harness === 'claude' ? {
|
|
175
|
+
claude: initialClaudeParseContext ?? createClaudeParseContext(),
|
|
176
|
+
isFinalResult: true
|
|
177
|
+
} : undefined;
|
|
178
|
+
const latestClaudeInitIndicesBySessionId = harness === 'claude' ? latestClaudeInitIndices(records) : undefined;
|
|
179
|
+
let pendingResult = initialClaudePendingResult ?? null;
|
|
180
|
+
if (parseContext?.claude !== undefined && pendingResult !== null) {
|
|
181
|
+
const firstInit = firstClaudeInit(records);
|
|
182
|
+
if (isStreamFinal || firstInit.hasInit) {
|
|
183
|
+
storedRecords.push(storedAgentSessionRow(finalizePendingResult(pendingResult, firstInit.sessionId !== undefined && firstInit.sessionId === parseContext.claude.sessionId, parseContext.claude.turn)));
|
|
184
|
+
pendingResult = null;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
for (const [index, record] of records.entries()){
|
|
174
188
|
if (record.type !== 'agent_session') {
|
|
175
189
|
storedRecords.push(record);
|
|
176
190
|
continue;
|
|
177
191
|
}
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
192
|
+
if (parseContext?.claude !== undefined && latestClaudeInitIndicesBySessionId !== undefined) {
|
|
193
|
+
parseContext.isFinalResult = !hasFutureClaudeInit(latestClaudeInitIndicesBySessionId, index, parseContext.claude.sessionId);
|
|
194
|
+
}
|
|
195
|
+
for (const row of parseSessionRecord(agentSessionRecord(record), harness, parseContext)){
|
|
196
|
+
if (parseContext?.claude !== undefined && !isStreamFinal && isClaudeFinalResultRow(row)) {
|
|
197
|
+
if (pendingResult !== null) {
|
|
198
|
+
storedRecords.push(storedAgentSessionRow(pendingResult));
|
|
199
|
+
}
|
|
200
|
+
pendingResult = row;
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
storedRecords.push(storedAgentSessionRow(row));
|
|
185
204
|
}
|
|
186
205
|
}
|
|
187
206
|
const body = Buffer.from(storedRecords.map((record)=>`${JSON.stringify(record)}\n`).join(''));
|
|
@@ -191,9 +210,64 @@ function buildStoredBody(records, harness) {
|
|
|
191
210
|
}
|
|
192
211
|
return {
|
|
193
212
|
body,
|
|
194
|
-
recordCounts
|
|
213
|
+
recordCounts,
|
|
214
|
+
claudeParseContext: parseContext?.claude,
|
|
215
|
+
claudePendingResult: parseContext?.claude === undefined ? undefined : pendingResult
|
|
195
216
|
};
|
|
196
217
|
}
|
|
218
|
+
function storedAgentSessionRow(row) {
|
|
219
|
+
return {
|
|
220
|
+
v: 1,
|
|
221
|
+
ts: row.timestamp,
|
|
222
|
+
type: 'agent_session',
|
|
223
|
+
row
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
function isClaudeFinalResultRow(row) {
|
|
227
|
+
return row.kind === 'lifecycle' && row.label === 'Session completed';
|
|
228
|
+
}
|
|
229
|
+
function finalizePendingResult(row, sameSession, turn) {
|
|
230
|
+
if (!sameSession) return row;
|
|
231
|
+
return {
|
|
232
|
+
...row,
|
|
233
|
+
label: `Turn ${turn} completed`,
|
|
234
|
+
meta: [
|
|
235
|
+
{
|
|
236
|
+
label: 'turn',
|
|
237
|
+
value: String(turn)
|
|
238
|
+
},
|
|
239
|
+
...row.meta
|
|
240
|
+
]
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
function firstClaudeInit(records) {
|
|
244
|
+
for (const record of records){
|
|
245
|
+
if (record.type !== 'agent_session') continue;
|
|
246
|
+
const sessionRecord = agentSessionRecord(record);
|
|
247
|
+
const sessionId = claudeInitSessionId(sessionRecord);
|
|
248
|
+
if (sessionId !== undefined) return {
|
|
249
|
+
hasInit: true,
|
|
250
|
+
sessionId
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
return {
|
|
254
|
+
hasInit: false,
|
|
255
|
+
sessionId: undefined
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
function latestClaudeInitIndices(records) {
|
|
259
|
+
const latestIndices = new Map();
|
|
260
|
+
for (const [index, record] of records.entries()){
|
|
261
|
+
if (record.type !== 'agent_session') continue;
|
|
262
|
+
const sessionId = claudeInitSessionId(agentSessionRecord(record));
|
|
263
|
+
if (sessionId !== undefined) latestIndices.set(sessionId, index);
|
|
264
|
+
}
|
|
265
|
+
return latestIndices;
|
|
266
|
+
}
|
|
267
|
+
function hasFutureClaudeInit(latestClaudeInitIndicesBySessionId, currentIndex, sessionId) {
|
|
268
|
+
if (sessionId === null) return false;
|
|
269
|
+
return (latestClaudeInitIndicesBySessionId.get(sessionId) ?? -1) > currentIndex;
|
|
270
|
+
}
|
|
197
271
|
function agentSessionRecord(record) {
|
|
198
272
|
return {
|
|
199
273
|
data: record.data,
|
|
@@ -222,7 +296,6 @@ function agentSessionRecord(record) {
|
|
|
222
296
|
}
|
|
223
297
|
const { declaredTotalBytes } = parsed;
|
|
224
298
|
const sessionHarness = parsed.hasAgentSessionRecord ? await getSessionHarness(workflows, params.stepId) : DEFAULT_HARNESS;
|
|
225
|
-
const stored = buildStoredBody(parsed.records, sessionHarness);
|
|
226
299
|
const commitByteLen = params.body.length;
|
|
227
300
|
const metrics = {
|
|
228
301
|
recordCounts: {},
|
|
@@ -261,6 +334,12 @@ function agentSessionRecord(record) {
|
|
|
261
334
|
capped: await isJobCapped(tx, params.jobId)
|
|
262
335
|
};
|
|
263
336
|
}
|
|
337
|
+
const parseHarness = sessionHarness === 'claude' || stream.claudePendingResult !== null ? 'claude' : sessionHarness;
|
|
338
|
+
const stored = buildStoredBody(parsed.records, parseHarness, parseHarness === 'claude' ? {
|
|
339
|
+
hasInit: stream.claudeHasInit,
|
|
340
|
+
sessionId: stream.claudeSessionId,
|
|
341
|
+
turn: stream.claudeTurn
|
|
342
|
+
} : undefined, parseHarness === 'claude' ? stream.claudePendingResult : undefined, declaredTotalBytes !== undefined);
|
|
264
343
|
const { recordCounts, stored: chunkStored, ...result } = await storeChunk(tx, {
|
|
265
344
|
params,
|
|
266
345
|
streamId: stream.id,
|
|
@@ -270,6 +349,15 @@ function agentSessionRecord(record) {
|
|
|
270
349
|
});
|
|
271
350
|
if (chunkStored) {
|
|
272
351
|
addRecordCounts(metrics.recordCounts, stored.recordCounts);
|
|
352
|
+
if (stored.claudeParseContext !== undefined) {
|
|
353
|
+
await setClaudeParseContext(tx, {
|
|
354
|
+
streamId: stream.id,
|
|
355
|
+
hasInit: stored.claudeParseContext.hasInit,
|
|
356
|
+
sessionId: stored.claudeParseContext.sessionId,
|
|
357
|
+
turn: stored.claudeParseContext.turn,
|
|
358
|
+
pendingResult: stored.claudePendingResult ?? null
|
|
359
|
+
});
|
|
360
|
+
}
|
|
273
361
|
}
|
|
274
362
|
addRecordCounts(metrics.recordCounts, recordCounts);
|
|
275
363
|
// The runner's end record was committed in this append (offset-CAS guarantees
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/core/append-logs.ts"],"sourcesContent":["import {Buffer} from 'node:buffer';\nimport {\n type LogRecord,\n parseLogRecordLine,\n parseRawLogRecordLine,\n type RawLogRecord,\n} from '@shipfox/api-logs-dto';\nimport type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {DEFAULT_HARNESS, type Harness} from '@shipfox/workflow-document';\nimport {config} from '#config.js';\nimport {accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped} from '#db/accounting.js';\nimport {insertChunk} from '#db/chunks.js';\nimport {db, type Transaction} from '#db/db.js';\nimport {\n casExtendCommittedLength,\n getAttemptStream,\n getOrCreateAttemptStreamWithStatus,\n setDeclaredTotalBytes,\n} from '#db/streams.js';\nimport {\n type LogRecordMetricKind,\n recordAppendedCount,\n streamClosedCount,\n streamOpenedCount,\n} from '#metrics/instance.js';\nimport {allowedBudget} from './budget.js';\nimport {closeStream, controlTombstone} from './close-stream.js';\nimport {MalformedLogChunkError, OffsetGapError} from './errors.js';\nimport {parseSessionRecord} from './session/parse-session.js';\nimport type {AgentSessionRecord} from './session/session-record.js';\n\nexport interface AppendLogsParams {\n jobId: string;\n workspaceId: string;\n projectId: string;\n workflowRunAttemptId: string;\n stepId: string;\n attempt: number;\n offset: number;\n body: Buffer;\n}\n\nexport interface AppendLogsResult {\n committedLength: number;\n capped: boolean;\n}\n\ninterface ParsedBody {\n declaredTotalBytes?: number;\n records: RawLogRecord[];\n hasAgentSessionRecord: boolean;\n}\n\n/**\n * Pure pre-transaction parse. Requires whole, newline-terminated lines so\n * `committed_length` always lands on a line boundary (one body = one chunk = whole\n * lines; no line ever spans two chunks). Runs before any lock is taken, so a\n * malformed or large body never holds a row. The offset CAS uses the raw append byte length; the\n * budget charges the normalized body built from these parsed records.\n *\n * Each line is validated against the raw record union (a forged server-only\n * `capped`/`runner_lost` fails here); the declared total is pulled from an `end`\n * record. A line that is a valid server-only record under the read union surfaces\n * its type via `forgedType` for the narrowed audit warn.\n */\nfunction parseAppendBody(body: Buffer): ParsedBody {\n if (body.length === 0) return {records: [], hasAgentSessionRecord: false};\n\n const text = body.toString('utf8');\n if (!text.endsWith('\\n')) {\n throw new MalformedLogChunkError('append body must end with a newline (whole records only)');\n }\n const lines = text.split('\\n');\n lines.pop();\n\n let declaredTotalBytes: number | undefined;\n const records: RawLogRecord[] = [];\n let hasAgentSessionRecord = false;\n for (const line of lines) {\n let record: ReturnType<typeof parseRawLogRecordLine>;\n try {\n record = parseRawLogRecordLine(line);\n } catch {\n throw new MalformedLogChunkError(\n 'append body contains an invalid NDJSON record',\n detectForgedType(line),\n );\n }\n records.push(record);\n if (record.type === 'end') {\n declaredTotalBytes = record.total_bytes;\n }\n // An agent_session line is one whole entry in one record; bound its size here (the DTO\n // schema leaves `data` uncapped because it cannot read this runtime config). An over-cap\n // line is rejected so a single entry can never blow the request body or the spool window.\n if (\n record.type === 'agent_session' &&\n Buffer.byteLength(line, 'utf8') > config.LOG_MAX_SESSION_LINE_BYTES\n ) {\n throw new MalformedLogChunkError(\n `agent_session line exceeds ${config.LOG_MAX_SESSION_LINE_BYTES} bytes`,\n );\n }\n if (record.type === 'agent_session') {\n hasAgentSessionRecord = true;\n }\n }\n\n return declaredTotalBytes === undefined\n ? {records, hasAgentSessionRecord}\n : {declaredTotalBytes, records, hasAgentSessionRecord};\n}\n\n/**\n * A line that fails the raw write union but is a valid record under the read union\n * can only be a server-only `capped`/`runner_lost` tombstone — i.e. a forgery\n * attempt. Returns its type for the audit warn, or undefined for plain garbage.\n */\nfunction detectForgedType(line: string): string | undefined {\n try {\n return parseLogRecordLine(line).type;\n } catch {\n return undefined;\n }\n}\n\nasync function getSessionHarness(\n workflows: WorkflowsModuleClient | undefined,\n stepId: string,\n): Promise<Harness> {\n return workflows ? (await workflows.getStepLogContext({stepId})).harness : DEFAULT_HARNESS;\n}\n\n/**\n * Empty-body heartbeat: report the current committed length without materializing\n * a stream, so a runner cannot mint unbounded rows with empty appends.\n */\nasync function readHeartbeat(tx: Transaction, params: AppendLogsParams): Promise<AppendLogsResult> {\n const existing = await getAttemptStream(tx, {\n jobId: params.jobId,\n stepId: params.stepId,\n attempt: params.attempt,\n });\n return {\n committedLength: existing?.committedLength ?? 0,\n capped: await isJobCapped(tx, params.jobId),\n };\n}\n\ninterface StoreChunkParams {\n params: AppendLogsParams;\n streamId: string;\n body: Buffer;\n committedLength: number;\n declaredTotalBytes: number | undefined;\n}\n\ninterface StoreChunkResult extends AppendLogsResult {\n /**\n * Whether the runner chunk was persisted. False only when the job was already\n * capped and the body (including any `end` record) was dropped, so the stream is\n * not whole and must not be declared-closed.\n */\n stored: boolean;\n recordCounts: Partial<Record<LogRecord['type'], number>>;\n}\n\n/**\n * Accrues the stored bytes, persists the chunk, and trips the per-job cap when\n * this append crosses the budget. Runs only after the offset-CAS extended\n * `committed_length`, so the committed length already reflects the accepted bytes.\n */\nasync function storeChunk(\n tx: Transaction,\n {params, streamId, body, committedLength, declaredTotalBytes}: StoreChunkParams,\n): Promise<StoreChunkResult> {\n await ensureJobAccounting(tx, {jobId: params.jobId, workspaceId: params.workspaceId});\n const storedByteLen = body.length;\n const accrued = await accrueStoredBytes(tx, {jobId: params.jobId, delta: storedByteLen});\n\n // Already capped: accept-and-drop. committed_length has advanced so the runner\n // drains its spool cleanly instead of retry-looping; nothing is stored.\n if (!accrued) return {committedLength, capped: true, stored: false, recordCounts: {}};\n\n await insertChunk(tx, {\n streamId,\n streamOffset: params.offset,\n byteLen: storedByteLen,\n data: body,\n origin: 'runner',\n });\n if (declaredTotalBytes !== undefined) {\n await setDeclaredTotalBytes(tx, {streamId, declaredTotalBytes});\n }\n\n const allowed = allowedBudget({\n baseBytes: config.LOG_BUDGET_BASE_BYTES,\n ratePerMinuteBytes: config.LOG_BUDGET_RATE_BYTES_PER_MINUTE,\n elapsedMs: Date.now() - accrued.startedAt.getTime(),\n });\n if (accrued.used <= allowed) {\n return {committedLength, capped: false, stored: true, recordCounts: {}};\n }\n\n // Over budget. No hard ceiling: this crossing append is stored in full (overshoot\n // bounded by one body). Claim the cap once and inject an in-band `capped` tombstone\n // for the winner.\n const won = await claimCap(tx, params.jobId);\n if (won) {\n const tombstone = controlTombstone('capped');\n await insertChunk(tx, {\n streamId,\n streamOffset: committedLength,\n byteLen: tombstone.length,\n data: tombstone,\n origin: 'control',\n });\n }\n return {\n committedLength,\n capped: true,\n stored: true,\n recordCounts: won ? {capped: 1} : {},\n };\n}\n\nfunction buildStoredBody(\n records: readonly RawLogRecord[],\n harness: Harness,\n): {body: Buffer; recordCounts: Partial<Record<LogRecord['type'], number>>} {\n const storedRecords: LogRecord[] = [];\n for (const record of records) {\n if (record.type !== 'agent_session') {\n storedRecords.push(record);\n continue;\n }\n\n for (const row of parseSessionRecord(agentSessionRecord(record), harness)) {\n storedRecords.push({v: 1, ts: row.timestamp, type: 'agent_session', row});\n }\n }\n\n const body = Buffer.from(storedRecords.map((record) => `${JSON.stringify(record)}\\n`).join(''));\n const recordCounts: Partial<Record<LogRecord['type'], number>> = {};\n for (const record of storedRecords) {\n recordCounts[record.type] = (recordCounts[record.type] ?? 0) + 1;\n }\n\n return {body, recordCounts};\n}\n\nfunction agentSessionRecord(\n record: Extract<RawLogRecord, {type: 'agent_session'}>,\n): AgentSessionRecord {\n return {data: record.data, ts: record.ts};\n}\n\n/**\n * Concurrency is serialized through Postgres row locks taken implicitly by the\n * conditional UPDATEs, not an explicit `SELECT ... FOR UPDATE`. Appends for one\n * job contend on its single accounting row, so the path is multi-instance safe\n * but not lock-free.\n */\nexport async function appendLogs(\n params: AppendLogsParams,\n workflows?: WorkflowsModuleClient,\n): Promise<AppendLogsResult> {\n let parsed: ParsedBody;\n try {\n parsed = parseAppendBody(params.body);\n } catch (error) {\n // Narrowed audit: only the detectable forgery case, never the payload or a token.\n if (error instanceof MalformedLogChunkError && error.forgedType !== undefined) {\n logger().warn(\n {\n jobId: params.jobId,\n stepId: params.stepId,\n offendingType: error.forgedType,\n },\n 'Rejected forged server-only log record on append',\n );\n }\n throw error;\n }\n const {declaredTotalBytes} = parsed;\n const sessionHarness = parsed.hasAgentSessionRecord\n ? await getSessionHarness(workflows, params.stepId)\n : DEFAULT_HARNESS;\n const stored = buildStoredBody(parsed.records, sessionHarness);\n const commitByteLen = params.body.length;\n const metrics = {\n recordCounts: {} as Partial<Record<LogRecordMetricKind, number>>,\n streamClosedReason: undefined as 'declared' | undefined,\n streamOpened: false,\n };\n\n const result = await db().transaction(async (tx) => {\n if (commitByteLen === 0) return readHeartbeat(tx, params);\n\n const {created, stream} = await getOrCreateAttemptStreamWithStatus(tx, {\n jobId: params.jobId,\n stepId: params.stepId,\n attempt: params.attempt,\n workspaceId: params.workspaceId,\n projectId: params.projectId,\n workflowRunAttemptId: params.workflowRunAttemptId,\n });\n metrics.streamOpened = created;\n\n // Closed stream (the runner's end already landed, or the job-terminated sweep ran):\n // accept-and-drop so a late chunk can never race compaction. committed_length is\n // frozen at close, so this reports the final offset and the runner stops cleanly.\n if (stream.state === 'closed') {\n return {committedLength: stream.committedLength, capped: await isJobCapped(tx, params.jobId)};\n }\n\n const cas = await casExtendCommittedLength(tx, {\n streamId: stream.id,\n offset: params.offset,\n byteLen: commitByteLen,\n });\n if (cas.outcome === 'gap') throw new OffsetGapError(cas.committedLength);\n if (cas.outcome === 'retry') {\n return {committedLength: cas.committedLength, capped: await isJobCapped(tx, params.jobId)};\n }\n\n const {\n recordCounts,\n stored: chunkStored,\n ...result\n } = await storeChunk(tx, {\n params,\n streamId: stream.id,\n body: stored.body,\n committedLength: cas.committedLength,\n declaredTotalBytes,\n });\n if (chunkStored) {\n addRecordCounts(metrics.recordCounts, stored.recordCounts);\n }\n addRecordCounts(metrics.recordCounts, recordCounts);\n\n // The runner's end record was committed in this append (offset-CAS guarantees\n // everything before it is already committed), so the stream is whole. Declared-close\n // it in-band so compaction starts at once instead of waiting for the timeout sweep.\n // Only when the chunk was actually stored: an end body dropped because the job was\n // already capped persists nothing, so the stream is not whole and stays open for the\n // timeout sweep to close it as truncated.\n if (declaredTotalBytes !== undefined && chunkStored) {\n const closed = await closeStream(tx, {streamId: stream.id, reason: 'declared'});\n if (closed) metrics.streamClosedReason = 'declared';\n }\n\n return result;\n });\n\n if (metrics.streamOpened) streamOpenedCount.add(1);\n for (const [kind, count] of Object.entries(metrics.recordCounts)) {\n if (count > 0) recordAppendedCount.add(count, {kind: kind as LogRecordMetricKind});\n }\n if (metrics.streamClosedReason) {\n streamClosedCount.add(1, {reason: metrics.streamClosedReason});\n }\n\n return result;\n}\n\nfunction addRecordCounts(\n target: Partial<Record<LogRecordMetricKind, number>>,\n source: Partial<Record<LogRecordMetricKind, number>>,\n): void {\n for (const [kind, count] of Object.entries(source)) {\n target[kind as LogRecordMetricKind] = (target[kind as LogRecordMetricKind] ?? 0) + (count ?? 0);\n }\n}\n"],"names":["Buffer","parseLogRecordLine","parseRawLogRecordLine","logger","DEFAULT_HARNESS","config","accrueStoredBytes","claimCap","ensureJobAccounting","isJobCapped","insertChunk","db","casExtendCommittedLength","getAttemptStream","getOrCreateAttemptStreamWithStatus","setDeclaredTotalBytes","recordAppendedCount","streamClosedCount","streamOpenedCount","allowedBudget","closeStream","controlTombstone","MalformedLogChunkError","OffsetGapError","parseSessionRecord","parseAppendBody","body","length","records","hasAgentSessionRecord","text","toString","endsWith","lines","split","pop","declaredTotalBytes","line","record","detectForgedType","push","type","total_bytes","byteLength","LOG_MAX_SESSION_LINE_BYTES","undefined","getSessionHarness","workflows","stepId","getStepLogContext","harness","readHeartbeat","tx","params","existing","jobId","attempt","committedLength","capped","storeChunk","streamId","workspaceId","storedByteLen","accrued","delta","stored","recordCounts","streamOffset","offset","byteLen","data","origin","allowed","baseBytes","LOG_BUDGET_BASE_BYTES","ratePerMinuteBytes","LOG_BUDGET_RATE_BYTES_PER_MINUTE","elapsedMs","Date","now","startedAt","getTime","used","won","tombstone","buildStoredBody","storedRecords","row","agentSessionRecord","v","ts","timestamp","from","map","JSON","stringify","join","appendLogs","parsed","error","forgedType","warn","offendingType","sessionHarness","commitByteLen","metrics","streamClosedReason","streamOpened","result","transaction","created","stream","projectId","workflowRunAttemptId","state","cas","id","outcome","chunkStored","addRecordCounts","closed","reason","add","kind","count","Object","entries","target","source"],"mappings":"AAAA,SAAQA,MAAM,QAAO,cAAc;AACnC,SAEEC,kBAAkB,EAClBC,qBAAqB,QAEhB,wBAAwB;AAE/B,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,eAAe,QAAqB,6BAA6B;AACzE,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAQC,iBAAiB,EAAEC,QAAQ,EAAEC,mBAAmB,EAAEC,WAAW,QAAO,oBAAoB;AAChG,SAAQC,WAAW,QAAO,gBAAgB;AAC1C,SAAQC,EAAE,QAAyB,YAAY;AAC/C,SACEC,wBAAwB,EACxBC,gBAAgB,EAChBC,kCAAkC,EAClCC,qBAAqB,QAChB,iBAAiB;AACxB,SAEEC,mBAAmB,EACnBC,iBAAiB,EACjBC,iBAAiB,QACZ,uBAAuB;AAC9B,SAAQC,aAAa,QAAO,cAAc;AAC1C,SAAQC,WAAW,EAAEC,gBAAgB,QAAO,oBAAoB;AAChE,SAAQC,sBAAsB,EAAEC,cAAc,QAAO,cAAc;AACnE,SAAQC,kBAAkB,QAAO,6BAA6B;AAyB9D;;;;;;;;;;;CAWC,GACD,SAASC,gBAAgBC,IAAY;IACnC,IAAIA,KAAKC,MAAM,KAAK,GAAG,OAAO;QAACC,SAAS,EAAE;QAAEC,uBAAuB;IAAK;IAExE,MAAMC,OAAOJ,KAAKK,QAAQ,CAAC;IAC3B,IAAI,CAACD,KAAKE,QAAQ,CAAC,OAAO;QACxB,MAAM,IAAIV,uBAAuB;IACnC;IACA,MAAMW,QAAQH,KAAKI,KAAK,CAAC;IACzBD,MAAME,GAAG;IAET,IAAIC;IACJ,MAAMR,UAA0B,EAAE;IAClC,IAAIC,wBAAwB;IAC5B,KAAK,MAAMQ,QAAQJ,MAAO;QACxB,IAAIK;QACJ,IAAI;YACFA,SAASpC,sBAAsBmC;QACjC,EAAE,OAAM;YACN,MAAM,IAAIf,uBACR,iDACAiB,iBAAiBF;QAErB;QACAT,QAAQY,IAAI,CAACF;QACb,IAAIA,OAAOG,IAAI,KAAK,OAAO;YACzBL,qBAAqBE,OAAOI,WAAW;QACzC;QACA,uFAAuF;QACvF,yFAAyF;QACzF,0FAA0F;QAC1F,IACEJ,OAAOG,IAAI,KAAK,mBAChBzC,OAAO2C,UAAU,CAACN,MAAM,UAAUhC,OAAOuC,0BAA0B,EACnE;YACA,MAAM,IAAItB,uBACR,CAAC,2BAA2B,EAAEjB,OAAOuC,0BAA0B,CAAC,MAAM,CAAC;QAE3E;QACA,IAAIN,OAAOG,IAAI,KAAK,iBAAiB;YACnCZ,wBAAwB;QAC1B;IACF;IAEA,OAAOO,uBAAuBS,YAC1B;QAACjB;QAASC;IAAqB,IAC/B;QAACO;QAAoBR;QAASC;IAAqB;AACzD;AAEA;;;;CAIC,GACD,SAASU,iBAAiBF,IAAY;IACpC,IAAI;QACF,OAAOpC,mBAAmBoC,MAAMI,IAAI;IACtC,EAAE,OAAM;QACN,OAAOI;IACT;AACF;AAEA,eAAeC,kBACbC,SAA4C,EAC5CC,MAAc;IAEd,OAAOD,YAAY,AAAC,CAAA,MAAMA,UAAUE,iBAAiB,CAAC;QAACD;IAAM,EAAC,EAAGE,OAAO,GAAG9C;AAC7E;AAEA;;;CAGC,GACD,eAAe+C,cAAcC,EAAe,EAAEC,MAAwB;IACpE,MAAMC,WAAW,MAAMzC,iBAAiBuC,IAAI;QAC1CG,OAAOF,OAAOE,KAAK;QACnBP,QAAQK,OAAOL,MAAM;QACrBQ,SAASH,OAAOG,OAAO;IACzB;IACA,OAAO;QACLC,iBAAiBH,UAAUG,mBAAmB;QAC9CC,QAAQ,MAAMjD,YAAY2C,IAAIC,OAAOE,KAAK;IAC5C;AACF;AAoBA;;;;CAIC,GACD,eAAeI,WACbP,EAAe,EACf,EAACC,MAAM,EAAEO,QAAQ,EAAElC,IAAI,EAAE+B,eAAe,EAAErB,kBAAkB,EAAmB;IAE/E,MAAM5B,oBAAoB4C,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAEM,aAAaR,OAAOQ,WAAW;IAAA;IACnF,MAAMC,gBAAgBpC,KAAKC,MAAM;IACjC,MAAMoC,UAAU,MAAMzD,kBAAkB8C,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAES,OAAOF;IAAa;IAEtF,+EAA+E;IAC/E,wEAAwE;IACxE,IAAI,CAACC,SAAS,OAAO;QAACN;QAAiBC,QAAQ;QAAMO,QAAQ;QAAOC,cAAc,CAAC;IAAC;IAEpF,MAAMxD,YAAY0C,IAAI;QACpBQ;QACAO,cAAcd,OAAOe,MAAM;QAC3BC,SAASP;QACTQ,MAAM5C;QACN6C,QAAQ;IACV;IACA,IAAInC,uBAAuBS,WAAW;QACpC,MAAM9B,sBAAsBqC,IAAI;YAACQ;YAAUxB;QAAkB;IAC/D;IAEA,MAAMoC,UAAUrD,cAAc;QAC5BsD,WAAWpE,OAAOqE,qBAAqB;QACvCC,oBAAoBtE,OAAOuE,gCAAgC;QAC3DC,WAAWC,KAAKC,GAAG,KAAKhB,QAAQiB,SAAS,CAACC,OAAO;IACnD;IACA,IAAIlB,QAAQmB,IAAI,IAAIV,SAAS;QAC3B,OAAO;YAACf;YAAiBC,QAAQ;YAAOO,QAAQ;YAAMC,cAAc,CAAC;QAAC;IACxE;IAEA,kFAAkF;IAClF,oFAAoF;IACpF,kBAAkB;IAClB,MAAMiB,MAAM,MAAM5E,SAAS6C,IAAIC,OAAOE,KAAK;IAC3C,IAAI4B,KAAK;QACP,MAAMC,YAAY/D,iBAAiB;QACnC,MAAMX,YAAY0C,IAAI;YACpBQ;YACAO,cAAcV;YACdY,SAASe,UAAUzD,MAAM;YACzB2C,MAAMc;YACNb,QAAQ;QACV;IACF;IACA,OAAO;QACLd;QACAC,QAAQ;QACRO,QAAQ;QACRC,cAAciB,MAAM;YAACzB,QAAQ;QAAC,IAAI,CAAC;IACrC;AACF;AAEA,SAAS2B,gBACPzD,OAAgC,EAChCsB,OAAgB;IAEhB,MAAMoC,gBAA6B,EAAE;IACrC,KAAK,MAAMhD,UAAUV,QAAS;QAC5B,IAAIU,OAAOG,IAAI,KAAK,iBAAiB;YACnC6C,cAAc9C,IAAI,CAACF;YACnB;QACF;QAEA,KAAK,MAAMiD,OAAO/D,mBAAmBgE,mBAAmBlD,SAASY,SAAU;YACzEoC,cAAc9C,IAAI,CAAC;gBAACiD,GAAG;gBAAGC,IAAIH,IAAII,SAAS;gBAAElD,MAAM;gBAAiB8C;YAAG;QACzE;IACF;IAEA,MAAM7D,OAAO1B,OAAO4F,IAAI,CAACN,cAAcO,GAAG,CAAC,CAACvD,SAAW,GAAGwD,KAAKC,SAAS,CAACzD,QAAQ,EAAE,CAAC,EAAE0D,IAAI,CAAC;IAC3F,MAAM9B,eAA2D,CAAC;IAClE,KAAK,MAAM5B,UAAUgD,cAAe;QAClCpB,YAAY,CAAC5B,OAAOG,IAAI,CAAC,GAAG,AAACyB,CAAAA,YAAY,CAAC5B,OAAOG,IAAI,CAAC,IAAI,CAAA,IAAK;IACjE;IAEA,OAAO;QAACf;QAAMwC;IAAY;AAC5B;AAEA,SAASsB,mBACPlD,MAAsD;IAEtD,OAAO;QAACgC,MAAMhC,OAAOgC,IAAI;QAAEoB,IAAIpD,OAAOoD,EAAE;IAAA;AAC1C;AAEA;;;;;CAKC,GACD,OAAO,eAAeO,WACpB5C,MAAwB,EACxBN,SAAiC;IAEjC,IAAImD;IACJ,IAAI;QACFA,SAASzE,gBAAgB4B,OAAO3B,IAAI;IACtC,EAAE,OAAOyE,OAAO;QACd,kFAAkF;QAClF,IAAIA,iBAAiB7E,0BAA0B6E,MAAMC,UAAU,KAAKvD,WAAW;YAC7E1C,SAASkG,IAAI,CACX;gBACE9C,OAAOF,OAAOE,KAAK;gBACnBP,QAAQK,OAAOL,MAAM;gBACrBsD,eAAeH,MAAMC,UAAU;YACjC,GACA;QAEJ;QACA,MAAMD;IACR;IACA,MAAM,EAAC/D,kBAAkB,EAAC,GAAG8D;IAC7B,MAAMK,iBAAiBL,OAAOrE,qBAAqB,GAC/C,MAAMiB,kBAAkBC,WAAWM,OAAOL,MAAM,IAChD5C;IACJ,MAAM6D,SAASoB,gBAAgBa,OAAOtE,OAAO,EAAE2E;IAC/C,MAAMC,gBAAgBnD,OAAO3B,IAAI,CAACC,MAAM;IACxC,MAAM8E,UAAU;QACdvC,cAAc,CAAC;QACfwC,oBAAoB7D;QACpB8D,cAAc;IAChB;IAEA,MAAMC,SAAS,MAAMjG,KAAKkG,WAAW,CAAC,OAAOzD;QAC3C,IAAIoD,kBAAkB,GAAG,OAAOrD,cAAcC,IAAIC;QAElD,MAAM,EAACyD,OAAO,EAAEC,MAAM,EAAC,GAAG,MAAMjG,mCAAmCsC,IAAI;YACrEG,OAAOF,OAAOE,KAAK;YACnBP,QAAQK,OAAOL,MAAM;YACrBQ,SAASH,OAAOG,OAAO;YACvBK,aAAaR,OAAOQ,WAAW;YAC/BmD,WAAW3D,OAAO2D,SAAS;YAC3BC,sBAAsB5D,OAAO4D,oBAAoB;QACnD;QACAR,QAAQE,YAAY,GAAGG;QAEvB,oFAAoF;QACpF,iFAAiF;QACjF,kFAAkF;QAClF,IAAIC,OAAOG,KAAK,KAAK,UAAU;YAC7B,OAAO;gBAACzD,iBAAiBsD,OAAOtD,eAAe;gBAAEC,QAAQ,MAAMjD,YAAY2C,IAAIC,OAAOE,KAAK;YAAC;QAC9F;QAEA,MAAM4D,MAAM,MAAMvG,yBAAyBwC,IAAI;YAC7CQ,UAAUmD,OAAOK,EAAE;YACnBhD,QAAQf,OAAOe,MAAM;YACrBC,SAASmC;QACX;QACA,IAAIW,IAAIE,OAAO,KAAK,OAAO,MAAM,IAAI9F,eAAe4F,IAAI1D,eAAe;QACvE,IAAI0D,IAAIE,OAAO,KAAK,SAAS;YAC3B,OAAO;gBAAC5D,iBAAiB0D,IAAI1D,eAAe;gBAAEC,QAAQ,MAAMjD,YAAY2C,IAAIC,OAAOE,KAAK;YAAC;QAC3F;QAEA,MAAM,EACJW,YAAY,EACZD,QAAQqD,WAAW,EACnB,GAAGV,QACJ,GAAG,MAAMjD,WAAWP,IAAI;YACvBC;YACAO,UAAUmD,OAAOK,EAAE;YACnB1F,MAAMuC,OAAOvC,IAAI;YACjB+B,iBAAiB0D,IAAI1D,eAAe;YACpCrB;QACF;QACA,IAAIkF,aAAa;YACfC,gBAAgBd,QAAQvC,YAAY,EAAED,OAAOC,YAAY;QAC3D;QACAqD,gBAAgBd,QAAQvC,YAAY,EAAEA;QAEtC,8EAA8E;QAC9E,qFAAqF;QACrF,oFAAoF;QACpF,mFAAmF;QACnF,qFAAqF;QACrF,0CAA0C;QAC1C,IAAI9B,uBAAuBS,aAAayE,aAAa;YACnD,MAAME,SAAS,MAAMpG,YAAYgC,IAAI;gBAACQ,UAAUmD,OAAOK,EAAE;gBAAEK,QAAQ;YAAU;YAC7E,IAAID,QAAQf,QAAQC,kBAAkB,GAAG;QAC3C;QAEA,OAAOE;IACT;IAEA,IAAIH,QAAQE,YAAY,EAAEzF,kBAAkBwG,GAAG,CAAC;IAChD,KAAK,MAAM,CAACC,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACrB,QAAQvC,YAAY,EAAG;QAChE,IAAI0D,QAAQ,GAAG5G,oBAAoB0G,GAAG,CAACE,OAAO;YAACD,MAAMA;QAA2B;IAClF;IACA,IAAIlB,QAAQC,kBAAkB,EAAE;QAC9BzF,kBAAkByG,GAAG,CAAC,GAAG;YAACD,QAAQhB,QAAQC,kBAAkB;QAAA;IAC9D;IAEA,OAAOE;AACT;AAEA,SAASW,gBACPQ,MAAoD,EACpDC,MAAoD;IAEpD,KAAK,MAAM,CAACL,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACE,QAAS;QAClDD,MAAM,CAACJ,KAA4B,GAAG,AAACI,CAAAA,MAAM,CAACJ,KAA4B,IAAI,CAAA,IAAMC,CAAAA,SAAS,CAAA;IAC/F;AACF"}
|
|
1
|
+
{"version":3,"sources":["../../src/core/append-logs.ts"],"sourcesContent":["import {Buffer} from 'node:buffer';\nimport {\n type LogRecord,\n parseLogRecordLine,\n parseRawLogRecordLine,\n type RawLogRecord,\n type SessionViewLifecycleRow,\n type SessionViewRow,\n} from '@shipfox/api-logs-dto';\nimport type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {DEFAULT_HARNESS, type Harness} from '@shipfox/workflow-document';\nimport {config} from '#config.js';\nimport {accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped} from '#db/accounting.js';\nimport {insertChunk} from '#db/chunks.js';\nimport {db, type Transaction} from '#db/db.js';\nimport {\n casExtendCommittedLength,\n getAttemptStream,\n getOrCreateAttemptStreamWithStatus,\n setClaudeParseContext,\n setDeclaredTotalBytes,\n} from '#db/streams.js';\nimport {\n type LogRecordMetricKind,\n recordAppendedCount,\n streamClosedCount,\n streamOpenedCount,\n} from '#metrics/instance.js';\nimport {allowedBudget} from './budget.js';\nimport {closeStream, controlTombstone} from './close-stream.js';\nimport {MalformedLogChunkError, OffsetGapError} from './errors.js';\nimport {\n type ClaudeParseContext,\n claudeInitSessionId,\n createClaudeParseContext,\n} from './session/claude-parser.js';\nimport type {SessionParseContext} from './session/parse-session.js';\nimport {parseSessionRecord} from './session/parse-session.js';\nimport type {AgentSessionRecord} from './session/session-record.js';\n\nexport interface AppendLogsParams {\n jobId: string;\n workspaceId: string;\n projectId: string;\n workflowRunAttemptId: string;\n stepId: string;\n attempt: number;\n offset: number;\n body: Buffer;\n}\n\nexport interface AppendLogsResult {\n committedLength: number;\n capped: boolean;\n}\n\ninterface ParsedBody {\n declaredTotalBytes?: number;\n records: RawLogRecord[];\n hasAgentSessionRecord: boolean;\n}\n\n/**\n * Pure pre-transaction parse. Requires whole, newline-terminated lines so\n * `committed_length` always lands on a line boundary (one body = one chunk = whole\n * lines; no line ever spans two chunks). Runs before any lock is taken, so a\n * malformed or large body never holds a row. The offset CAS uses the raw append byte length; the\n * budget charges the normalized body built from these parsed records.\n *\n * Each line is validated against the raw record union (a forged server-only\n * `capped`/`runner_lost` fails here); the declared total is pulled from an `end`\n * record. A line that is a valid server-only record under the read union surfaces\n * its type via `forgedType` for the narrowed audit warn.\n */\nfunction parseAppendBody(body: Buffer): ParsedBody {\n if (body.length === 0) return {records: [], hasAgentSessionRecord: false};\n\n const text = body.toString('utf8');\n if (!text.endsWith('\\n')) {\n throw new MalformedLogChunkError('append body must end with a newline (whole records only)');\n }\n const lines = text.split('\\n');\n lines.pop();\n\n let declaredTotalBytes: number | undefined;\n const records: RawLogRecord[] = [];\n let hasAgentSessionRecord = false;\n for (const line of lines) {\n let record: ReturnType<typeof parseRawLogRecordLine>;\n try {\n record = parseRawLogRecordLine(line);\n } catch {\n throw new MalformedLogChunkError(\n 'append body contains an invalid NDJSON record',\n detectForgedType(line),\n );\n }\n records.push(record);\n if (record.type === 'end') {\n declaredTotalBytes = record.total_bytes;\n }\n // An agent_session line is one whole entry in one record; bound its size here (the DTO\n // schema leaves `data` uncapped because it cannot read this runtime config). An over-cap\n // line is rejected so a single entry can never blow the request body or the spool window.\n if (\n record.type === 'agent_session' &&\n Buffer.byteLength(line, 'utf8') > config.LOG_MAX_SESSION_LINE_BYTES\n ) {\n throw new MalformedLogChunkError(\n `agent_session line exceeds ${config.LOG_MAX_SESSION_LINE_BYTES} bytes`,\n );\n }\n if (record.type === 'agent_session') {\n hasAgentSessionRecord = true;\n }\n }\n\n return declaredTotalBytes === undefined\n ? {records, hasAgentSessionRecord}\n : {declaredTotalBytes, records, hasAgentSessionRecord};\n}\n\n/**\n * A line that fails the raw write union but is a valid record under the read union\n * can only be a server-only `capped`/`runner_lost` tombstone — i.e. a forgery\n * attempt. Returns its type for the audit warn, or undefined for plain garbage.\n */\nfunction detectForgedType(line: string): string | undefined {\n try {\n return parseLogRecordLine(line).type;\n } catch {\n return undefined;\n }\n}\n\nasync function getSessionHarness(\n workflows: WorkflowsModuleClient | undefined,\n stepId: string,\n): Promise<Harness> {\n return workflows ? (await workflows.getStepLogContext({stepId})).harness : DEFAULT_HARNESS;\n}\n\n/**\n * Empty-body heartbeat: report the current committed length without materializing\n * a stream, so a runner cannot mint unbounded rows with empty appends.\n */\nasync function readHeartbeat(tx: Transaction, params: AppendLogsParams): Promise<AppendLogsResult> {\n const existing = await getAttemptStream(tx, {\n jobId: params.jobId,\n stepId: params.stepId,\n attempt: params.attempt,\n });\n return {\n committedLength: existing?.committedLength ?? 0,\n capped: await isJobCapped(tx, params.jobId),\n };\n}\n\ninterface StoreChunkParams {\n params: AppendLogsParams;\n streamId: string;\n body: Buffer;\n committedLength: number;\n declaredTotalBytes: number | undefined;\n}\n\ninterface StoreChunkResult extends AppendLogsResult {\n /**\n * Whether the runner chunk was persisted. False only when the job was already\n * capped and the body (including any `end` record) was dropped, so the stream is\n * not whole and must not be declared-closed.\n */\n stored: boolean;\n recordCounts: Partial<Record<LogRecord['type'], number>>;\n}\n\ninterface StoredBody {\n body: Buffer;\n recordCounts: Partial<Record<LogRecord['type'], number>>;\n claudeParseContext: ClaudeParseContext | undefined;\n claudePendingResult: SessionViewLifecycleRow | null | undefined;\n}\n\n/**\n * Accrues the stored bytes, persists the chunk, and trips the per-job cap when\n * this append crosses the budget. Runs only after the offset-CAS extended\n * `committed_length`, so the committed length already reflects the accepted bytes.\n */\nasync function storeChunk(\n tx: Transaction,\n {params, streamId, body, committedLength, declaredTotalBytes}: StoreChunkParams,\n): Promise<StoreChunkResult> {\n await ensureJobAccounting(tx, {jobId: params.jobId, workspaceId: params.workspaceId});\n const storedByteLen = body.length;\n const accrued = await accrueStoredBytes(tx, {jobId: params.jobId, delta: storedByteLen});\n\n // Already capped: accept-and-drop. committed_length has advanced so the runner\n // drains its spool cleanly instead of retry-looping; nothing is stored.\n if (!accrued) return {committedLength, capped: true, stored: false, recordCounts: {}};\n\n await insertChunk(tx, {\n streamId,\n streamOffset: params.offset,\n byteLen: storedByteLen,\n data: body,\n origin: 'runner',\n });\n if (declaredTotalBytes !== undefined) {\n await setDeclaredTotalBytes(tx, {streamId, declaredTotalBytes});\n }\n\n const allowed = allowedBudget({\n baseBytes: config.LOG_BUDGET_BASE_BYTES,\n ratePerMinuteBytes: config.LOG_BUDGET_RATE_BYTES_PER_MINUTE,\n elapsedMs: Date.now() - accrued.startedAt.getTime(),\n });\n if (accrued.used <= allowed) {\n return {committedLength, capped: false, stored: true, recordCounts: {}};\n }\n\n // Over budget. No hard ceiling: this crossing append is stored in full (overshoot\n // bounded by one body). Claim the cap once and inject an in-band `capped` tombstone\n // for the winner.\n const won = await claimCap(tx, params.jobId);\n if (won) {\n const tombstone = controlTombstone('capped');\n await insertChunk(tx, {\n streamId,\n streamOffset: committedLength,\n byteLen: tombstone.length,\n data: tombstone,\n origin: 'control',\n });\n }\n return {\n committedLength,\n capped: true,\n stored: true,\n recordCounts: won ? {capped: 1} : {},\n };\n}\n\nfunction buildStoredBody(\n records: readonly RawLogRecord[],\n harness: Harness,\n initialClaudeParseContext?: ClaudeParseContext,\n initialClaudePendingResult?: SessionViewLifecycleRow | null,\n isStreamFinal = false,\n): StoredBody {\n const storedRecords: LogRecord[] = [];\n const parseContext: SessionParseContext | undefined =\n harness === 'claude'\n ? {\n claude: initialClaudeParseContext ?? createClaudeParseContext(),\n isFinalResult: true,\n }\n : undefined;\n const latestClaudeInitIndicesBySessionId =\n harness === 'claude' ? latestClaudeInitIndices(records) : undefined;\n let pendingResult = initialClaudePendingResult ?? null;\n\n if (parseContext?.claude !== undefined && pendingResult !== null) {\n const firstInit = firstClaudeInit(records);\n if (isStreamFinal || firstInit.hasInit) {\n storedRecords.push(\n storedAgentSessionRow(\n finalizePendingResult(\n pendingResult,\n firstInit.sessionId !== undefined &&\n firstInit.sessionId === parseContext.claude.sessionId,\n parseContext.claude.turn,\n ),\n ),\n );\n pendingResult = null;\n }\n }\n\n for (const [index, record] of records.entries()) {\n if (record.type !== 'agent_session') {\n storedRecords.push(record);\n continue;\n }\n\n if (parseContext?.claude !== undefined && latestClaudeInitIndicesBySessionId !== undefined) {\n parseContext.isFinalResult = !hasFutureClaudeInit(\n latestClaudeInitIndicesBySessionId,\n index,\n parseContext.claude.sessionId,\n );\n }\n\n for (const row of parseSessionRecord(agentSessionRecord(record), harness, parseContext)) {\n if (parseContext?.claude !== undefined && !isStreamFinal && isClaudeFinalResultRow(row)) {\n if (pendingResult !== null) {\n storedRecords.push(storedAgentSessionRow(pendingResult));\n }\n pendingResult = row;\n continue;\n }\n storedRecords.push(storedAgentSessionRow(row));\n }\n }\n\n const body = Buffer.from(storedRecords.map((record) => `${JSON.stringify(record)}\\n`).join(''));\n const recordCounts: Partial<Record<LogRecord['type'], number>> = {};\n for (const record of storedRecords) {\n recordCounts[record.type] = (recordCounts[record.type] ?? 0) + 1;\n }\n\n return {\n body,\n recordCounts,\n claudeParseContext: parseContext?.claude,\n claudePendingResult: parseContext?.claude === undefined ? undefined : pendingResult,\n };\n}\n\nfunction storedAgentSessionRow(row: SessionViewRow): LogRecord {\n return {v: 1, ts: row.timestamp, type: 'agent_session', row};\n}\n\nfunction isClaudeFinalResultRow(row: SessionViewRow): row is SessionViewLifecycleRow {\n return row.kind === 'lifecycle' && row.label === 'Session completed';\n}\n\nfunction finalizePendingResult(\n row: SessionViewLifecycleRow,\n sameSession: boolean,\n turn: number,\n): SessionViewLifecycleRow {\n if (!sameSession) return row;\n\n return {\n ...row,\n label: `Turn ${turn} completed`,\n meta: [{label: 'turn', value: String(turn)}, ...row.meta],\n };\n}\n\nfunction firstClaudeInit(records: readonly RawLogRecord[]): {\n hasInit: boolean;\n sessionId: string | undefined;\n} {\n for (const record of records) {\n if (record.type !== 'agent_session') continue;\n const sessionRecord = agentSessionRecord(record);\n const sessionId = claudeInitSessionId(sessionRecord);\n if (sessionId !== undefined) return {hasInit: true, sessionId};\n }\n\n return {hasInit: false, sessionId: undefined};\n}\n\nfunction latestClaudeInitIndices(records: readonly RawLogRecord[]): Map<string, number> {\n const latestIndices = new Map<string, number>();\n\n for (const [index, record] of records.entries()) {\n if (record.type !== 'agent_session') continue;\n\n const sessionId = claudeInitSessionId(agentSessionRecord(record));\n if (sessionId !== undefined) latestIndices.set(sessionId, index);\n }\n\n return latestIndices;\n}\n\nfunction hasFutureClaudeInit(\n latestClaudeInitIndicesBySessionId: ReadonlyMap<string, number>,\n currentIndex: number,\n sessionId: string | null,\n): boolean {\n if (sessionId === null) return false;\n\n return (latestClaudeInitIndicesBySessionId.get(sessionId) ?? -1) > currentIndex;\n}\n\nfunction agentSessionRecord(\n record: Extract<RawLogRecord, {type: 'agent_session'}>,\n): AgentSessionRecord {\n return {data: record.data, ts: record.ts};\n}\n\n/**\n * Concurrency is serialized through Postgres row locks taken implicitly by the\n * conditional UPDATEs, not an explicit `SELECT ... FOR UPDATE`. Appends for one\n * job contend on its single accounting row, so the path is multi-instance safe\n * but not lock-free.\n */\nexport async function appendLogs(\n params: AppendLogsParams,\n workflows?: WorkflowsModuleClient,\n): Promise<AppendLogsResult> {\n let parsed: ParsedBody;\n try {\n parsed = parseAppendBody(params.body);\n } catch (error) {\n // Narrowed audit: only the detectable forgery case, never the payload or a token.\n if (error instanceof MalformedLogChunkError && error.forgedType !== undefined) {\n logger().warn(\n {\n jobId: params.jobId,\n stepId: params.stepId,\n offendingType: error.forgedType,\n },\n 'Rejected forged server-only log record on append',\n );\n }\n throw error;\n }\n const {declaredTotalBytes} = parsed;\n const sessionHarness = parsed.hasAgentSessionRecord\n ? await getSessionHarness(workflows, params.stepId)\n : DEFAULT_HARNESS;\n const commitByteLen = params.body.length;\n const metrics = {\n recordCounts: {} as Partial<Record<LogRecordMetricKind, number>>,\n streamClosedReason: undefined as 'declared' | undefined,\n streamOpened: false,\n };\n\n const result = await db().transaction(async (tx) => {\n if (commitByteLen === 0) return readHeartbeat(tx, params);\n\n const {created, stream} = await getOrCreateAttemptStreamWithStatus(tx, {\n jobId: params.jobId,\n stepId: params.stepId,\n attempt: params.attempt,\n workspaceId: params.workspaceId,\n projectId: params.projectId,\n workflowRunAttemptId: params.workflowRunAttemptId,\n });\n metrics.streamOpened = created;\n\n // Closed stream (the runner's end already landed, or the job-terminated sweep ran):\n // accept-and-drop so a late chunk can never race compaction. committed_length is\n // frozen at close, so this reports the final offset and the runner stops cleanly.\n if (stream.state === 'closed') {\n return {committedLength: stream.committedLength, capped: await isJobCapped(tx, params.jobId)};\n }\n\n const cas = await casExtendCommittedLength(tx, {\n streamId: stream.id,\n offset: params.offset,\n byteLen: commitByteLen,\n });\n if (cas.outcome === 'gap') throw new OffsetGapError(cas.committedLength);\n if (cas.outcome === 'retry') {\n return {committedLength: cas.committedLength, capped: await isJobCapped(tx, params.jobId)};\n }\n\n const parseHarness =\n sessionHarness === 'claude' || stream.claudePendingResult !== null\n ? 'claude'\n : sessionHarness;\n const stored = buildStoredBody(\n parsed.records,\n parseHarness,\n parseHarness === 'claude'\n ? {\n hasInit: stream.claudeHasInit,\n sessionId: stream.claudeSessionId,\n turn: stream.claudeTurn,\n }\n : undefined,\n parseHarness === 'claude' ? stream.claudePendingResult : undefined,\n declaredTotalBytes !== undefined,\n );\n\n const {\n recordCounts,\n stored: chunkStored,\n ...result\n } = await storeChunk(tx, {\n params,\n streamId: stream.id,\n body: stored.body,\n committedLength: cas.committedLength,\n declaredTotalBytes,\n });\n if (chunkStored) {\n addRecordCounts(metrics.recordCounts, stored.recordCounts);\n if (stored.claudeParseContext !== undefined) {\n await setClaudeParseContext(tx, {\n streamId: stream.id,\n hasInit: stored.claudeParseContext.hasInit,\n sessionId: stored.claudeParseContext.sessionId,\n turn: stored.claudeParseContext.turn,\n pendingResult: stored.claudePendingResult ?? null,\n });\n }\n }\n addRecordCounts(metrics.recordCounts, recordCounts);\n\n // The runner's end record was committed in this append (offset-CAS guarantees\n // everything before it is already committed), so the stream is whole. Declared-close\n // it in-band so compaction starts at once instead of waiting for the timeout sweep.\n // Only when the chunk was actually stored: an end body dropped because the job was\n // already capped persists nothing, so the stream is not whole and stays open for the\n // timeout sweep to close it as truncated.\n if (declaredTotalBytes !== undefined && chunkStored) {\n const closed = await closeStream(tx, {streamId: stream.id, reason: 'declared'});\n if (closed) metrics.streamClosedReason = 'declared';\n }\n\n return result;\n });\n\n if (metrics.streamOpened) streamOpenedCount.add(1);\n for (const [kind, count] of Object.entries(metrics.recordCounts)) {\n if (count > 0) recordAppendedCount.add(count, {kind: kind as LogRecordMetricKind});\n }\n if (metrics.streamClosedReason) {\n streamClosedCount.add(1, {reason: metrics.streamClosedReason});\n }\n\n return result;\n}\n\nfunction addRecordCounts(\n target: Partial<Record<LogRecordMetricKind, number>>,\n source: Partial<Record<LogRecordMetricKind, number>>,\n): void {\n for (const [kind, count] of Object.entries(source)) {\n target[kind as LogRecordMetricKind] = (target[kind as LogRecordMetricKind] ?? 0) + (count ?? 0);\n }\n}\n"],"names":["Buffer","parseLogRecordLine","parseRawLogRecordLine","logger","DEFAULT_HARNESS","config","accrueStoredBytes","claimCap","ensureJobAccounting","isJobCapped","insertChunk","db","casExtendCommittedLength","getAttemptStream","getOrCreateAttemptStreamWithStatus","setClaudeParseContext","setDeclaredTotalBytes","recordAppendedCount","streamClosedCount","streamOpenedCount","allowedBudget","closeStream","controlTombstone","MalformedLogChunkError","OffsetGapError","claudeInitSessionId","createClaudeParseContext","parseSessionRecord","parseAppendBody","body","length","records","hasAgentSessionRecord","text","toString","endsWith","lines","split","pop","declaredTotalBytes","line","record","detectForgedType","push","type","total_bytes","byteLength","LOG_MAX_SESSION_LINE_BYTES","undefined","getSessionHarness","workflows","stepId","getStepLogContext","harness","readHeartbeat","tx","params","existing","jobId","attempt","committedLength","capped","storeChunk","streamId","workspaceId","storedByteLen","accrued","delta","stored","recordCounts","streamOffset","offset","byteLen","data","origin","allowed","baseBytes","LOG_BUDGET_BASE_BYTES","ratePerMinuteBytes","LOG_BUDGET_RATE_BYTES_PER_MINUTE","elapsedMs","Date","now","startedAt","getTime","used","won","tombstone","buildStoredBody","initialClaudeParseContext","initialClaudePendingResult","isStreamFinal","storedRecords","parseContext","claude","isFinalResult","latestClaudeInitIndicesBySessionId","latestClaudeInitIndices","pendingResult","firstInit","firstClaudeInit","hasInit","storedAgentSessionRow","finalizePendingResult","sessionId","turn","index","entries","hasFutureClaudeInit","row","agentSessionRecord","isClaudeFinalResultRow","from","map","JSON","stringify","join","claudeParseContext","claudePendingResult","v","ts","timestamp","kind","label","sameSession","meta","value","String","sessionRecord","latestIndices","Map","set","currentIndex","get","appendLogs","parsed","error","forgedType","warn","offendingType","sessionHarness","commitByteLen","metrics","streamClosedReason","streamOpened","result","transaction","created","stream","projectId","workflowRunAttemptId","state","cas","id","outcome","parseHarness","claudeHasInit","claudeSessionId","claudeTurn","chunkStored","addRecordCounts","closed","reason","add","count","Object","target","source"],"mappings":"AAAA,SAAQA,MAAM,QAAO,cAAc;AACnC,SAEEC,kBAAkB,EAClBC,qBAAqB,QAIhB,wBAAwB;AAE/B,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,eAAe,QAAqB,6BAA6B;AACzE,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAQC,iBAAiB,EAAEC,QAAQ,EAAEC,mBAAmB,EAAEC,WAAW,QAAO,oBAAoB;AAChG,SAAQC,WAAW,QAAO,gBAAgB;AAC1C,SAAQC,EAAE,QAAyB,YAAY;AAC/C,SACEC,wBAAwB,EACxBC,gBAAgB,EAChBC,kCAAkC,EAClCC,qBAAqB,EACrBC,qBAAqB,QAChB,iBAAiB;AACxB,SAEEC,mBAAmB,EACnBC,iBAAiB,EACjBC,iBAAiB,QACZ,uBAAuB;AAC9B,SAAQC,aAAa,QAAO,cAAc;AAC1C,SAAQC,WAAW,EAAEC,gBAAgB,QAAO,oBAAoB;AAChE,SAAQC,sBAAsB,EAAEC,cAAc,QAAO,cAAc;AACnE,SAEEC,mBAAmB,EACnBC,wBAAwB,QACnB,6BAA6B;AAEpC,SAAQC,kBAAkB,QAAO,6BAA6B;AAyB9D;;;;;;;;;;;CAWC,GACD,SAASC,gBAAgBC,IAAY;IACnC,IAAIA,KAAKC,MAAM,KAAK,GAAG,OAAO;QAACC,SAAS,EAAE;QAAEC,uBAAuB;IAAK;IAExE,MAAMC,OAAOJ,KAAKK,QAAQ,CAAC;IAC3B,IAAI,CAACD,KAAKE,QAAQ,CAAC,OAAO;QACxB,MAAM,IAAIZ,uBAAuB;IACnC;IACA,MAAMa,QAAQH,KAAKI,KAAK,CAAC;IACzBD,MAAME,GAAG;IAET,IAAIC;IACJ,MAAMR,UAA0B,EAAE;IAClC,IAAIC,wBAAwB;IAC5B,KAAK,MAAMQ,QAAQJ,MAAO;QACxB,IAAIK;QACJ,IAAI;YACFA,SAASvC,sBAAsBsC;QACjC,EAAE,OAAM;YACN,MAAM,IAAIjB,uBACR,iDACAmB,iBAAiBF;QAErB;QACAT,QAAQY,IAAI,CAACF;QACb,IAAIA,OAAOG,IAAI,KAAK,OAAO;YACzBL,qBAAqBE,OAAOI,WAAW;QACzC;QACA,uFAAuF;QACvF,yFAAyF;QACzF,0FAA0F;QAC1F,IACEJ,OAAOG,IAAI,KAAK,mBAChB5C,OAAO8C,UAAU,CAACN,MAAM,UAAUnC,OAAO0C,0BAA0B,EACnE;YACA,MAAM,IAAIxB,uBACR,CAAC,2BAA2B,EAAElB,OAAO0C,0BAA0B,CAAC,MAAM,CAAC;QAE3E;QACA,IAAIN,OAAOG,IAAI,KAAK,iBAAiB;YACnCZ,wBAAwB;QAC1B;IACF;IAEA,OAAOO,uBAAuBS,YAC1B;QAACjB;QAASC;IAAqB,IAC/B;QAACO;QAAoBR;QAASC;IAAqB;AACzD;AAEA;;;;CAIC,GACD,SAASU,iBAAiBF,IAAY;IACpC,IAAI;QACF,OAAOvC,mBAAmBuC,MAAMI,IAAI;IACtC,EAAE,OAAM;QACN,OAAOI;IACT;AACF;AAEA,eAAeC,kBACbC,SAA4C,EAC5CC,MAAc;IAEd,OAAOD,YAAY,AAAC,CAAA,MAAMA,UAAUE,iBAAiB,CAAC;QAACD;IAAM,EAAC,EAAGE,OAAO,GAAGjD;AAC7E;AAEA;;;CAGC,GACD,eAAekD,cAAcC,EAAe,EAAEC,MAAwB;IACpE,MAAMC,WAAW,MAAM5C,iBAAiB0C,IAAI;QAC1CG,OAAOF,OAAOE,KAAK;QACnBP,QAAQK,OAAOL,MAAM;QACrBQ,SAASH,OAAOG,OAAO;IACzB;IACA,OAAO;QACLC,iBAAiBH,UAAUG,mBAAmB;QAC9CC,QAAQ,MAAMpD,YAAY8C,IAAIC,OAAOE,KAAK;IAC5C;AACF;AA2BA;;;;CAIC,GACD,eAAeI,WACbP,EAAe,EACf,EAACC,MAAM,EAAEO,QAAQ,EAAElC,IAAI,EAAE+B,eAAe,EAAErB,kBAAkB,EAAmB;IAE/E,MAAM/B,oBAAoB+C,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAEM,aAAaR,OAAOQ,WAAW;IAAA;IACnF,MAAMC,gBAAgBpC,KAAKC,MAAM;IACjC,MAAMoC,UAAU,MAAM5D,kBAAkBiD,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAES,OAAOF;IAAa;IAEtF,+EAA+E;IAC/E,wEAAwE;IACxE,IAAI,CAACC,SAAS,OAAO;QAACN;QAAiBC,QAAQ;QAAMO,QAAQ;QAAOC,cAAc,CAAC;IAAC;IAEpF,MAAM3D,YAAY6C,IAAI;QACpBQ;QACAO,cAAcd,OAAOe,MAAM;QAC3BC,SAASP;QACTQ,MAAM5C;QACN6C,QAAQ;IACV;IACA,IAAInC,uBAAuBS,WAAW;QACpC,MAAMhC,sBAAsBuC,IAAI;YAACQ;YAAUxB;QAAkB;IAC/D;IAEA,MAAMoC,UAAUvD,cAAc;QAC5BwD,WAAWvE,OAAOwE,qBAAqB;QACvCC,oBAAoBzE,OAAO0E,gCAAgC;QAC3DC,WAAWC,KAAKC,GAAG,KAAKhB,QAAQiB,SAAS,CAACC,OAAO;IACnD;IACA,IAAIlB,QAAQmB,IAAI,IAAIV,SAAS;QAC3B,OAAO;YAACf;YAAiBC,QAAQ;YAAOO,QAAQ;YAAMC,cAAc,CAAC;QAAC;IACxE;IAEA,kFAAkF;IAClF,oFAAoF;IACpF,kBAAkB;IAClB,MAAMiB,MAAM,MAAM/E,SAASgD,IAAIC,OAAOE,KAAK;IAC3C,IAAI4B,KAAK;QACP,MAAMC,YAAYjE,iBAAiB;QACnC,MAAMZ,YAAY6C,IAAI;YACpBQ;YACAO,cAAcV;YACdY,SAASe,UAAUzD,MAAM;YACzB2C,MAAMc;YACNb,QAAQ;QACV;IACF;IACA,OAAO;QACLd;QACAC,QAAQ;QACRO,QAAQ;QACRC,cAAciB,MAAM;YAACzB,QAAQ;QAAC,IAAI,CAAC;IACrC;AACF;AAEA,SAAS2B,gBACPzD,OAAgC,EAChCsB,OAAgB,EAChBoC,yBAA8C,EAC9CC,0BAA2D,EAC3DC,gBAAgB,KAAK;IAErB,MAAMC,gBAA6B,EAAE;IACrC,MAAMC,eACJxC,YAAY,WACR;QACEyC,QAAQL,6BAA6B/D;QACrCqE,eAAe;IACjB,IACA/C;IACN,MAAMgD,qCACJ3C,YAAY,WAAW4C,wBAAwBlE,WAAWiB;IAC5D,IAAIkD,gBAAgBR,8BAA8B;IAElD,IAAIG,cAAcC,WAAW9C,aAAakD,kBAAkB,MAAM;QAChE,MAAMC,YAAYC,gBAAgBrE;QAClC,IAAI4D,iBAAiBQ,UAAUE,OAAO,EAAE;YACtCT,cAAcjD,IAAI,CAChB2D,sBACEC,sBACEL,eACAC,UAAUK,SAAS,KAAKxD,aACtBmD,UAAUK,SAAS,KAAKX,aAAaC,MAAM,CAACU,SAAS,EACvDX,aAAaC,MAAM,CAACW,IAAI;YAI9BP,gBAAgB;QAClB;IACF;IAEA,KAAK,MAAM,CAACQ,OAAOjE,OAAO,IAAIV,QAAQ4E,OAAO,GAAI;QAC/C,IAAIlE,OAAOG,IAAI,KAAK,iBAAiB;YACnCgD,cAAcjD,IAAI,CAACF;YACnB;QACF;QAEA,IAAIoD,cAAcC,WAAW9C,aAAagD,uCAAuChD,WAAW;YAC1F6C,aAAaE,aAAa,GAAG,CAACa,oBAC5BZ,oCACAU,OACAb,aAAaC,MAAM,CAACU,SAAS;QAEjC;QAEA,KAAK,MAAMK,OAAOlF,mBAAmBmF,mBAAmBrE,SAASY,SAASwC,cAAe;YACvF,IAAIA,cAAcC,WAAW9C,aAAa,CAAC2C,iBAAiBoB,uBAAuBF,MAAM;gBACvF,IAAIX,kBAAkB,MAAM;oBAC1BN,cAAcjD,IAAI,CAAC2D,sBAAsBJ;gBAC3C;gBACAA,gBAAgBW;gBAChB;YACF;YACAjB,cAAcjD,IAAI,CAAC2D,sBAAsBO;QAC3C;IACF;IAEA,MAAMhF,OAAO7B,OAAOgH,IAAI,CAACpB,cAAcqB,GAAG,CAAC,CAACxE,SAAW,GAAGyE,KAAKC,SAAS,CAAC1E,QAAQ,EAAE,CAAC,EAAE2E,IAAI,CAAC;IAC3F,MAAM/C,eAA2D,CAAC;IAClE,KAAK,MAAM5B,UAAUmD,cAAe;QAClCvB,YAAY,CAAC5B,OAAOG,IAAI,CAAC,GAAG,AAACyB,CAAAA,YAAY,CAAC5B,OAAOG,IAAI,CAAC,IAAI,CAAA,IAAK;IACjE;IAEA,OAAO;QACLf;QACAwC;QACAgD,oBAAoBxB,cAAcC;QAClCwB,qBAAqBzB,cAAcC,WAAW9C,YAAYA,YAAYkD;IACxE;AACF;AAEA,SAASI,sBAAsBO,GAAmB;IAChD,OAAO;QAACU,GAAG;QAAGC,IAAIX,IAAIY,SAAS;QAAE7E,MAAM;QAAiBiE;IAAG;AAC7D;AAEA,SAASE,uBAAuBF,GAAmB;IACjD,OAAOA,IAAIa,IAAI,KAAK,eAAeb,IAAIc,KAAK,KAAK;AACnD;AAEA,SAASpB,sBACPM,GAA4B,EAC5Be,WAAoB,EACpBnB,IAAY;IAEZ,IAAI,CAACmB,aAAa,OAAOf;IAEzB,OAAO;QACL,GAAGA,GAAG;QACNc,OAAO,CAAC,KAAK,EAAElB,KAAK,UAAU,CAAC;QAC/BoB,MAAM;YAAC;gBAACF,OAAO;gBAAQG,OAAOC,OAAOtB;YAAK;eAAMI,IAAIgB,IAAI;SAAC;IAC3D;AACF;AAEA,SAASzB,gBAAgBrE,OAAgC;IAIvD,KAAK,MAAMU,UAAUV,QAAS;QAC5B,IAAIU,OAAOG,IAAI,KAAK,iBAAiB;QACrC,MAAMoF,gBAAgBlB,mBAAmBrE;QACzC,MAAM+D,YAAY/E,oBAAoBuG;QACtC,IAAIxB,cAAcxD,WAAW,OAAO;YAACqD,SAAS;YAAMG;QAAS;IAC/D;IAEA,OAAO;QAACH,SAAS;QAAOG,WAAWxD;IAAS;AAC9C;AAEA,SAASiD,wBAAwBlE,OAAgC;IAC/D,MAAMkG,gBAAgB,IAAIC;IAE1B,KAAK,MAAM,CAACxB,OAAOjE,OAAO,IAAIV,QAAQ4E,OAAO,GAAI;QAC/C,IAAIlE,OAAOG,IAAI,KAAK,iBAAiB;QAErC,MAAM4D,YAAY/E,oBAAoBqF,mBAAmBrE;QACzD,IAAI+D,cAAcxD,WAAWiF,cAAcE,GAAG,CAAC3B,WAAWE;IAC5D;IAEA,OAAOuB;AACT;AAEA,SAASrB,oBACPZ,kCAA+D,EAC/DoC,YAAoB,EACpB5B,SAAwB;IAExB,IAAIA,cAAc,MAAM,OAAO;IAE/B,OAAO,AAACR,CAAAA,mCAAmCqC,GAAG,CAAC7B,cAAc,CAAC,CAAA,IAAK4B;AACrE;AAEA,SAAStB,mBACPrE,MAAsD;IAEtD,OAAO;QAACgC,MAAMhC,OAAOgC,IAAI;QAAE+C,IAAI/E,OAAO+E,EAAE;IAAA;AAC1C;AAEA;;;;;CAKC,GACD,OAAO,eAAec,WACpB9E,MAAwB,EACxBN,SAAiC;IAEjC,IAAIqF;IACJ,IAAI;QACFA,SAAS3G,gBAAgB4B,OAAO3B,IAAI;IACtC,EAAE,OAAO2G,OAAO;QACd,kFAAkF;QAClF,IAAIA,iBAAiBjH,0BAA0BiH,MAAMC,UAAU,KAAKzF,WAAW;YAC7E7C,SAASuI,IAAI,CACX;gBACEhF,OAAOF,OAAOE,KAAK;gBACnBP,QAAQK,OAAOL,MAAM;gBACrBwF,eAAeH,MAAMC,UAAU;YACjC,GACA;QAEJ;QACA,MAAMD;IACR;IACA,MAAM,EAACjG,kBAAkB,EAAC,GAAGgG;IAC7B,MAAMK,iBAAiBL,OAAOvG,qBAAqB,GAC/C,MAAMiB,kBAAkBC,WAAWM,OAAOL,MAAM,IAChD/C;IACJ,MAAMyI,gBAAgBrF,OAAO3B,IAAI,CAACC,MAAM;IACxC,MAAMgH,UAAU;QACdzE,cAAc,CAAC;QACf0E,oBAAoB/F;QACpBgG,cAAc;IAChB;IAEA,MAAMC,SAAS,MAAMtI,KAAKuI,WAAW,CAAC,OAAO3F;QAC3C,IAAIsF,kBAAkB,GAAG,OAAOvF,cAAcC,IAAIC;QAElD,MAAM,EAAC2F,OAAO,EAAEC,MAAM,EAAC,GAAG,MAAMtI,mCAAmCyC,IAAI;YACrEG,OAAOF,OAAOE,KAAK;YACnBP,QAAQK,OAAOL,MAAM;YACrBQ,SAASH,OAAOG,OAAO;YACvBK,aAAaR,OAAOQ,WAAW;YAC/BqF,WAAW7F,OAAO6F,SAAS;YAC3BC,sBAAsB9F,OAAO8F,oBAAoB;QACnD;QACAR,QAAQE,YAAY,GAAGG;QAEvB,oFAAoF;QACpF,iFAAiF;QACjF,kFAAkF;QAClF,IAAIC,OAAOG,KAAK,KAAK,UAAU;YAC7B,OAAO;gBAAC3F,iBAAiBwF,OAAOxF,eAAe;gBAAEC,QAAQ,MAAMpD,YAAY8C,IAAIC,OAAOE,KAAK;YAAC;QAC9F;QAEA,MAAM8F,MAAM,MAAM5I,yBAAyB2C,IAAI;YAC7CQ,UAAUqF,OAAOK,EAAE;YACnBlF,QAAQf,OAAOe,MAAM;YACrBC,SAASqE;QACX;QACA,IAAIW,IAAIE,OAAO,KAAK,OAAO,MAAM,IAAIlI,eAAegI,IAAI5F,eAAe;QACvE,IAAI4F,IAAIE,OAAO,KAAK,SAAS;YAC3B,OAAO;gBAAC9F,iBAAiB4F,IAAI5F,eAAe;gBAAEC,QAAQ,MAAMpD,YAAY8C,IAAIC,OAAOE,KAAK;YAAC;QAC3F;QAEA,MAAMiG,eACJf,mBAAmB,YAAYQ,OAAO9B,mBAAmB,KAAK,OAC1D,WACAsB;QACN,MAAMxE,SAASoB,gBACb+C,OAAOxG,OAAO,EACd4H,cACAA,iBAAiB,WACb;YACEtD,SAAS+C,OAAOQ,aAAa;YAC7BpD,WAAW4C,OAAOS,eAAe;YACjCpD,MAAM2C,OAAOU,UAAU;QACzB,IACA9G,WACJ2G,iBAAiB,WAAWP,OAAO9B,mBAAmB,GAAGtE,WACzDT,uBAAuBS;QAGzB,MAAM,EACJqB,YAAY,EACZD,QAAQ2F,WAAW,EACnB,GAAGd,QACJ,GAAG,MAAMnF,WAAWP,IAAI;YACvBC;YACAO,UAAUqF,OAAOK,EAAE;YACnB5H,MAAMuC,OAAOvC,IAAI;YACjB+B,iBAAiB4F,IAAI5F,eAAe;YACpCrB;QACF;QACA,IAAIwH,aAAa;YACfC,gBAAgBlB,QAAQzE,YAAY,EAAED,OAAOC,YAAY;YACzD,IAAID,OAAOiD,kBAAkB,KAAKrE,WAAW;gBAC3C,MAAMjC,sBAAsBwC,IAAI;oBAC9BQ,UAAUqF,OAAOK,EAAE;oBACnBpD,SAASjC,OAAOiD,kBAAkB,CAAChB,OAAO;oBAC1CG,WAAWpC,OAAOiD,kBAAkB,CAACb,SAAS;oBAC9CC,MAAMrC,OAAOiD,kBAAkB,CAACZ,IAAI;oBACpCP,eAAe9B,OAAOkD,mBAAmB,IAAI;gBAC/C;YACF;QACF;QACA0C,gBAAgBlB,QAAQzE,YAAY,EAAEA;QAEtC,8EAA8E;QAC9E,qFAAqF;QACrF,oFAAoF;QACpF,mFAAmF;QACnF,qFAAqF;QACrF,0CAA0C;QAC1C,IAAI9B,uBAAuBS,aAAa+G,aAAa;YACnD,MAAME,SAAS,MAAM5I,YAAYkC,IAAI;gBAACQ,UAAUqF,OAAOK,EAAE;gBAAES,QAAQ;YAAU;YAC7E,IAAID,QAAQnB,QAAQC,kBAAkB,GAAG;QAC3C;QAEA,OAAOE;IACT;IAEA,IAAIH,QAAQE,YAAY,EAAE7H,kBAAkBgJ,GAAG,CAAC;IAChD,KAAK,MAAM,CAACzC,MAAM0C,MAAM,IAAIC,OAAO1D,OAAO,CAACmC,QAAQzE,YAAY,EAAG;QAChE,IAAI+F,QAAQ,GAAGnJ,oBAAoBkJ,GAAG,CAACC,OAAO;YAAC1C,MAAMA;QAA2B;IAClF;IACA,IAAIoB,QAAQC,kBAAkB,EAAE;QAC9B7H,kBAAkBiJ,GAAG,CAAC,GAAG;YAACD,QAAQpB,QAAQC,kBAAkB;QAAA;IAC9D;IAEA,OAAOE;AACT;AAEA,SAASe,gBACPM,MAAoD,EACpDC,MAAoD;IAEpD,KAAK,MAAM,CAAC7C,MAAM0C,MAAM,IAAIC,OAAO1D,OAAO,CAAC4D,QAAS;QAClDD,MAAM,CAAC5C,KAA4B,GAAG,AAAC4C,CAAAA,MAAM,CAAC5C,KAA4B,IAAI,CAAA,IAAM0C,CAAAA,SAAS,CAAA;IAC/F;AACF"}
|
|
@@ -14,12 +14,13 @@ export interface CloseStreamParams {
|
|
|
14
14
|
reason: StreamCloseReason;
|
|
15
15
|
}
|
|
16
16
|
/**
|
|
17
|
-
* The guarded UPDATE (`WHERE state='open'`) in `markStreamClosed`
|
|
18
|
-
* the idempotency gate: a stream already closed by the other path (append-time
|
|
17
|
+
* The row lock plus guarded UPDATE (`WHERE state='open'`) in `markStreamClosed` provide
|
|
18
|
+
* the lock and idempotency gate: a stream already closed by the other path (append-time
|
|
19
19
|
* declared close vs the job-terminated timeout sweep) returns null, so no duplicate
|
|
20
|
-
* `LOG_STREAM_CLOSED` is written and no second tombstone lands. A
|
|
21
|
-
*
|
|
22
|
-
* if any, was injected earlier at
|
|
20
|
+
* `LOG_STREAM_CLOSED` is written and no second tombstone lands. A pending Claude result
|
|
21
|
+
* is materialized before the close event, then a timeout close sets `truncated` and injects
|
|
22
|
+
* a `runner_lost` tombstone in-band (a `capped` tombstone, if any, was injected earlier at
|
|
23
|
+
* the append that tripped the cap).
|
|
23
24
|
*
|
|
24
25
|
* The event drives compaction; it is written in the same transaction as the flip.
|
|
25
26
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"close-stream.d.ts","sourceRoot":"","sources":["../../src/core/close-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AAGnC,OAAO,KAAK,EAAC,aAAa,EAAE,iBAAiB,EAAC,MAAM,kCAAkC,CAAC;AAEvF,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,WAAW,CAAC;AAI3C,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,aAAa,CAAC;AAErD;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,aAAa,GAAG,MAAM,CAG5D;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,iBAAiB,CAAC;CAC3B;AAED
|
|
1
|
+
{"version":3,"file":"close-stream.d.ts","sourceRoot":"","sources":["../../src/core/close-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AAGnC,OAAO,KAAK,EAAC,aAAa,EAAE,iBAAiB,EAAC,MAAM,kCAAkC,CAAC;AAEvF,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,WAAW,CAAC;AAI3C,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,aAAa,CAAC;AAErD;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,aAAa,GAAG,MAAM,CAG5D;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,iBAAiB,CAAC;CAC3B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,WAAW,EACf,MAAM,EAAE,iBAAiB,GACxB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAiD/B"}
|
|
@@ -18,21 +18,39 @@ import { markStreamClosed } from '#db/streams.js';
|
|
|
18
18
|
return Buffer.from(`${JSON.stringify(record)}\n`, 'utf8');
|
|
19
19
|
}
|
|
20
20
|
/**
|
|
21
|
-
* The guarded UPDATE (`WHERE state='open'`) in `markStreamClosed`
|
|
22
|
-
* the idempotency gate: a stream already closed by the other path (append-time
|
|
21
|
+
* The row lock plus guarded UPDATE (`WHERE state='open'`) in `markStreamClosed` provide
|
|
22
|
+
* the lock and idempotency gate: a stream already closed by the other path (append-time
|
|
23
23
|
* declared close vs the job-terminated timeout sweep) returns null, so no duplicate
|
|
24
|
-
* `LOG_STREAM_CLOSED` is written and no second tombstone lands. A
|
|
25
|
-
*
|
|
26
|
-
* if any, was injected earlier at
|
|
24
|
+
* `LOG_STREAM_CLOSED` is written and no second tombstone lands. A pending Claude result
|
|
25
|
+
* is materialized before the close event, then a timeout close sets `truncated` and injects
|
|
26
|
+
* a `runner_lost` tombstone in-band (a `capped` tombstone, if any, was injected earlier at
|
|
27
|
+
* the append that tripped the cap).
|
|
27
28
|
*
|
|
28
29
|
* The event drives compaction; it is written in the same transaction as the flip.
|
|
29
30
|
*/ export async function closeStream(tx, params) {
|
|
30
|
-
const
|
|
31
|
+
const closedResult = await markStreamClosed(tx, {
|
|
31
32
|
streamId: params.streamId,
|
|
32
33
|
reason: params.reason,
|
|
33
34
|
markTruncated: params.reason === 'timeout'
|
|
34
35
|
});
|
|
35
|
-
if (!
|
|
36
|
+
if (!closedResult) return null;
|
|
37
|
+
const { stream: closed, pendingClaudeResult } = closedResult;
|
|
38
|
+
if (pendingClaudeResult !== null) {
|
|
39
|
+
const record = {
|
|
40
|
+
v: 1,
|
|
41
|
+
ts: pendingClaudeResult.timestamp,
|
|
42
|
+
type: 'agent_session',
|
|
43
|
+
row: pendingClaudeResult
|
|
44
|
+
};
|
|
45
|
+
const data = Buffer.from(`${JSON.stringify(record)}\n`, 'utf8');
|
|
46
|
+
await insertChunk(tx, {
|
|
47
|
+
streamId: closed.id,
|
|
48
|
+
streamOffset: closed.committedLength,
|
|
49
|
+
byteLen: data.length,
|
|
50
|
+
data,
|
|
51
|
+
origin: 'control'
|
|
52
|
+
});
|
|
53
|
+
}
|
|
36
54
|
if (params.reason === 'timeout') {
|
|
37
55
|
const tombstone = controlTombstone('runner_lost');
|
|
38
56
|
await insertChunk(tx, {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/core/close-stream.ts"],"sourcesContent":["import {Buffer} from 'node:buffer';\nimport {LOG_STREAM_CLOSED, type LogRecord, type LogsEventMap} from '@shipfox/api-logs-dto';\nimport {writeOutboxEvent} from '@shipfox/node-outbox';\nimport type {AttemptStream, StreamCloseReason} from '#core/entities/attempt-stream.js';\nimport {insertChunk} from '#db/chunks.js';\nimport type {Transaction} from '#db/db.js';\nimport {logsOutbox} from '#db/schema/outbox.js';\nimport {markStreamClosed} from '#db/streams.js';\n\nexport type TombstoneKind = 'capped' | 'runner_lost';\n\n/**\n * Frames a server tombstone as one newline-terminated record. Typed against the\n * shared contract so an envelope change breaks this at compile time. Stored as a\n * `control`-origin chunk; like every server record it does NOT advance\n * `committed_length`, so the CAS axis stays equal to runner spool bytes.\n */\nexport function controlTombstone(kind: TombstoneKind): Buffer {\n const record: LogRecord = {v: 1, ts: Date.now(), type: kind};\n return Buffer.from(`${JSON.stringify(record)}\\n`, 'utf8');\n}\n\nexport interface CloseStreamParams {\n streamId: string;\n reason: StreamCloseReason;\n}\n\n/**\n * The guarded UPDATE (`WHERE state='open'`) in `markStreamClosed`
|
|
1
|
+
{"version":3,"sources":["../../src/core/close-stream.ts"],"sourcesContent":["import {Buffer} from 'node:buffer';\nimport {LOG_STREAM_CLOSED, type LogRecord, type LogsEventMap} from '@shipfox/api-logs-dto';\nimport {writeOutboxEvent} from '@shipfox/node-outbox';\nimport type {AttemptStream, StreamCloseReason} from '#core/entities/attempt-stream.js';\nimport {insertChunk} from '#db/chunks.js';\nimport type {Transaction} from '#db/db.js';\nimport {logsOutbox} from '#db/schema/outbox.js';\nimport {markStreamClosed} from '#db/streams.js';\n\nexport type TombstoneKind = 'capped' | 'runner_lost';\n\n/**\n * Frames a server tombstone as one newline-terminated record. Typed against the\n * shared contract so an envelope change breaks this at compile time. Stored as a\n * `control`-origin chunk; like every server record it does NOT advance\n * `committed_length`, so the CAS axis stays equal to runner spool bytes.\n */\nexport function controlTombstone(kind: TombstoneKind): Buffer {\n const record: LogRecord = {v: 1, ts: Date.now(), type: kind};\n return Buffer.from(`${JSON.stringify(record)}\\n`, 'utf8');\n}\n\nexport interface CloseStreamParams {\n streamId: string;\n reason: StreamCloseReason;\n}\n\n/**\n * The row lock plus guarded UPDATE (`WHERE state='open'`) in `markStreamClosed` provide\n * the lock and idempotency gate: a stream already closed by the other path (append-time\n * declared close vs the job-terminated timeout sweep) returns null, so no duplicate\n * `LOG_STREAM_CLOSED` is written and no second tombstone lands. A pending Claude result\n * is materialized before the close event, then a timeout close sets `truncated` and injects\n * a `runner_lost` tombstone in-band (a `capped` tombstone, if any, was injected earlier at\n * the append that tripped the cap).\n *\n * The event drives compaction; it is written in the same transaction as the flip.\n */\nexport async function closeStream(\n tx: Transaction,\n params: CloseStreamParams,\n): Promise<AttemptStream | null> {\n const closedResult = await markStreamClosed(tx, {\n streamId: params.streamId,\n reason: params.reason,\n markTruncated: params.reason === 'timeout',\n });\n if (!closedResult) return null;\n\n const {stream: closed, pendingClaudeResult} = closedResult;\n if (pendingClaudeResult !== null) {\n const record: LogRecord = {\n v: 1,\n ts: pendingClaudeResult.timestamp,\n type: 'agent_session',\n row: pendingClaudeResult,\n };\n const data = Buffer.from(`${JSON.stringify(record)}\\n`, 'utf8');\n await insertChunk(tx, {\n streamId: closed.id,\n streamOffset: closed.committedLength,\n byteLen: data.length,\n data,\n origin: 'control',\n });\n }\n\n if (params.reason === 'timeout') {\n const tombstone = controlTombstone('runner_lost');\n await insertChunk(tx, {\n streamId: closed.id,\n streamOffset: closed.committedLength,\n byteLen: tombstone.length,\n data: tombstone,\n origin: 'control',\n });\n }\n\n await writeOutboxEvent<LogsEventMap>(tx, logsOutbox, {\n type: LOG_STREAM_CLOSED,\n payload: {\n workspaceId: closed.workspaceId,\n jobId: closed.jobId,\n stepId: closed.stepId,\n attempt: closed.attempt,\n streamId: closed.id,\n },\n });\n\n return closed;\n}\n"],"names":["Buffer","LOG_STREAM_CLOSED","writeOutboxEvent","insertChunk","logsOutbox","markStreamClosed","controlTombstone","kind","record","v","ts","Date","now","type","from","JSON","stringify","closeStream","tx","params","closedResult","streamId","reason","markTruncated","stream","closed","pendingClaudeResult","timestamp","row","data","id","streamOffset","committedLength","byteLen","length","origin","tombstone","payload","workspaceId","jobId","stepId","attempt"],"mappings":"AAAA,SAAQA,MAAM,QAAO,cAAc;AACnC,SAAQC,iBAAiB,QAA0C,wBAAwB;AAC3F,SAAQC,gBAAgB,QAAO,uBAAuB;AAEtD,SAAQC,WAAW,QAAO,gBAAgB;AAE1C,SAAQC,UAAU,QAAO,uBAAuB;AAChD,SAAQC,gBAAgB,QAAO,iBAAiB;AAIhD;;;;;CAKC,GACD,OAAO,SAASC,iBAAiBC,IAAmB;IAClD,MAAMC,SAAoB;QAACC,GAAG;QAAGC,IAAIC,KAAKC,GAAG;QAAIC,MAAMN;IAAI;IAC3D,OAAOP,OAAOc,IAAI,CAAC,GAAGC,KAAKC,SAAS,CAACR,QAAQ,EAAE,CAAC,EAAE;AACpD;AAOA;;;;;;;;;;CAUC,GACD,OAAO,eAAeS,YACpBC,EAAe,EACfC,MAAyB;IAEzB,MAAMC,eAAe,MAAMf,iBAAiBa,IAAI;QAC9CG,UAAUF,OAAOE,QAAQ;QACzBC,QAAQH,OAAOG,MAAM;QACrBC,eAAeJ,OAAOG,MAAM,KAAK;IACnC;IACA,IAAI,CAACF,cAAc,OAAO;IAE1B,MAAM,EAACI,QAAQC,MAAM,EAAEC,mBAAmB,EAAC,GAAGN;IAC9C,IAAIM,wBAAwB,MAAM;QAChC,MAAMlB,SAAoB;YACxBC,GAAG;YACHC,IAAIgB,oBAAoBC,SAAS;YACjCd,MAAM;YACNe,KAAKF;QACP;QACA,MAAMG,OAAO7B,OAAOc,IAAI,CAAC,GAAGC,KAAKC,SAAS,CAACR,QAAQ,EAAE,CAAC,EAAE;QACxD,MAAML,YAAYe,IAAI;YACpBG,UAAUI,OAAOK,EAAE;YACnBC,cAAcN,OAAOO,eAAe;YACpCC,SAASJ,KAAKK,MAAM;YACpBL;YACAM,QAAQ;QACV;IACF;IAEA,IAAIhB,OAAOG,MAAM,KAAK,WAAW;QAC/B,MAAMc,YAAY9B,iBAAiB;QACnC,MAAMH,YAAYe,IAAI;YACpBG,UAAUI,OAAOK,EAAE;YACnBC,cAAcN,OAAOO,eAAe;YACpCC,SAASG,UAAUF,MAAM;YACzBL,MAAMO;YACND,QAAQ;QACV;IACF;IAEA,MAAMjC,iBAA+BgB,IAAId,YAAY;QACnDS,MAAMZ;QACNoC,SAAS;YACPC,aAAab,OAAOa,WAAW;YAC/BC,OAAOd,OAAOc,KAAK;YACnBC,QAAQf,OAAOe,MAAM;YACrBC,SAAShB,OAAOgB,OAAO;YACvBpB,UAAUI,OAAOK,EAAE;QACrB;IACF;IAEA,OAAOL;AACT"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { SessionViewLifecycleRow } from '@shipfox/api-logs-dto';
|
|
1
2
|
/** Lifecycle state of an attempt's log stream. */
|
|
2
3
|
export type StreamState = 'open' | 'closed';
|
|
3
4
|
/** Why a stream was closed, once it is. */
|
|
@@ -27,6 +28,10 @@ export interface AttemptStream {
|
|
|
27
28
|
state: StreamState;
|
|
28
29
|
closeReason: StreamCloseReason | null;
|
|
29
30
|
declaredTotalBytes: number | null;
|
|
31
|
+
claudeHasInit: boolean;
|
|
32
|
+
claudeSessionId: string | null;
|
|
33
|
+
claudeTurn: number;
|
|
34
|
+
claudePendingResult: SessionViewLifecycleRow | null;
|
|
30
35
|
truncated: boolean;
|
|
31
36
|
objectKey: string | null;
|
|
32
37
|
createdAt: Date;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"attempt-stream.d.ts","sourceRoot":"","sources":["../../../src/core/entities/attempt-stream.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,QAAQ,CAAC;AAE5C,2CAA2C;AAC3C,MAAM,MAAM,iBAAiB,GAAG,UAAU,GAAG,SAAS,CAAC;AAEvD;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,WAAW,CAAC;IACnB,WAAW,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACtC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,yFAAyF;IACzF,QAAQ,EAAE,IAAI,GAAG,IAAI,CAAC;CACvB"}
|
|
1
|
+
{"version":3,"file":"attempt-stream.d.ts","sourceRoot":"","sources":["../../../src/core/entities/attempt-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,uBAAuB,EAAC,MAAM,uBAAuB,CAAC;AAEnE,kDAAkD;AAClD,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,QAAQ,CAAC;AAE5C,2CAA2C;AAC3C,MAAM,MAAM,iBAAiB,GAAG,UAAU,GAAG,SAAS,CAAC;AAEvD;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,WAAW,CAAC;IACnB,WAAW,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACtC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,aAAa,EAAE,OAAO,CAAC;IACvB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,mBAAmB,EAAE,uBAAuB,GAAG,IAAI,CAAC;IACpD,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,yFAAyF;IACzF,QAAQ,EAAE,IAAI,GAAG,IAAI,CAAC;CACvB"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
/**
|
|
1
|
+
/**
|
|
2
2
|
* A single `(job, step, attempt)` log stream. Identity is scoped to the lease's
|
|
3
3
|
* job. `workspaceId`, `projectId`, and `workflowRunAttemptId` are stamped from the lease on
|
|
4
4
|
* stream creation and asserted on subsequent appends, so reads can authorize
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/core/entities/attempt-stream.ts"],"sourcesContent":["/** Lifecycle state of an attempt's log stream. */\nexport type StreamState = 'open' | 'closed';\n\n/** Why a stream was closed, once it is. */\nexport type StreamCloseReason = 'declared' | 'timeout';\n\n/**\n * A single `(job, step, attempt)` log stream. Identity is scoped to the lease's\n * job. `workspaceId`, `projectId`, and `workflowRunAttemptId` are stamped from the lease on\n * stream creation and asserted on subsequent appends, so reads can authorize\n * per-project without joining back to workflows. `committedLength` is the\n * offset-CAS axis: raw NDJSON spool bytes the server has durably accepted from\n * the runner.\n *\n * `truncated` is an out-of-band terminal flag set when the timeout sweep\n * force-closes a stream the runner never ended. Budget exhaustion (\"capped\") is a\n * job-level signal recorded in `job_accounting` and surfaced in-band by a `capped`\n * tombstone record, not on the stream row.\n */\nexport interface AttemptStream {\n id: string;\n jobId: string;\n stepId: string;\n attempt: number;\n workspaceId: string;\n projectId: string;\n workflowRunAttemptId: string;\n committedLength: number;\n state: StreamState;\n closeReason: StreamCloseReason | null;\n declaredTotalBytes: number | null;\n truncated: boolean;\n objectKey: string | null;\n createdAt: Date;\n updatedAt: Date;\n /** When the stream was closed (either close path); null while open. Retention anchor. */\n closedAt: Date | null;\n}\n"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"sources":["../../../src/core/entities/attempt-stream.ts"],"sourcesContent":["import type {SessionViewLifecycleRow} from '@shipfox/api-logs-dto';\n\n/** Lifecycle state of an attempt's log stream. */\nexport type StreamState = 'open' | 'closed';\n\n/** Why a stream was closed, once it is. */\nexport type StreamCloseReason = 'declared' | 'timeout';\n\n/**\n * A single `(job, step, attempt)` log stream. Identity is scoped to the lease's\n * job. `workspaceId`, `projectId`, and `workflowRunAttemptId` are stamped from the lease on\n * stream creation and asserted on subsequent appends, so reads can authorize\n * per-project without joining back to workflows. `committedLength` is the\n * offset-CAS axis: raw NDJSON spool bytes the server has durably accepted from\n * the runner.\n *\n * `truncated` is an out-of-band terminal flag set when the timeout sweep\n * force-closes a stream the runner never ended. Budget exhaustion (\"capped\") is a\n * job-level signal recorded in `job_accounting` and surfaced in-band by a `capped`\n * tombstone record, not on the stream row.\n */\nexport interface AttemptStream {\n id: string;\n jobId: string;\n stepId: string;\n attempt: number;\n workspaceId: string;\n projectId: string;\n workflowRunAttemptId: string;\n committedLength: number;\n state: StreamState;\n closeReason: StreamCloseReason | null;\n declaredTotalBytes: number | null;\n claudeHasInit: boolean;\n claudeSessionId: string | null;\n claudeTurn: number;\n claudePendingResult: SessionViewLifecycleRow | null;\n truncated: boolean;\n objectKey: string | null;\n createdAt: Date;\n updatedAt: Date;\n /** When the stream was closed (either close path); null while open. Retention anchor. */\n closedAt: Date | null;\n}\n"],"names":[],"mappings":"AAQA;;;;;;;;;;;;CAYC,GACD,WAsBC"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import type { SessionViewLifecycleRow, SessionViewRow } from '@shipfox/api-logs-dto';
|
|
2
|
-
|
|
2
|
+
import type { ClaudeParseContext } from '../claude-parser.js';
|
|
3
|
+
export declare const PURE_PROGRESS_CLAUDE_SYSTEM_SUBTYPES: Set<string>;
|
|
4
|
+
export declare function systemRow(timestamp: number, message: Record<string, unknown>, context: ClaudeParseContext): SessionViewLifecycleRow;
|
|
3
5
|
export declare function assistantRows(timestamp: number, message: Record<string, unknown>): readonly SessionViewRow[];
|
|
4
6
|
export declare function userRows(timestamp: number, message: Record<string, unknown>): readonly SessionViewRow[];
|
|
5
|
-
export declare function resultRow(timestamp: number, message: Record<string, unknown
|
|
7
|
+
export declare function resultRow(timestamp: number, message: Record<string, unknown>, turn: number, isFinalResult: boolean): SessionViewLifecycleRow;
|
|
6
8
|
//# sourceMappingURL=rows.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rows.d.ts","sourceRoot":"","sources":["../../../../src/core/session/claude/rows.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,uBAAuB,EACvB,cAAc,EAIf,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"rows.d.ts","sourceRoot":"","sources":["../../../../src/core/session/claude/rows.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,uBAAuB,EACvB,cAAc,EAIf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,qBAAqB,CAAC;AAc5D,eAAO,MAAM,oCAAoC,aAY/C,CAAC;AAGH,wBAAgB,SAAS,CACvB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,kBAAkB,GAC1B,uBAAuB,CA4BzB;AAED,wBAAgB,aAAa,CAC3B,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,SAAS,cAAc,EAAE,CA6C3B;AAED,wBAAgB,QAAQ,CACtB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,SAAS,cAAc,EAAE,CA6B3B;AAED,wBAAgB,SAAS,CACvB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,OAAO,GACrB,uBAAuB,CA6BzB"}
|
|
@@ -1,16 +1,40 @@
|
|
|
1
1
|
import { asLooseObject } from '../entry-schema.js';
|
|
2
2
|
import { booleanField, field, formatNumber, isMeta, metaItem, stringField, stringifyValue, toJson } from '../object.js';
|
|
3
3
|
import { lifecycleRow, messageRow, thinkingRow } from '../rows.js';
|
|
4
|
-
export
|
|
4
|
+
export const PURE_PROGRESS_CLAUDE_SYSTEM_SUBTYPES = new Set([
|
|
5
|
+
'thinking_tokens',
|
|
6
|
+
'status',
|
|
7
|
+
'session_state_changed',
|
|
8
|
+
'task_progress',
|
|
9
|
+
'hook_progress',
|
|
10
|
+
'hook_started',
|
|
11
|
+
'commands_changed',
|
|
12
|
+
'files_persisted',
|
|
13
|
+
'memory_recall',
|
|
14
|
+
'local_command_output',
|
|
15
|
+
'plugin_install'
|
|
16
|
+
]);
|
|
17
|
+
const OUTPUT_REPROMPT_PREFIX = 'The previous turn ended without setting required workflow outputs:';
|
|
18
|
+
export function systemRow(timestamp, message, context) {
|
|
5
19
|
const subtype = stringField(message, 'subtype');
|
|
6
|
-
const
|
|
7
|
-
const
|
|
8
|
-
const
|
|
20
|
+
const sessionId = stringField(message, 'session_id') ?? stringField(message, 'sessionId');
|
|
21
|
+
const isInit = subtype === 'init' || message.type === 'init';
|
|
22
|
+
const baseMeta = [
|
|
9
23
|
metaItem('cwd', stringField(message, 'cwd'), false),
|
|
10
24
|
metaItem('model', stringField(message, 'model')),
|
|
11
25
|
metaItem('permission', stringField(message, 'permissionMode') ?? stringField(message, 'permission_mode'))
|
|
12
26
|
].filter(isMeta);
|
|
13
|
-
return lifecycleRow(timestamp,
|
|
27
|
+
if (!isInit) return lifecycleRow(timestamp, 'Session event', sessionId ?? null, 'default', false, baseMeta);
|
|
28
|
+
const isNewSession = !context.hasInit || sessionId === undefined || sessionId !== context.sessionId;
|
|
29
|
+
context.hasInit = true;
|
|
30
|
+
context.sessionId = sessionId ?? null;
|
|
31
|
+
context.turn = isNewSession ? 1 : context.turn + 1;
|
|
32
|
+
const label = isNewSession ? 'Session started' : `Turn ${context.turn} started`;
|
|
33
|
+
const meta = [
|
|
34
|
+
metaItem('turn', isNewSession ? null : String(context.turn)),
|
|
35
|
+
...baseMeta
|
|
36
|
+
].filter(isMeta);
|
|
37
|
+
return lifecycleRow(timestamp, label, sessionId ?? null, 'default', false, meta);
|
|
14
38
|
}
|
|
15
39
|
export function assistantRows(timestamp, message) {
|
|
16
40
|
const sdkMessage = asLooseObject(message.message) ?? message;
|
|
@@ -55,11 +79,12 @@ export function assistantRows(timestamp, message) {
|
|
|
55
79
|
}
|
|
56
80
|
export function userRows(timestamp, message) {
|
|
57
81
|
const sdkMessage = asLooseObject(message.message) ?? message;
|
|
82
|
+
const role = isPlatformMessage(sdkMessage) ? 'platform' : 'user';
|
|
58
83
|
const rows = [];
|
|
59
84
|
const textParts = [];
|
|
60
85
|
const pushText = ()=>{
|
|
61
86
|
if (textParts.length === 0) return;
|
|
62
|
-
rows.push(messageRow(timestamp,
|
|
87
|
+
rows.push(messageRow(timestamp, role, role, textParts.join('\n\n'), false));
|
|
63
88
|
textParts.length = 0;
|
|
64
89
|
};
|
|
65
90
|
for (const block of contentBlocks(sdkMessage)){
|
|
@@ -76,21 +101,31 @@ export function userRows(timestamp, message) {
|
|
|
76
101
|
if (rows.length > 0) return rows;
|
|
77
102
|
const content = stringField(sdkMessage, 'content');
|
|
78
103
|
return [
|
|
79
|
-
messageRow(timestamp,
|
|
104
|
+
messageRow(timestamp, role, role, content ?? toJson(message), false)
|
|
80
105
|
];
|
|
81
106
|
}
|
|
82
|
-
export function resultRow(timestamp, message) {
|
|
107
|
+
export function resultRow(timestamp, message, turn, isFinalResult) {
|
|
83
108
|
const isError = booleanField(message, 'is_error') || booleanField(message, 'isError');
|
|
84
109
|
const subtype = stringField(message, 'subtype');
|
|
85
110
|
const terminalFailure = isError || subtype === 'error';
|
|
86
111
|
const detail = stringField(message, 'result') ?? stringField(message, 'error') ?? stringField(message, 'message') ?? null;
|
|
87
112
|
const meta = [
|
|
113
|
+
metaItem('turn', !terminalFailure && !isFinalResult && turn > 0 ? String(turn) : null),
|
|
88
114
|
numberMeta(message, 'duration_ms', 'duration', 'ms'),
|
|
89
115
|
numberMeta(message, 'duration_api_ms', 'api duration', 'ms'),
|
|
90
116
|
numberMeta(message, 'num_turns', 'turns'),
|
|
91
117
|
costMeta(message)
|
|
92
118
|
].filter(isMeta);
|
|
93
|
-
return lifecycleRow(timestamp, terminalFailure ? 'Session failed' : 'Session completed'
|
|
119
|
+
return lifecycleRow(timestamp, terminalFailure ? 'Session failed' : isFinalResult || turn === 0 ? 'Session completed' : `Turn ${turn} completed`, detail, terminalFailure ? 'error' : 'default', terminalFailure, meta);
|
|
120
|
+
}
|
|
121
|
+
function isPlatformMessage(message) {
|
|
122
|
+
const content = field(message, 'content');
|
|
123
|
+
if (typeof content === 'string') return content.startsWith(OUTPUT_REPROMPT_PREFIX);
|
|
124
|
+
if (!Array.isArray(content)) return false;
|
|
125
|
+
return content.some((block)=>{
|
|
126
|
+
const object = asLooseObject(block);
|
|
127
|
+
return stringField(object, 'text')?.startsWith(OUTPUT_REPROMPT_PREFIX) === true;
|
|
128
|
+
});
|
|
94
129
|
}
|
|
95
130
|
function contentBlocks(message) {
|
|
96
131
|
const content = message.content;
|