@shipfox/api-logs 9.3.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +35 -0
- package/dist/core/append-logs.d.ts.map +1 -1
- package/dist/core/append-logs.js +100 -12
- package/dist/core/append-logs.js.map +1 -1
- package/dist/core/close-stream.d.ts +6 -5
- package/dist/core/close-stream.d.ts.map +1 -1
- package/dist/core/close-stream.js +25 -7
- package/dist/core/close-stream.js.map +1 -1
- package/dist/core/entities/attempt-stream.d.ts +5 -0
- package/dist/core/entities/attempt-stream.d.ts.map +1 -1
- package/dist/core/entities/attempt-stream.js +1 -1
- package/dist/core/entities/attempt-stream.js.map +1 -1
- package/dist/core/session/claude/rows.d.ts +4 -2
- package/dist/core/session/claude/rows.d.ts.map +1 -1
- package/dist/core/session/claude/rows.js +137 -10
- package/dist/core/session/claude/rows.js.map +1 -1
- package/dist/core/session/claude-parser.d.ts +8 -1
- package/dist/core/session/claude-parser.d.ts.map +1 -1
- package/dist/core/session/claude-parser.js +30 -4
- package/dist/core/session/claude-parser.js.map +1 -1
- package/dist/core/session/parse-session.d.ts +6 -1
- package/dist/core/session/parse-session.d.ts.map +1 -1
- package/dist/core/session/parse-session.js +2 -2
- package/dist/core/session/parse-session.js.map +1 -1
- package/dist/db/db.d.ts +94 -0
- package/dist/db/db.d.ts.map +1 -1
- package/dist/db/schema/attempt-streams.d.ts +94 -0
- package/dist/db/schema/attempt-streams.d.ts.map +1 -1
- package/dist/db/schema/attempt-streams.js +9 -1
- package/dist/db/schema/attempt-streams.js.map +1 -1
- package/dist/db/streams.d.ts +17 -6
- package/dist/db/streams.d.ts.map +1 -1
- package/dist/db/streams.js +21 -6
- package/dist/db/streams.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/drizzle/0001_strong_major_mapleleaf.sql +4 -0
- package/drizzle/meta/0001_snapshot.json +555 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +5 -5
- package/src/core/append-logs.test.ts +144 -4
- package/src/core/append-logs.ts +158 -6
- package/src/core/close-stream.ts +26 -7
- package/src/core/entities/attempt-stream.ts +6 -0
- package/src/core/finalize-attempt-stream.test.ts +57 -0
- package/src/core/session/claude/rows.ts +176 -7
- package/src/core/session/claude-parser.test.ts +279 -2
- package/src/core/session/claude-parser.ts +55 -4
- package/src/core/session/parse-session.ts +10 -1
- package/src/db/schema/attempt-streams.ts +10 -0
- package/src/db/streams.ts +43 -7
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { uuidv7PrimaryKey } from '@shipfox/node-drizzle';
|
|
2
2
|
import { sql } from 'drizzle-orm';
|
|
3
|
-
import { bigint, boolean, index, integer, text, timestamp, uniqueIndex, uuid } from 'drizzle-orm/pg-core';
|
|
3
|
+
import { bigint, boolean, index, integer, jsonb, text, timestamp, uniqueIndex, uuid } from 'drizzle-orm/pg-core';
|
|
4
4
|
import { pgTable } from './common.js';
|
|
5
5
|
/**
|
|
6
6
|
* One stream per (job, step, attempt). Identity is scoped by `job_id` (from the
|
|
@@ -47,6 +47,10 @@ import { pgTable } from './common.js';
|
|
|
47
47
|
declaredTotalBytes: bigint('declared_total_bytes', {
|
|
48
48
|
mode: 'number'
|
|
49
49
|
}),
|
|
50
|
+
claudeHasInit: boolean('claude_has_init').notNull().default(false),
|
|
51
|
+
claudeSessionId: text('claude_session_id'),
|
|
52
|
+
claudeTurn: integer('claude_turn').notNull().default(0),
|
|
53
|
+
claudePendingResult: jsonb('claude_pending_result').$type(),
|
|
50
54
|
truncated: boolean('truncated').notNull().default(false),
|
|
51
55
|
objectKey: text('object_key'),
|
|
52
56
|
createdAt: timestamp('created_at', {
|
|
@@ -89,6 +93,10 @@ export function toAttemptStream(row) {
|
|
|
89
93
|
state: row.state,
|
|
90
94
|
closeReason: row.closeReason,
|
|
91
95
|
declaredTotalBytes: row.declaredTotalBytes,
|
|
96
|
+
claudeHasInit: row.claudeHasInit,
|
|
97
|
+
claudeSessionId: row.claudeSessionId,
|
|
98
|
+
claudeTurn: row.claudeTurn,
|
|
99
|
+
claudePendingResult: row.claudePendingResult ?? null,
|
|
92
100
|
truncated: row.truncated,
|
|
93
101
|
objectKey: row.objectKey,
|
|
94
102
|
createdAt: row.createdAt,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/db/schema/attempt-streams.ts"],"sourcesContent":["import {uuidv7PrimaryKey} from '@shipfox/node-drizzle';\nimport {sql} from 'drizzle-orm';\nimport {\n bigint,\n boolean,\n index,\n integer,\n text,\n timestamp,\n uniqueIndex,\n uuid,\n} from 'drizzle-orm/pg-core';\nimport type {AttemptStream} from '#core/entities/attempt-stream.js';\nimport {pgTable} from './common.js';\n\n/**\n * One stream per (job, step, attempt). Identity is scoped by `job_id` (from the\n * lease), so a lease can only ever reach its own job's streams — cross-job log\n * injection is structurally impossible. `workspace_id`, `project_id`, and `workflow_run_attempt_id`\n * are stamped from the lease at create time; they are functionally determined by\n * `job_id` via workflows FKs, so they are denormalized here for self-contained\n * authorization (per-project read filtering, audit) without joining back to\n * workflows. `committed_length` is the offset-CAS axis (raw NDJSON spool bytes the\n * server has durably accepted from the runner).\n *\n * `truncated` is an out-of-band terminal flag set when the timeout sweep\n * force-closes a stream the runner never ended; the `AttemptStream` entity is the\n * canonical reference for what it means.\n *\n * Per-row `committed_length` and `declared_total_bytes` are bounded by the\n * per-job budget, so `mode: 'number'` is safe on the hot path. Any cross-row\n * aggregate (workspace or system-wide totals) MUST read as bigint at the\n * query site — the global sum is unbounded and would silently lose precision\n * past 2^53 as a JS number.\n */\nexport const attemptStreams = pgTable(\n 'attempt_streams',\n {\n id: uuidv7PrimaryKey(),\n jobId: uuid('job_id').notNull(),\n stepId: uuid('step_id').notNull(),\n attempt: integer('attempt').notNull(),\n workspaceId: uuid('workspace_id').notNull(),\n projectId: uuid('project_id').notNull(),\n workflowRunAttemptId: uuid('workflow_run_attempt_id').notNull(),\n committedLength: bigint('committed_length', {mode: 'number'}).notNull().default(0),\n state: text('state', {enum: ['open', 'closed']})\n .notNull()\n .default('open'),\n closeReason: text('close_reason', {enum: ['declared', 'timeout']}),\n declaredTotalBytes: bigint('declared_total_bytes', {mode: 'number'}),\n truncated: boolean('truncated').notNull().default(false),\n objectKey: text('object_key'),\n createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),\n updatedAt: timestamp('updated_at', {withTimezone: true}).notNull().defaultNow(),\n closedAt: timestamp('closed_at', {withTimezone: true}),\n },\n (table) => [\n uniqueIndex('logs_attempt_streams_identity_unique').on(\n table.jobId,\n table.stepId,\n table.attempt,\n ),\n // The session read path looks up a stream by (step, attempt) without the job id, so the\n // identity unique (which leads with job_id) can't serve it. This index keeps that lookup\n // off a sequential scan as the table grows.\n index('logs_attempt_streams_step_attempt_idx').on(table.stepId, table.attempt),\n // Timeout-close sweeps an open job's streams by job_id; the partial index keeps\n // it off the closed tail.\n index('logs_attempt_streams_open_by_job_idx').on(table.jobId).where(sql`\"state\" = 'open'`),\n // Open-age reaping needs an index that stays off the closed tail; created_at is\n // immutable, so appends do not churn it.\n index('logs_attempt_streams_open_age_idx').on(table.createdAt).where(sql`\"state\" = 'open'`),\n // Retention scans closed streams by close age; partial so it never carries the\n // open (in-flight) set.\n index('logs_attempt_streams_retention_idx').on(table.closedAt).where(sql`\"state\" = 'closed'`),\n // Reconcile re-drives closed streams that never got an object key.\n index('logs_attempt_streams_uncompacted_idx')\n .on(table.closedAt)\n .where(sql`\"state\" = 'closed' and \"object_key\" is null`),\n ],\n);\n\nexport type AttemptStreamDb = typeof attemptStreams.$inferSelect;\nexport type AttemptStreamInsertDb = typeof attemptStreams.$inferInsert;\n\nexport function toAttemptStream(row: AttemptStreamDb): AttemptStream {\n return {\n id: row.id,\n jobId: row.jobId,\n stepId: row.stepId,\n attempt: row.attempt,\n workspaceId: row.workspaceId,\n projectId: row.projectId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n committedLength: row.committedLength,\n state: row.state,\n closeReason: row.closeReason,\n declaredTotalBytes: row.declaredTotalBytes,\n truncated: row.truncated,\n objectKey: row.objectKey,\n createdAt: row.createdAt,\n updatedAt: row.updatedAt,\n closedAt: row.closedAt,\n };\n}\n"],"names":["uuidv7PrimaryKey","sql","bigint","boolean","index","integer","text","timestamp","uniqueIndex","uuid","pgTable","attemptStreams","id","jobId","notNull","stepId","attempt","workspaceId","projectId","workflowRunAttemptId","committedLength","mode","default","state","enum","closeReason","declaredTotalBytes","truncated","objectKey","createdAt","withTimezone","defaultNow","updatedAt","closedAt","table","on","where","toAttemptStream","row"],"mappings":"
|
|
1
|
+
{"version":3,"sources":["../../../src/db/schema/attempt-streams.ts"],"sourcesContent":["import type {SessionViewLifecycleRow} from '@shipfox/api-logs-dto';\nimport {uuidv7PrimaryKey} from '@shipfox/node-drizzle';\nimport {sql} from 'drizzle-orm';\nimport {\n bigint,\n boolean,\n index,\n integer,\n jsonb,\n text,\n timestamp,\n uniqueIndex,\n uuid,\n} from 'drizzle-orm/pg-core';\nimport type {AttemptStream} from '#core/entities/attempt-stream.js';\nimport {pgTable} from './common.js';\n\n/**\n * One stream per (job, step, attempt). Identity is scoped by `job_id` (from the\n * lease), so a lease can only ever reach its own job's streams — cross-job log\n * injection is structurally impossible. `workspace_id`, `project_id`, and `workflow_run_attempt_id`\n * are stamped from the lease at create time; they are functionally determined by\n * `job_id` via workflows FKs, so they are denormalized here for self-contained\n * authorization (per-project read filtering, audit) without joining back to\n * workflows. `committed_length` is the offset-CAS axis (raw NDJSON spool bytes the\n * server has durably accepted from the runner).\n *\n * `truncated` is an out-of-band terminal flag set when the timeout sweep\n * force-closes a stream the runner never ended; the `AttemptStream` entity is the\n * canonical reference for what it means.\n *\n * Per-row `committed_length` and `declared_total_bytes` are bounded by the\n * per-job budget, so `mode: 'number'` is safe on the hot path. Any cross-row\n * aggregate (workspace or system-wide totals) MUST read as bigint at the\n * query site — the global sum is unbounded and would silently lose precision\n * past 2^53 as a JS number.\n */\nexport const attemptStreams = pgTable(\n 'attempt_streams',\n {\n id: uuidv7PrimaryKey(),\n jobId: uuid('job_id').notNull(),\n stepId: uuid('step_id').notNull(),\n attempt: integer('attempt').notNull(),\n workspaceId: uuid('workspace_id').notNull(),\n projectId: uuid('project_id').notNull(),\n workflowRunAttemptId: uuid('workflow_run_attempt_id').notNull(),\n committedLength: bigint('committed_length', {mode: 'number'}).notNull().default(0),\n state: text('state', {enum: ['open', 'closed']})\n .notNull()\n .default('open'),\n closeReason: text('close_reason', {enum: ['declared', 'timeout']}),\n declaredTotalBytes: bigint('declared_total_bytes', {mode: 'number'}),\n claudeHasInit: boolean('claude_has_init').notNull().default(false),\n claudeSessionId: text('claude_session_id'),\n claudeTurn: integer('claude_turn').notNull().default(0),\n claudePendingResult: jsonb('claude_pending_result').$type<SessionViewLifecycleRow>(),\n truncated: boolean('truncated').notNull().default(false),\n objectKey: text('object_key'),\n createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),\n updatedAt: timestamp('updated_at', {withTimezone: true}).notNull().defaultNow(),\n closedAt: timestamp('closed_at', {withTimezone: true}),\n },\n (table) => [\n uniqueIndex('logs_attempt_streams_identity_unique').on(\n table.jobId,\n table.stepId,\n table.attempt,\n ),\n // The session read path looks up a stream by (step, attempt) without the job id, so the\n // identity unique (which leads with job_id) can't serve it. This index keeps that lookup\n // off a sequential scan as the table grows.\n index('logs_attempt_streams_step_attempt_idx').on(table.stepId, table.attempt),\n // Timeout-close sweeps an open job's streams by job_id; the partial index keeps\n // it off the closed tail.\n index('logs_attempt_streams_open_by_job_idx').on(table.jobId).where(sql`\"state\" = 'open'`),\n // Open-age reaping needs an index that stays off the closed tail; created_at is\n // immutable, so appends do not churn it.\n index('logs_attempt_streams_open_age_idx').on(table.createdAt).where(sql`\"state\" = 'open'`),\n // Retention scans closed streams by close age; partial so it never carries the\n // open (in-flight) set.\n index('logs_attempt_streams_retention_idx').on(table.closedAt).where(sql`\"state\" = 'closed'`),\n // Reconcile re-drives closed streams that never got an object key.\n index('logs_attempt_streams_uncompacted_idx')\n .on(table.closedAt)\n .where(sql`\"state\" = 'closed' and \"object_key\" is null`),\n ],\n);\n\nexport type AttemptStreamDb = typeof attemptStreams.$inferSelect;\nexport type AttemptStreamInsertDb = typeof attemptStreams.$inferInsert;\n\nexport function toAttemptStream(row: AttemptStreamDb): AttemptStream {\n return {\n id: row.id,\n jobId: row.jobId,\n stepId: row.stepId,\n attempt: row.attempt,\n workspaceId: row.workspaceId,\n projectId: row.projectId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n committedLength: row.committedLength,\n state: row.state,\n closeReason: row.closeReason,\n declaredTotalBytes: row.declaredTotalBytes,\n claudeHasInit: row.claudeHasInit,\n claudeSessionId: row.claudeSessionId,\n claudeTurn: row.claudeTurn,\n claudePendingResult: row.claudePendingResult ?? null,\n truncated: row.truncated,\n objectKey: row.objectKey,\n createdAt: row.createdAt,\n updatedAt: row.updatedAt,\n closedAt: row.closedAt,\n };\n}\n"],"names":["uuidv7PrimaryKey","sql","bigint","boolean","index","integer","jsonb","text","timestamp","uniqueIndex","uuid","pgTable","attemptStreams","id","jobId","notNull","stepId","attempt","workspaceId","projectId","workflowRunAttemptId","committedLength","mode","default","state","enum","closeReason","declaredTotalBytes","claudeHasInit","claudeSessionId","claudeTurn","claudePendingResult","$type","truncated","objectKey","createdAt","withTimezone","defaultNow","updatedAt","closedAt","table","on","where","toAttemptStream","row"],"mappings":"AACA,SAAQA,gBAAgB,QAAO,wBAAwB;AACvD,SAAQC,GAAG,QAAO,cAAc;AAChC,SACEC,MAAM,EACNC,OAAO,EACPC,KAAK,EACLC,OAAO,EACPC,KAAK,EACLC,IAAI,EACJC,SAAS,EACTC,WAAW,EACXC,IAAI,QACC,sBAAsB;AAE7B,SAAQC,OAAO,QAAO,cAAc;AAEpC;;;;;;;;;;;;;;;;;;;CAmBC,GACD,OAAO,MAAMC,iBAAiBD,QAC5B,mBACA;IACEE,IAAIb;IACJc,OAAOJ,KAAK,UAAUK,OAAO;IAC7BC,QAAQN,KAAK,WAAWK,OAAO;IAC/BE,SAASZ,QAAQ,WAAWU,OAAO;IACnCG,aAAaR,KAAK,gBAAgBK,OAAO;IACzCI,WAAWT,KAAK,cAAcK,OAAO;IACrCK,sBAAsBV,KAAK,2BAA2BK,OAAO;IAC7DM,iBAAiBnB,OAAO,oBAAoB;QAACoB,MAAM;IAAQ,GAAGP,OAAO,GAAGQ,OAAO,CAAC;IAChFC,OAAOjB,KAAK,SAAS;QAACkB,MAAM;YAAC;YAAQ;SAAS;IAAA,GAC3CV,OAAO,GACPQ,OAAO,CAAC;IACXG,aAAanB,KAAK,gBAAgB;QAACkB,MAAM;YAAC;YAAY;SAAU;IAAA;IAChEE,oBAAoBzB,OAAO,wBAAwB;QAACoB,MAAM;IAAQ;IAClEM,eAAezB,QAAQ,mBAAmBY,OAAO,GAAGQ,OAAO,CAAC;IAC5DM,iBAAiBtB,KAAK;IACtBuB,YAAYzB,QAAQ,eAAeU,OAAO,GAAGQ,OAAO,CAAC;IACrDQ,qBAAqBzB,MAAM,yBAAyB0B,KAAK;IACzDC,WAAW9B,QAAQ,aAAaY,OAAO,GAAGQ,OAAO,CAAC;IAClDW,WAAW3B,KAAK;IAChB4B,WAAW3B,UAAU,cAAc;QAAC4B,cAAc;IAAI,GAAGrB,OAAO,GAAGsB,UAAU;IAC7EC,WAAW9B,UAAU,cAAc;QAAC4B,cAAc;IAAI,GAAGrB,OAAO,GAAGsB,UAAU;IAC7EE,UAAU/B,UAAU,aAAa;QAAC4B,cAAc;IAAI;AACtD,GACA,CAACI,QAAU;QACT/B,YAAY,wCAAwCgC,EAAE,CACpDD,MAAM1B,KAAK,EACX0B,MAAMxB,MAAM,EACZwB,MAAMvB,OAAO;QAEf,wFAAwF;QACxF,yFAAyF;QACzF,4CAA4C;QAC5Cb,MAAM,yCAAyCqC,EAAE,CAACD,MAAMxB,MAAM,EAAEwB,MAAMvB,OAAO;QAC7E,gFAAgF;QAChF,0BAA0B;QAC1Bb,MAAM,wCAAwCqC,EAAE,CAACD,MAAM1B,KAAK,EAAE4B,KAAK,CAACzC,GAAG,CAAC,gBAAgB,CAAC;QACzF,gFAAgF;QAChF,yCAAyC;QACzCG,MAAM,qCAAqCqC,EAAE,CAACD,MAAML,SAAS,EAAEO,KAAK,CAACzC,GAAG,CAAC,gBAAgB,CAAC;QAC1F,+EAA+E;QAC/E,wBAAwB;QACxBG,MAAM,sCAAsCqC,EAAE,CAACD,MAAMD,QAAQ,EAAEG,KAAK,CAACzC,GAAG,CAAC,kBAAkB,CAAC;QAC5F,mEAAmE;QACnEG,MAAM,wCACHqC,EAAE,CAACD,MAAMD,QAAQ,EACjBG,KAAK,CAACzC,GAAG,CAAC,2CAA2C,CAAC;KAC1D,EACD;AAKF,OAAO,SAAS0C,gBAAgBC,GAAoB;IAClD,OAAO;QACL/B,IAAI+B,IAAI/B,EAAE;QACVC,OAAO8B,IAAI9B,KAAK;QAChBE,QAAQ4B,IAAI5B,MAAM;QAClBC,SAAS2B,IAAI3B,OAAO;QACpBC,aAAa0B,IAAI1B,WAAW;QAC5BC,WAAWyB,IAAIzB,SAAS;QACxBC,sBAAsBwB,IAAIxB,oBAAoB;QAC9CC,iBAAiBuB,IAAIvB,eAAe;QACpCG,OAAOoB,IAAIpB,KAAK;QAChBE,aAAakB,IAAIlB,WAAW;QAC5BC,oBAAoBiB,IAAIjB,kBAAkB;QAC1CC,eAAegB,IAAIhB,aAAa;QAChCC,iBAAiBe,IAAIf,eAAe;QACpCC,YAAYc,IAAId,UAAU;QAC1BC,qBAAqBa,IAAIb,mBAAmB,IAAI;QAChDE,WAAWW,IAAIX,SAAS;QACxBC,WAAWU,IAAIV,SAAS;QACxBC,WAAWS,IAAIT,SAAS;QACxBG,WAAWM,IAAIN,SAAS;QACxBC,UAAUK,IAAIL,QAAQ;IACxB;AACF"}
|
package/dist/db/streams.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { SessionViewLifecycleRow } from '@shipfox/api-logs-dto';
|
|
1
2
|
import type { AttemptStream, StreamCloseReason } from '#core/entities/attempt-stream.js';
|
|
2
3
|
import { type Transaction } from './db.js';
|
|
3
4
|
export interface AttemptStreamIdentity {
|
|
@@ -62,18 +63,28 @@ export declare function setDeclaredTotalBytes(tx: Transaction, params: {
|
|
|
62
63
|
streamId: string;
|
|
63
64
|
declaredTotalBytes: number;
|
|
64
65
|
}): Promise<void>;
|
|
66
|
+
export declare function setClaudeParseContext(tx: Transaction, params: {
|
|
67
|
+
streamId: string;
|
|
68
|
+
hasInit: boolean;
|
|
69
|
+
sessionId: string | null;
|
|
70
|
+
turn: number;
|
|
71
|
+
pendingResult: SessionViewLifecycleRow | null;
|
|
72
|
+
}): Promise<void>;
|
|
65
73
|
/**
|
|
66
|
-
* Guarded close: flips `open → closed
|
|
67
|
-
*
|
|
68
|
-
* it
|
|
69
|
-
*
|
|
70
|
-
*
|
|
74
|
+
* Guarded close: locks the open row, flips `open → closed`, clears any pending Claude result,
|
|
75
|
+
* and stamps `closed_at`. The pending result is returned separately so the caller can
|
|
76
|
+
* materialize it before publishing the close event. Returns null when another path already
|
|
77
|
+
* closed the stream. `committed_length` is left untouched; it stays equal to the runner spool
|
|
78
|
+
* bytes. `truncated` is set only on the timeout path.
|
|
71
79
|
*/
|
|
72
80
|
export declare function markStreamClosed(tx: Transaction, params: {
|
|
73
81
|
streamId: string;
|
|
74
82
|
reason: StreamCloseReason;
|
|
75
83
|
markTruncated: boolean;
|
|
76
|
-
}): Promise<
|
|
84
|
+
}): Promise<{
|
|
85
|
+
stream: AttemptStream;
|
|
86
|
+
pendingClaudeResult: SessionViewLifecycleRow | null;
|
|
87
|
+
} | null>;
|
|
77
88
|
/** Open streams of a job, for the timeout sweep to force-close after the grace period. */
|
|
78
89
|
export declare function listOpenStreamsByJob(jobId: string): Promise<AttemptStream[]>;
|
|
79
90
|
/** For compaction, which is driven by the closed event that carries the stream id. */
|
package/dist/db/streams.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../src/db/streams.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../src/db/streams.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,uBAAuB,EAAC,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAC,aAAa,EAAE,iBAAiB,EAAC,MAAM,kCAAkC,CAAC;AAEvF,OAAO,EAAK,KAAK,WAAW,EAAC,MAAM,SAAS,CAAC;AAI7C,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,WAAW,EACf,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAY/B;AAED;;;;;GAKG;AACH,wBAAsB,sBAAsB,CAAC,MAAM,EAAE;IACnD,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAQhC;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;;;;GAKG;AACH,wBAAsB,kCAAkC,CACtD,EAAE,EAAE,WAAW,EACf,QAAQ,EAAE,qBAAqB,GAC9B,OAAO,CAAC,8BAA8B,CAAC,CAqBzC;AAED,wBAAsB,wBAAwB,CAC5C,EAAE,EAAE,WAAW,EACf,QAAQ,EAAE,qBAAqB,GAC9B,OAAO,CAAC,aAAa,CAAC,CAGxB;AAED,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG,OAAO,GAAG,KAAK,CAAC;AAEtD,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,UAAU,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,EAAE,EAAE,WAAW,EACf,MAAM,EAAE;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GAC1D,OAAO,CAAC,SAAS,CAAC,CAyBpB;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,WAAW,EACf,MAAM,EAAE;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,kBAAkB,EAAE,MAAM,CAAA;CAAC,GACrD,OAAO,CAAC,IAAI,CAAC,CAKf;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,WAAW,EACf,MAAM,EAAE;IACN,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,uBAAuB,GAAG,IAAI,CAAC;CAC/C,GACA,OAAO,CAAC,IAAI,CAAC,CAWf;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,WAAW,EACf,MAAM,EAAE;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,iBAAiB,CAAC;IAAC,aAAa,EAAE,OAAO,CAAA;CAAC,GAC5E,OAAO,CAAC;IAAC,MAAM,EAAE,aAAa,CAAC;IAAC,mBAAmB,EAAE,uBAAuB,GAAG,IAAI,CAAA;CAAC,GAAG,IAAI,CAAC,CA2B9F;AAED,0FAA0F;AAC1F,wBAAsB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAOlF;AAED,sFAAsF;AACtF,wBAAsB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAG1F;AAED,wBAAsB,iCAAiC,CACrD,EAAE,EAAE,WAAW,EACf,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAG/B;AAED;;;;;;;;GAQG;AACH,wBAAsB,2BAA2B,CAC/C,EAAE,EAAE,WAAW,EACf,MAAM,EAAE;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAC,GAC5C,OAAO,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAC,CAAC,CAiB7B;AAED;;;;;GAKG;AACH,wBAAsB,2BAA2B,CAAC,MAAM,EAAE;IACxD,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAe3B;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,EAAE;IACjD,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAiB3B;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC,CAO1D;AAED;;;;;;GAMG;AACH,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;CACnC,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAkB3B;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CACvC,EAAE,EAAE,WAAW,EACf,MAAM,EAAE;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAA;CAAC,GAC3D,OAAO,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;CAAC,CAAC,CAcnD;AAED,wBAAsB,sBAAsB,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAQ7F"}
|
package/dist/db/streams.js
CHANGED
|
@@ -87,23 +87,38 @@ export async function setDeclaredTotalBytes(tx, params) {
|
|
|
87
87
|
updatedAt: sql`now()`
|
|
88
88
|
}).where(eq(attemptStreams.id, params.streamId));
|
|
89
89
|
}
|
|
90
|
+
export async function setClaudeParseContext(tx, params) {
|
|
91
|
+
await tx.update(attemptStreams).set({
|
|
92
|
+
claudeHasInit: params.hasInit,
|
|
93
|
+
claudeSessionId: params.sessionId,
|
|
94
|
+
claudeTurn: params.turn,
|
|
95
|
+
claudePendingResult: params.pendingResult,
|
|
96
|
+
updatedAt: sql`now()`
|
|
97
|
+
}).where(eq(attemptStreams.id, params.streamId));
|
|
98
|
+
}
|
|
90
99
|
/**
|
|
91
|
-
* Guarded close: flips `open → closed
|
|
92
|
-
*
|
|
93
|
-
* it
|
|
94
|
-
*
|
|
95
|
-
*
|
|
100
|
+
* Guarded close: locks the open row, flips `open → closed`, clears any pending Claude result,
|
|
101
|
+
* and stamps `closed_at`. The pending result is returned separately so the caller can
|
|
102
|
+
* materialize it before publishing the close event. Returns null when another path already
|
|
103
|
+
* closed the stream. `committed_length` is left untouched; it stays equal to the runner spool
|
|
104
|
+
* bytes. `truncated` is set only on the timeout path.
|
|
96
105
|
*/ export async function markStreamClosed(tx, params) {
|
|
106
|
+
const [open] = await tx.select().from(attemptStreams).where(and(eq(attemptStreams.id, params.streamId), eq(attemptStreams.state, 'open'))).for('update');
|
|
107
|
+
if (!open) return null;
|
|
97
108
|
const [row] = await tx.update(attemptStreams).set({
|
|
98
109
|
state: 'closed',
|
|
99
110
|
closeReason: params.reason,
|
|
100
111
|
closedAt: sql`now()`,
|
|
101
112
|
updatedAt: sql`now()`,
|
|
113
|
+
claudePendingResult: null,
|
|
102
114
|
...params.markTruncated ? {
|
|
103
115
|
truncated: true
|
|
104
116
|
} : {}
|
|
105
117
|
}).where(and(eq(attemptStreams.id, params.streamId), eq(attemptStreams.state, 'open'))).returning();
|
|
106
|
-
return row ?
|
|
118
|
+
return row ? {
|
|
119
|
+
stream: toAttemptStream(row),
|
|
120
|
+
pendingClaudeResult: open.claudePendingResult ?? null
|
|
121
|
+
} : null;
|
|
107
122
|
}
|
|
108
123
|
/** Open streams of a job, for the timeout sweep to force-close after the grace period. */ export async function listOpenStreamsByJob(jobId) {
|
|
109
124
|
const rows = await db().select().from(attemptStreams).where(and(eq(attemptStreams.jobId, jobId), eq(attemptStreams.state, 'open')));
|
package/dist/db/streams.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/db/streams.ts"],"sourcesContent":["import {and, asc, eq, getTableColumns, isNull, lt, notInArray, sql} from 'drizzle-orm';\nimport type {AttemptStream, StreamCloseReason} from '#core/entities/attempt-stream.js';\nimport {LeaseStreamMismatchError} from '#core/errors.js';\nimport {db, type Transaction} from './db.js';\nimport {attemptStreams, toAttemptStream} from './schema/attempt-streams.js';\nimport {logChunks} from './schema/chunks.js';\n\nexport interface AttemptStreamIdentity {\n jobId: string;\n stepId: string;\n attempt: number;\n workspaceId: string;\n projectId: string;\n workflowRunAttemptId: string;\n}\n\nexport interface AttemptStreamLookup {\n jobId: string;\n stepId: string;\n attempt: number;\n}\n\n/**\n * Reads the stream for `(job, step, attempt)` without creating it. Used by the\n * empty-body heartbeat path so a runner cannot mint rows with empty appends.\n */\nexport async function getAttemptStream(\n tx: Transaction,\n lookup: AttemptStreamLookup,\n): Promise<AttemptStream | null> {\n const [row] = await tx\n .select()\n .from(attemptStreams)\n .where(\n and(\n eq(attemptStreams.jobId, lookup.jobId),\n eq(attemptStreams.stepId, lookup.stepId),\n eq(attemptStreams.attempt, lookup.attempt),\n ),\n );\n return row ? toAttemptStream(row) : null;\n}\n\n/**\n * Reads the stream for `(step, attempt)` for the session read path, which holds no lease\n * and so has no `jobId`. `stepId` is a globally unique uuidv7, so `(stepId, attempt)`\n * selects at most one row even though stream identity is `(jobId, stepId, attempt)`. The\n * caller authorizes against the row's denormalized `workspaceId`.\n */\nexport async function getStreamByStepAttempt(lookup: {\n stepId: string;\n attempt: number;\n}): Promise<AttemptStream | null> {\n const [row] = await db()\n .select()\n .from(attemptStreams)\n .where(\n and(eq(attemptStreams.stepId, lookup.stepId), eq(attemptStreams.attempt, lookup.attempt)),\n );\n return row ? toAttemptStream(row) : null;\n}\n\nexport interface GetOrCreateAttemptStreamResult {\n stream: AttemptStream;\n created: boolean;\n}\n\n/**\n * Loads the stream for `(job, step, attempt)`, creating it on first append.\n * Scoped by `jobId` from the lease, so a lease never reaches another job's stream.\n * The create path reports whether the row was inserted so callers can record\n * first-open side effects only after their enclosing transaction commits.\n */\nexport async function getOrCreateAttemptStreamWithStatus(\n tx: Transaction,\n identity: AttemptStreamIdentity,\n): Promise<GetOrCreateAttemptStreamResult> {\n const [row] = await tx\n .insert(attemptStreams)\n .values(identity)\n .onConflictDoUpdate({\n target: [attemptStreams.jobId, attemptStreams.stepId, attemptStreams.attempt],\n set: {updatedAt: sql`now()`},\n })\n // Postgres sets `xmax` only on the conflict-update path, so this keeps\n // first-open detection in the single upsert round trip.\n .returning({...getTableColumns(attemptStreams), created: sql<boolean>`xmax = 0`});\n\n if (!row) throw new Error('attempt stream missing after get-or-create');\n if (\n row.workspaceId !== identity.workspaceId ||\n row.projectId !== identity.projectId ||\n row.workflowRunAttemptId !== identity.workflowRunAttemptId\n ) {\n throw new LeaseStreamMismatchError();\n }\n return {stream: toAttemptStream(row), created: row.created};\n}\n\nexport async function getOrCreateAttemptStream(\n tx: Transaction,\n identity: AttemptStreamIdentity,\n): Promise<AttemptStream> {\n const result = await getOrCreateAttemptStreamWithStatus(tx, identity);\n return result.stream;\n}\n\nexport type CasOutcome = 'extended' | 'retry' | 'gap';\n\nexport interface CasResult {\n outcome: CasOutcome;\n committedLength: number;\n}\n\n/**\n * Offset-CAS extend: advances `committed_length` by `byteLen` iff it currently\n * equals `offset`. On mismatch, re-reads the committed length and classifies the\n * caller's chunk: a retry (fully already applied) only when `offset + byteLen`\n * does not run past the committed length; otherwise a gap. A chunk that straddles\n * the committed point (offset behind it, but extending past it) is a gap too, so\n * the runner rewinds rather than getting a 200 that silently drops its tail.\n */\nexport async function casExtendCommittedLength(\n tx: Transaction,\n params: {streamId: string; offset: number; byteLen: number},\n): Promise<CasResult> {\n const extended = await tx\n .update(attemptStreams)\n .set({\n committedLength: sql`${attemptStreams.committedLength} + ${params.byteLen}`,\n updatedAt: sql`now()`,\n })\n .where(\n and(\n eq(attemptStreams.id, params.streamId),\n eq(attemptStreams.committedLength, params.offset),\n ),\n )\n .returning({committedLength: attemptStreams.committedLength});\n\n if (extended[0]) return {outcome: 'extended', committedLength: extended[0].committedLength};\n\n const [current] = await tx\n .select({committedLength: attemptStreams.committedLength})\n .from(attemptStreams)\n .where(eq(attemptStreams.id, params.streamId));\n const committedLength = current?.committedLength ?? 0;\n\n const fullyApplied = params.offset + params.byteLen <= committedLength;\n return {outcome: fullyApplied ? 'retry' : 'gap', committedLength};\n}\n\nexport async function setDeclaredTotalBytes(\n tx: Transaction,\n params: {streamId: string; declaredTotalBytes: number},\n): Promise<void> {\n await tx\n .update(attemptStreams)\n .set({declaredTotalBytes: params.declaredTotalBytes, updatedAt: sql`now()`})\n .where(eq(attemptStreams.id, params.streamId));\n}\n\n/**\n * Guarded close: flips `open → closed` and stamps `closed_at` iff the stream is\n * still open, returning the closed row (or null when another path already closed\n * it). The `WHERE state='open'` predicate is both the row lock and the\n * exactly-once gate. `committed_length` is left untouched; it stays equal to the\n * runner spool bytes. `truncated` is set only on the timeout path.\n */\nexport async function markStreamClosed(\n tx: Transaction,\n params: {streamId: string; reason: StreamCloseReason; markTruncated: boolean},\n): Promise<AttemptStream | null> {\n const [row] = await tx\n .update(attemptStreams)\n .set({\n state: 'closed',\n closeReason: params.reason,\n closedAt: sql`now()`,\n updatedAt: sql`now()`,\n ...(params.markTruncated ? {truncated: true} : {}),\n })\n .where(and(eq(attemptStreams.id, params.streamId), eq(attemptStreams.state, 'open')))\n .returning();\n\n return row ? toAttemptStream(row) : null;\n}\n\n/** Open streams of a job, for the timeout sweep to force-close after the grace period. */\nexport async function listOpenStreamsByJob(jobId: string): Promise<AttemptStream[]> {\n const rows = await db()\n .select()\n .from(attemptStreams)\n .where(and(eq(attemptStreams.jobId, jobId), eq(attemptStreams.state, 'open')));\n\n return rows.map(toAttemptStream);\n}\n\n/** For compaction, which is driven by the closed event that carries the stream id. */\nexport async function getAttemptStreamById(streamId: string): Promise<AttemptStream | null> {\n const [row] = await db().select().from(attemptStreams).where(eq(attemptStreams.id, streamId));\n return row ? toAttemptStream(row) : null;\n}\n\nexport async function getAttemptStreamByIdInTransaction(\n tx: Transaction,\n streamId: string,\n): Promise<AttemptStream | null> {\n const [row] = await tx.select().from(attemptStreams).where(eq(attemptStreams.id, streamId));\n return row ? toAttemptStream(row) : null;\n}\n\n/**\n * Final compaction step: records the object key and deletes the now-cold chunk rows in one\n * transaction. The guard is `state='closed' AND object_key IS NULL`, so the publish is a\n * single winner even when two compaction attempts (e.g. a heartbeat-timed-out run and its\n * retry) race: each uploads to its own key, and only the first to land here writes a key and\n * drops the chunks. A 0-row result means the row was either hard-deleted by retention or\n * already published by another attempt; the caller re-reads to tell those apart and deletes\n * its own now-orphaned upload either way. Returns whether this attempt won the publish.\n */\nexport async function setObjectKeyAndDeleteChunks(\n tx: Transaction,\n params: {streamId: string; objectKey: string},\n): Promise<{updated: boolean}> {\n const updated = await tx\n .update(attemptStreams)\n .set({objectKey: params.objectKey, updatedAt: sql`now()`})\n .where(\n and(\n eq(attemptStreams.id, params.streamId),\n eq(attemptStreams.state, 'closed'),\n isNull(attemptStreams.objectKey),\n ),\n )\n .returning({id: attemptStreams.id});\n\n if (updated.length === 0) return {updated: false};\n\n await tx.delete(logChunks).where(eq(logChunks.streamId, params.streamId));\n return {updated: true};\n}\n\n/**\n * Closed streams that never got an object key and have sat that way past the stale\n * window: the reconcile cron re-drives compaction for these. Hits the partial index\n * `logs_attempt_streams_uncompacted_idx`; ordered by `closed_at` so the oldest backlog\n * drains first, bounded by `limit` per tick.\n */\nexport async function listStaleUncompactedStreams(params: {\n olderThanSeconds: number;\n limit: number;\n}): Promise<AttemptStream[]> {\n const rows = await db()\n .select()\n .from(attemptStreams)\n .where(\n and(\n eq(attemptStreams.state, 'closed'),\n isNull(attemptStreams.objectKey),\n lt(attemptStreams.closedAt, sql`now() - make_interval(secs => ${params.olderThanSeconds})`),\n ),\n )\n .orderBy(asc(attemptStreams.closedAt))\n .limit(params.limit);\n\n return rows.map(toAttemptStream);\n}\n\n/**\n * Lists open streams old enough to force-close. The age check is safe only when\n * `olderThanSeconds` exceeds the lease TTL: leases are minted before a stream can\n * be created, so no valid append can arrive after `created_at + leaseTTL`.\n */\nexport async function listStaleOpenStreams(params: {\n olderThanSeconds: number;\n limit: number;\n}): Promise<AttemptStream[]> {\n const rows = await db()\n .select()\n .from(attemptStreams)\n .where(\n and(\n eq(attemptStreams.state, 'open'),\n lt(\n attemptStreams.createdAt,\n sql`now() - make_interval(secs => ${params.olderThanSeconds})`,\n ),\n ),\n )\n .orderBy(asc(attemptStreams.createdAt))\n .limit(params.limit);\n\n return rows.map(toAttemptStream);\n}\n\nexport async function getOpenStreamCount(): Promise<bigint> {\n const [row] = await db()\n .select({value: sql<bigint>`count(*)`.mapWith(BigInt)})\n .from(attemptStreams)\n .where(eq(attemptStreams.state, 'open'));\n\n return row?.value ?? 0n;\n}\n\n/**\n * Lists expired closed streams for retention. `excludeIds` keeps failed or raced rows from\n * starving younger rows in the same run, and avoids cursoring on `closed_at`, whose microsecond\n * precision would be lost through JS `Date`.\n *\n * No `object_key` filter: retention deletes compacted and never-compacted streams alike.\n */\nexport async function listExpiredClosedStreams(params: {\n retentionDays: number;\n limit: number;\n excludeIds?: string[] | undefined;\n}): Promise<AttemptStream[]> {\n const rows = await db()\n .select()\n .from(attemptStreams)\n .where(\n and(\n eq(attemptStreams.state, 'closed'),\n lt(attemptStreams.closedAt, sql`now() - make_interval(days => ${params.retentionDays})`),\n params.excludeIds && params.excludeIds.length > 0\n ? notInArray(attemptStreams.id, params.excludeIds)\n : undefined,\n ),\n )\n .orderBy(asc(attemptStreams.closedAt), asc(attemptStreams.id))\n .limit(params.limit)\n .for('update', {skipLocked: true});\n\n return rows.map(toAttemptStream);\n}\n\n/**\n * Hard-deletes one expired stream row, guarded on the observed `object_key` so a racing\n * compaction publish leaves the row for the next sweep.\n */\nexport async function deleteExpiredStream(\n tx: Transaction,\n params: {streamId: string; observedObjectKey: string | null},\n): Promise<{deleted: boolean; jobId: string | null}> {\n const [row] = await tx\n .delete(attemptStreams)\n .where(\n and(\n eq(attemptStreams.id, params.streamId),\n params.observedObjectKey === null\n ? isNull(attemptStreams.objectKey)\n : eq(attemptStreams.objectKey, params.observedObjectKey),\n ),\n )\n .returning({jobId: attemptStreams.jobId});\n\n return {deleted: Boolean(row), jobId: row?.jobId ?? null};\n}\n\nexport async function accountingHasNoStreams(tx: Transaction, jobId: string): Promise<boolean> {\n const [row] = await tx\n .select({id: attemptStreams.id})\n .from(attemptStreams)\n .where(eq(attemptStreams.jobId, jobId))\n .limit(1);\n\n return !row;\n}\n"],"names":["and","asc","eq","getTableColumns","isNull","lt","notInArray","sql","LeaseStreamMismatchError","db","attemptStreams","toAttemptStream","logChunks","getAttemptStream","tx","lookup","row","select","from","where","jobId","stepId","attempt","getStreamByStepAttempt","getOrCreateAttemptStreamWithStatus","identity","insert","values","onConflictDoUpdate","target","set","updatedAt","returning","created","Error","workspaceId","projectId","workflowRunAttemptId","stream","getOrCreateAttemptStream","result","casExtendCommittedLength","params","extended","update","committedLength","byteLen","id","streamId","offset","outcome","current","fullyApplied","setDeclaredTotalBytes","declaredTotalBytes","markStreamClosed","state","closeReason","reason","closedAt","markTruncated","truncated","listOpenStreamsByJob","rows","map","getAttemptStreamById","getAttemptStreamByIdInTransaction","setObjectKeyAndDeleteChunks","updated","objectKey","length","delete","listStaleUncompactedStreams","olderThanSeconds","orderBy","limit","listStaleOpenStreams","createdAt","getOpenStreamCount","value","mapWith","BigInt","listExpiredClosedStreams","retentionDays","excludeIds","undefined","for","skipLocked","deleteExpiredStream","observedObjectKey","deleted","Boolean","accountingHasNoStreams"],"mappings":"AAAA,SAAQA,GAAG,EAAEC,GAAG,EAAEC,EAAE,EAAEC,eAAe,EAAEC,MAAM,EAAEC,EAAE,EAAEC,UAAU,EAAEC,GAAG,QAAO,cAAc;AAEvF,SAAQC,wBAAwB,QAAO,kBAAkB;AACzD,SAAQC,EAAE,QAAyB,UAAU;AAC7C,SAAQC,cAAc,EAAEC,eAAe,QAAO,8BAA8B;AAC5E,SAAQC,SAAS,QAAO,qBAAqB;AAiB7C;;;CAGC,GACD,OAAO,eAAeC,iBACpBC,EAAe,EACfC,MAA2B;IAE3B,MAAM,CAACC,IAAI,GAAG,MAAMF,GACjBG,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CACJnB,IACEE,GAAGQ,eAAeU,KAAK,EAAEL,OAAOK,KAAK,GACrClB,GAAGQ,eAAeW,MAAM,EAAEN,OAAOM,MAAM,GACvCnB,GAAGQ,eAAeY,OAAO,EAAEP,OAAOO,OAAO;IAG/C,OAAON,MAAML,gBAAgBK,OAAO;AACtC;AAEA;;;;;CAKC,GACD,OAAO,eAAeO,uBAAuBR,MAG5C;IACC,MAAM,CAACC,IAAI,GAAG,MAAMP,KACjBQ,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CACJnB,IAAIE,GAAGQ,eAAeW,MAAM,EAAEN,OAAOM,MAAM,GAAGnB,GAAGQ,eAAeY,OAAO,EAAEP,OAAOO,OAAO;IAE3F,OAAON,MAAML,gBAAgBK,OAAO;AACtC;AAOA;;;;;CAKC,GACD,OAAO,eAAeQ,mCACpBV,EAAe,EACfW,QAA+B;IAE/B,MAAM,CAACT,IAAI,GAAG,MAAMF,GACjBY,MAAM,CAAChB,gBACPiB,MAAM,CAACF,UACPG,kBAAkB,CAAC;QAClBC,QAAQ;YAACnB,eAAeU,KAAK;YAAEV,eAAeW,MAAM;YAAEX,eAAeY,OAAO;SAAC;QAC7EQ,KAAK;YAACC,WAAWxB,GAAG,CAAC,KAAK,CAAC;QAAA;IAC7B,EACA,uEAAuE;IACvE,wDAAwD;KACvDyB,SAAS,CAAC;QAAC,GAAG7B,gBAAgBO,eAAe;QAAEuB,SAAS1B,GAAY,CAAC,QAAQ,CAAC;IAAA;IAEjF,IAAI,CAACS,KAAK,MAAM,IAAIkB,MAAM;IAC1B,IACElB,IAAImB,WAAW,KAAKV,SAASU,WAAW,IACxCnB,IAAIoB,SAAS,KAAKX,SAASW,SAAS,IACpCpB,IAAIqB,oBAAoB,KAAKZ,SAASY,oBAAoB,EAC1D;QACA,MAAM,IAAI7B;IACZ;IACA,OAAO;QAAC8B,QAAQ3B,gBAAgBK;QAAMiB,SAASjB,IAAIiB,OAAO;IAAA;AAC5D;AAEA,OAAO,eAAeM,yBACpBzB,EAAe,EACfW,QAA+B;IAE/B,MAAMe,SAAS,MAAMhB,mCAAmCV,IAAIW;IAC5D,OAAOe,OAAOF,MAAM;AACtB;AASA;;;;;;;CAOC,GACD,OAAO,eAAeG,yBACpB3B,EAAe,EACf4B,MAA2D;IAE3D,MAAMC,WAAW,MAAM7B,GACpB8B,MAAM,CAAClC,gBACPoB,GAAG,CAAC;QACHe,iBAAiBtC,GAAG,CAAC,EAAEG,eAAemC,eAAe,CAAC,GAAG,EAAEH,OAAOI,OAAO,CAAC,CAAC;QAC3Ef,WAAWxB,GAAG,CAAC,KAAK,CAAC;IACvB,GACCY,KAAK,CACJnB,IACEE,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ,GACrC9C,GAAGQ,eAAemC,eAAe,EAAEH,OAAOO,MAAM,IAGnDjB,SAAS,CAAC;QAACa,iBAAiBnC,eAAemC,eAAe;IAAA;IAE7D,IAAIF,QAAQ,CAAC,EAAE,EAAE,OAAO;QAACO,SAAS;QAAYL,iBAAiBF,QAAQ,CAAC,EAAE,CAACE,eAAe;IAAA;IAE1F,MAAM,CAACM,QAAQ,GAAG,MAAMrC,GACrBG,MAAM,CAAC;QAAC4B,iBAAiBnC,eAAemC,eAAe;IAAA,GACvD3B,IAAI,CAACR,gBACLS,KAAK,CAACjB,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ;IAC9C,MAAMH,kBAAkBM,SAASN,mBAAmB;IAEpD,MAAMO,eAAeV,OAAOO,MAAM,GAAGP,OAAOI,OAAO,IAAID;IACvD,OAAO;QAACK,SAASE,eAAe,UAAU;QAAOP;IAAe;AAClE;AAEA,OAAO,eAAeQ,sBACpBvC,EAAe,EACf4B,MAAsD;IAEtD,MAAM5B,GACH8B,MAAM,CAAClC,gBACPoB,GAAG,CAAC;QAACwB,oBAAoBZ,OAAOY,kBAAkB;QAAEvB,WAAWxB,GAAG,CAAC,KAAK,CAAC;IAAA,GACzEY,KAAK,CAACjB,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ;AAChD;AAEA;;;;;;CAMC,GACD,OAAO,eAAeO,iBACpBzC,EAAe,EACf4B,MAA6E;IAE7E,MAAM,CAAC1B,IAAI,GAAG,MAAMF,GACjB8B,MAAM,CAAClC,gBACPoB,GAAG,CAAC;QACH0B,OAAO;QACPC,aAAaf,OAAOgB,MAAM;QAC1BC,UAAUpD,GAAG,CAAC,KAAK,CAAC;QACpBwB,WAAWxB,GAAG,CAAC,KAAK,CAAC;QACrB,GAAImC,OAAOkB,aAAa,GAAG;YAACC,WAAW;QAAI,IAAI,CAAC,CAAC;IACnD,GACC1C,KAAK,CAACnB,IAAIE,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ,GAAG9C,GAAGQ,eAAe8C,KAAK,EAAE,UAC3ExB,SAAS;IAEZ,OAAOhB,MAAML,gBAAgBK,OAAO;AACtC;AAEA,wFAAwF,GACxF,OAAO,eAAe8C,qBAAqB1C,KAAa;IACtD,MAAM2C,OAAO,MAAMtD,KAChBQ,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CAACnB,IAAIE,GAAGQ,eAAeU,KAAK,EAAEA,QAAQlB,GAAGQ,eAAe8C,KAAK,EAAE;IAEvE,OAAOO,KAAKC,GAAG,CAACrD;AAClB;AAEA,oFAAoF,GACpF,OAAO,eAAesD,qBAAqBjB,QAAgB;IACzD,MAAM,CAAChC,IAAI,GAAG,MAAMP,KAAKQ,MAAM,GAAGC,IAAI,CAACR,gBAAgBS,KAAK,CAACjB,GAAGQ,eAAeqC,EAAE,EAAEC;IACnF,OAAOhC,MAAML,gBAAgBK,OAAO;AACtC;AAEA,OAAO,eAAekD,kCACpBpD,EAAe,EACfkC,QAAgB;IAEhB,MAAM,CAAChC,IAAI,GAAG,MAAMF,GAAGG,MAAM,GAAGC,IAAI,CAACR,gBAAgBS,KAAK,CAACjB,GAAGQ,eAAeqC,EAAE,EAAEC;IACjF,OAAOhC,MAAML,gBAAgBK,OAAO;AACtC;AAEA;;;;;;;;CAQC,GACD,OAAO,eAAemD,4BACpBrD,EAAe,EACf4B,MAA6C;IAE7C,MAAM0B,UAAU,MAAMtD,GACnB8B,MAAM,CAAClC,gBACPoB,GAAG,CAAC;QAACuC,WAAW3B,OAAO2B,SAAS;QAAEtC,WAAWxB,GAAG,CAAC,KAAK,CAAC;IAAA,GACvDY,KAAK,CACJnB,IACEE,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ,GACrC9C,GAAGQ,eAAe8C,KAAK,EAAE,WACzBpD,OAAOM,eAAe2D,SAAS,IAGlCrC,SAAS,CAAC;QAACe,IAAIrC,eAAeqC,EAAE;IAAA;IAEnC,IAAIqB,QAAQE,MAAM,KAAK,GAAG,OAAO;QAACF,SAAS;IAAK;IAEhD,MAAMtD,GAAGyD,MAAM,CAAC3D,WAAWO,KAAK,CAACjB,GAAGU,UAAUoC,QAAQ,EAAEN,OAAOM,QAAQ;IACvE,OAAO;QAACoB,SAAS;IAAI;AACvB;AAEA;;;;;CAKC,GACD,OAAO,eAAeI,4BAA4B9B,MAGjD;IACC,MAAMqB,OAAO,MAAMtD,KAChBQ,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CACJnB,IACEE,GAAGQ,eAAe8C,KAAK,EAAE,WACzBpD,OAAOM,eAAe2D,SAAS,GAC/BhE,GAAGK,eAAeiD,QAAQ,EAAEpD,GAAG,CAAC,8BAA8B,EAAEmC,OAAO+B,gBAAgB,CAAC,CAAC,CAAC,IAG7FC,OAAO,CAACzE,IAAIS,eAAeiD,QAAQ,GACnCgB,KAAK,CAACjC,OAAOiC,KAAK;IAErB,OAAOZ,KAAKC,GAAG,CAACrD;AAClB;AAEA;;;;CAIC,GACD,OAAO,eAAeiE,qBAAqBlC,MAG1C;IACC,MAAMqB,OAAO,MAAMtD,KAChBQ,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CACJnB,IACEE,GAAGQ,eAAe8C,KAAK,EAAE,SACzBnD,GACEK,eAAemE,SAAS,EACxBtE,GAAG,CAAC,8BAA8B,EAAEmC,OAAO+B,gBAAgB,CAAC,CAAC,CAAC,IAInEC,OAAO,CAACzE,IAAIS,eAAemE,SAAS,GACpCF,KAAK,CAACjC,OAAOiC,KAAK;IAErB,OAAOZ,KAAKC,GAAG,CAACrD;AAClB;AAEA,OAAO,eAAemE;IACpB,MAAM,CAAC9D,IAAI,GAAG,MAAMP,KACjBQ,MAAM,CAAC;QAAC8D,OAAOxE,GAAW,CAAC,QAAQ,CAAC,CAACyE,OAAO,CAACC;IAAO,GACpD/D,IAAI,CAACR,gBACLS,KAAK,CAACjB,GAAGQ,eAAe8C,KAAK,EAAE;IAElC,OAAOxC,KAAK+D,SAAS,EAAE;AACzB;AAEA;;;;;;CAMC,GACD,OAAO,eAAeG,yBAAyBxC,MAI9C;IACC,MAAMqB,OAAO,MAAMtD,KAChBQ,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CACJnB,IACEE,GAAGQ,eAAe8C,KAAK,EAAE,WACzBnD,GAAGK,eAAeiD,QAAQ,EAAEpD,GAAG,CAAC,8BAA8B,EAAEmC,OAAOyC,aAAa,CAAC,CAAC,CAAC,GACvFzC,OAAO0C,UAAU,IAAI1C,OAAO0C,UAAU,CAACd,MAAM,GAAG,IAC5ChE,WAAWI,eAAeqC,EAAE,EAAEL,OAAO0C,UAAU,IAC/CC,YAGPX,OAAO,CAACzE,IAAIS,eAAeiD,QAAQ,GAAG1D,IAAIS,eAAeqC,EAAE,GAC3D4B,KAAK,CAACjC,OAAOiC,KAAK,EAClBW,GAAG,CAAC,UAAU;QAACC,YAAY;IAAI;IAElC,OAAOxB,KAAKC,GAAG,CAACrD;AAClB;AAEA;;;CAGC,GACD,OAAO,eAAe6E,oBACpB1E,EAAe,EACf4B,MAA4D;IAE5D,MAAM,CAAC1B,IAAI,GAAG,MAAMF,GACjByD,MAAM,CAAC7D,gBACPS,KAAK,CACJnB,IACEE,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ,GACrCN,OAAO+C,iBAAiB,KAAK,OACzBrF,OAAOM,eAAe2D,SAAS,IAC/BnE,GAAGQ,eAAe2D,SAAS,EAAE3B,OAAO+C,iBAAiB,IAG5DzD,SAAS,CAAC;QAACZ,OAAOV,eAAeU,KAAK;IAAA;IAEzC,OAAO;QAACsE,SAASC,QAAQ3E;QAAMI,OAAOJ,KAAKI,SAAS;IAAI;AAC1D;AAEA,OAAO,eAAewE,uBAAuB9E,EAAe,EAAEM,KAAa;IACzE,MAAM,CAACJ,IAAI,GAAG,MAAMF,GACjBG,MAAM,CAAC;QAAC8B,IAAIrC,eAAeqC,EAAE;IAAA,GAC7B7B,IAAI,CAACR,gBACLS,KAAK,CAACjB,GAAGQ,eAAeU,KAAK,EAAEA,QAC/BuD,KAAK,CAAC;IAET,OAAO,CAAC3D;AACV"}
|
|
1
|
+
{"version":3,"sources":["../../src/db/streams.ts"],"sourcesContent":["import type {SessionViewLifecycleRow} from '@shipfox/api-logs-dto';\nimport {and, asc, eq, getTableColumns, isNull, lt, notInArray, sql} from 'drizzle-orm';\nimport type {AttemptStream, StreamCloseReason} from '#core/entities/attempt-stream.js';\nimport {LeaseStreamMismatchError} from '#core/errors.js';\nimport {db, type Transaction} from './db.js';\nimport {attemptStreams, toAttemptStream} from './schema/attempt-streams.js';\nimport {logChunks} from './schema/chunks.js';\n\nexport interface AttemptStreamIdentity {\n jobId: string;\n stepId: string;\n attempt: number;\n workspaceId: string;\n projectId: string;\n workflowRunAttemptId: string;\n}\n\nexport interface AttemptStreamLookup {\n jobId: string;\n stepId: string;\n attempt: number;\n}\n\n/**\n * Reads the stream for `(job, step, attempt)` without creating it. Used by the\n * empty-body heartbeat path so a runner cannot mint rows with empty appends.\n */\nexport async function getAttemptStream(\n tx: Transaction,\n lookup: AttemptStreamLookup,\n): Promise<AttemptStream | null> {\n const [row] = await tx\n .select()\n .from(attemptStreams)\n .where(\n and(\n eq(attemptStreams.jobId, lookup.jobId),\n eq(attemptStreams.stepId, lookup.stepId),\n eq(attemptStreams.attempt, lookup.attempt),\n ),\n );\n return row ? toAttemptStream(row) : null;\n}\n\n/**\n * Reads the stream for `(step, attempt)` for the session read path, which holds no lease\n * and so has no `jobId`. `stepId` is a globally unique uuidv7, so `(stepId, attempt)`\n * selects at most one row even though stream identity is `(jobId, stepId, attempt)`. The\n * caller authorizes against the row's denormalized `workspaceId`.\n */\nexport async function getStreamByStepAttempt(lookup: {\n stepId: string;\n attempt: number;\n}): Promise<AttemptStream | null> {\n const [row] = await db()\n .select()\n .from(attemptStreams)\n .where(\n and(eq(attemptStreams.stepId, lookup.stepId), eq(attemptStreams.attempt, lookup.attempt)),\n );\n return row ? toAttemptStream(row) : null;\n}\n\nexport interface GetOrCreateAttemptStreamResult {\n stream: AttemptStream;\n created: boolean;\n}\n\n/**\n * Loads the stream for `(job, step, attempt)`, creating it on first append.\n * Scoped by `jobId` from the lease, so a lease never reaches another job's stream.\n * The create path reports whether the row was inserted so callers can record\n * first-open side effects only after their enclosing transaction commits.\n */\nexport async function getOrCreateAttemptStreamWithStatus(\n tx: Transaction,\n identity: AttemptStreamIdentity,\n): Promise<GetOrCreateAttemptStreamResult> {\n const [row] = await tx\n .insert(attemptStreams)\n .values(identity)\n .onConflictDoUpdate({\n target: [attemptStreams.jobId, attemptStreams.stepId, attemptStreams.attempt],\n set: {updatedAt: sql`now()`},\n })\n // Postgres sets `xmax` only on the conflict-update path, so this keeps\n // first-open detection in the single upsert round trip.\n .returning({...getTableColumns(attemptStreams), created: sql<boolean>`xmax = 0`});\n\n if (!row) throw new Error('attempt stream missing after get-or-create');\n if (\n row.workspaceId !== identity.workspaceId ||\n row.projectId !== identity.projectId ||\n row.workflowRunAttemptId !== identity.workflowRunAttemptId\n ) {\n throw new LeaseStreamMismatchError();\n }\n return {stream: toAttemptStream(row), created: row.created};\n}\n\nexport async function getOrCreateAttemptStream(\n tx: Transaction,\n identity: AttemptStreamIdentity,\n): Promise<AttemptStream> {\n const result = await getOrCreateAttemptStreamWithStatus(tx, identity);\n return result.stream;\n}\n\nexport type CasOutcome = 'extended' | 'retry' | 'gap';\n\nexport interface CasResult {\n outcome: CasOutcome;\n committedLength: number;\n}\n\n/**\n * Offset-CAS extend: advances `committed_length` by `byteLen` iff it currently\n * equals `offset`. On mismatch, re-reads the committed length and classifies the\n * caller's chunk: a retry (fully already applied) only when `offset + byteLen`\n * does not run past the committed length; otherwise a gap. A chunk that straddles\n * the committed point (offset behind it, but extending past it) is a gap too, so\n * the runner rewinds rather than getting a 200 that silently drops its tail.\n */\nexport async function casExtendCommittedLength(\n tx: Transaction,\n params: {streamId: string; offset: number; byteLen: number},\n): Promise<CasResult> {\n const extended = await tx\n .update(attemptStreams)\n .set({\n committedLength: sql`${attemptStreams.committedLength} + ${params.byteLen}`,\n updatedAt: sql`now()`,\n })\n .where(\n and(\n eq(attemptStreams.id, params.streamId),\n eq(attemptStreams.committedLength, params.offset),\n ),\n )\n .returning({committedLength: attemptStreams.committedLength});\n\n if (extended[0]) return {outcome: 'extended', committedLength: extended[0].committedLength};\n\n const [current] = await tx\n .select({committedLength: attemptStreams.committedLength})\n .from(attemptStreams)\n .where(eq(attemptStreams.id, params.streamId));\n const committedLength = current?.committedLength ?? 0;\n\n const fullyApplied = params.offset + params.byteLen <= committedLength;\n return {outcome: fullyApplied ? 'retry' : 'gap', committedLength};\n}\n\nexport async function setDeclaredTotalBytes(\n tx: Transaction,\n params: {streamId: string; declaredTotalBytes: number},\n): Promise<void> {\n await tx\n .update(attemptStreams)\n .set({declaredTotalBytes: params.declaredTotalBytes, updatedAt: sql`now()`})\n .where(eq(attemptStreams.id, params.streamId));\n}\n\nexport async function setClaudeParseContext(\n tx: Transaction,\n params: {\n streamId: string;\n hasInit: boolean;\n sessionId: string | null;\n turn: number;\n pendingResult: SessionViewLifecycleRow | null;\n },\n): Promise<void> {\n await tx\n .update(attemptStreams)\n .set({\n claudeHasInit: params.hasInit,\n claudeSessionId: params.sessionId,\n claudeTurn: params.turn,\n claudePendingResult: params.pendingResult,\n updatedAt: sql`now()`,\n })\n .where(eq(attemptStreams.id, params.streamId));\n}\n\n/**\n * Guarded close: locks the open row, flips `open → closed`, clears any pending Claude result,\n * and stamps `closed_at`. The pending result is returned separately so the caller can\n * materialize it before publishing the close event. Returns null when another path already\n * closed the stream. `committed_length` is left untouched; it stays equal to the runner spool\n * bytes. `truncated` is set only on the timeout path.\n */\nexport async function markStreamClosed(\n tx: Transaction,\n params: {streamId: string; reason: StreamCloseReason; markTruncated: boolean},\n): Promise<{stream: AttemptStream; pendingClaudeResult: SessionViewLifecycleRow | null} | null> {\n const [open] = await tx\n .select()\n .from(attemptStreams)\n .where(and(eq(attemptStreams.id, params.streamId), eq(attemptStreams.state, 'open')))\n .for('update');\n if (!open) return null;\n\n const [row] = await tx\n .update(attemptStreams)\n .set({\n state: 'closed',\n closeReason: params.reason,\n closedAt: sql`now()`,\n updatedAt: sql`now()`,\n claudePendingResult: null,\n ...(params.markTruncated ? {truncated: true} : {}),\n })\n .where(and(eq(attemptStreams.id, params.streamId), eq(attemptStreams.state, 'open')))\n .returning();\n\n return row\n ? {\n stream: toAttemptStream(row),\n pendingClaudeResult: open.claudePendingResult ?? null,\n }\n : null;\n}\n\n/** Open streams of a job, for the timeout sweep to force-close after the grace period. */\nexport async function listOpenStreamsByJob(jobId: string): Promise<AttemptStream[]> {\n const rows = await db()\n .select()\n .from(attemptStreams)\n .where(and(eq(attemptStreams.jobId, jobId), eq(attemptStreams.state, 'open')));\n\n return rows.map(toAttemptStream);\n}\n\n/** For compaction, which is driven by the closed event that carries the stream id. */\nexport async function getAttemptStreamById(streamId: string): Promise<AttemptStream | null> {\n const [row] = await db().select().from(attemptStreams).where(eq(attemptStreams.id, streamId));\n return row ? toAttemptStream(row) : null;\n}\n\nexport async function getAttemptStreamByIdInTransaction(\n tx: Transaction,\n streamId: string,\n): Promise<AttemptStream | null> {\n const [row] = await tx.select().from(attemptStreams).where(eq(attemptStreams.id, streamId));\n return row ? toAttemptStream(row) : null;\n}\n\n/**\n * Final compaction step: records the object key and deletes the now-cold chunk rows in one\n * transaction. The guard is `state='closed' AND object_key IS NULL`, so the publish is a\n * single winner even when two compaction attempts (e.g. a heartbeat-timed-out run and its\n * retry) race: each uploads to its own key, and only the first to land here writes a key and\n * drops the chunks. A 0-row result means the row was either hard-deleted by retention or\n * already published by another attempt; the caller re-reads to tell those apart and deletes\n * its own now-orphaned upload either way. Returns whether this attempt won the publish.\n */\nexport async function setObjectKeyAndDeleteChunks(\n tx: Transaction,\n params: {streamId: string; objectKey: string},\n): Promise<{updated: boolean}> {\n const updated = await tx\n .update(attemptStreams)\n .set({objectKey: params.objectKey, updatedAt: sql`now()`})\n .where(\n and(\n eq(attemptStreams.id, params.streamId),\n eq(attemptStreams.state, 'closed'),\n isNull(attemptStreams.objectKey),\n ),\n )\n .returning({id: attemptStreams.id});\n\n if (updated.length === 0) return {updated: false};\n\n await tx.delete(logChunks).where(eq(logChunks.streamId, params.streamId));\n return {updated: true};\n}\n\n/**\n * Closed streams that never got an object key and have sat that way past the stale\n * window: the reconcile cron re-drives compaction for these. Hits the partial index\n * `logs_attempt_streams_uncompacted_idx`; ordered by `closed_at` so the oldest backlog\n * drains first, bounded by `limit` per tick.\n */\nexport async function listStaleUncompactedStreams(params: {\n olderThanSeconds: number;\n limit: number;\n}): Promise<AttemptStream[]> {\n const rows = await db()\n .select()\n .from(attemptStreams)\n .where(\n and(\n eq(attemptStreams.state, 'closed'),\n isNull(attemptStreams.objectKey),\n lt(attemptStreams.closedAt, sql`now() - make_interval(secs => ${params.olderThanSeconds})`),\n ),\n )\n .orderBy(asc(attemptStreams.closedAt))\n .limit(params.limit);\n\n return rows.map(toAttemptStream);\n}\n\n/**\n * Lists open streams old enough to force-close. The age check is safe only when\n * `olderThanSeconds` exceeds the lease TTL: leases are minted before a stream can\n * be created, so no valid append can arrive after `created_at + leaseTTL`.\n */\nexport async function listStaleOpenStreams(params: {\n olderThanSeconds: number;\n limit: number;\n}): Promise<AttemptStream[]> {\n const rows = await db()\n .select()\n .from(attemptStreams)\n .where(\n and(\n eq(attemptStreams.state, 'open'),\n lt(\n attemptStreams.createdAt,\n sql`now() - make_interval(secs => ${params.olderThanSeconds})`,\n ),\n ),\n )\n .orderBy(asc(attemptStreams.createdAt))\n .limit(params.limit);\n\n return rows.map(toAttemptStream);\n}\n\nexport async function getOpenStreamCount(): Promise<bigint> {\n const [row] = await db()\n .select({value: sql<bigint>`count(*)`.mapWith(BigInt)})\n .from(attemptStreams)\n .where(eq(attemptStreams.state, 'open'));\n\n return row?.value ?? 0n;\n}\n\n/**\n * Lists expired closed streams for retention. `excludeIds` keeps failed or raced rows from\n * starving younger rows in the same run, and avoids cursoring on `closed_at`, whose microsecond\n * precision would be lost through JS `Date`.\n *\n * No `object_key` filter: retention deletes compacted and never-compacted streams alike.\n */\nexport async function listExpiredClosedStreams(params: {\n retentionDays: number;\n limit: number;\n excludeIds?: string[] | undefined;\n}): Promise<AttemptStream[]> {\n const rows = await db()\n .select()\n .from(attemptStreams)\n .where(\n and(\n eq(attemptStreams.state, 'closed'),\n lt(attemptStreams.closedAt, sql`now() - make_interval(days => ${params.retentionDays})`),\n params.excludeIds && params.excludeIds.length > 0\n ? notInArray(attemptStreams.id, params.excludeIds)\n : undefined,\n ),\n )\n .orderBy(asc(attemptStreams.closedAt), asc(attemptStreams.id))\n .limit(params.limit)\n .for('update', {skipLocked: true});\n\n return rows.map(toAttemptStream);\n}\n\n/**\n * Hard-deletes one expired stream row, guarded on the observed `object_key` so a racing\n * compaction publish leaves the row for the next sweep.\n */\nexport async function deleteExpiredStream(\n tx: Transaction,\n params: {streamId: string; observedObjectKey: string | null},\n): Promise<{deleted: boolean; jobId: string | null}> {\n const [row] = await tx\n .delete(attemptStreams)\n .where(\n and(\n eq(attemptStreams.id, params.streamId),\n params.observedObjectKey === null\n ? isNull(attemptStreams.objectKey)\n : eq(attemptStreams.objectKey, params.observedObjectKey),\n ),\n )\n .returning({jobId: attemptStreams.jobId});\n\n return {deleted: Boolean(row), jobId: row?.jobId ?? null};\n}\n\nexport async function accountingHasNoStreams(tx: Transaction, jobId: string): Promise<boolean> {\n const [row] = await tx\n .select({id: attemptStreams.id})\n .from(attemptStreams)\n .where(eq(attemptStreams.jobId, jobId))\n .limit(1);\n\n return !row;\n}\n"],"names":["and","asc","eq","getTableColumns","isNull","lt","notInArray","sql","LeaseStreamMismatchError","db","attemptStreams","toAttemptStream","logChunks","getAttemptStream","tx","lookup","row","select","from","where","jobId","stepId","attempt","getStreamByStepAttempt","getOrCreateAttemptStreamWithStatus","identity","insert","values","onConflictDoUpdate","target","set","updatedAt","returning","created","Error","workspaceId","projectId","workflowRunAttemptId","stream","getOrCreateAttemptStream","result","casExtendCommittedLength","params","extended","update","committedLength","byteLen","id","streamId","offset","outcome","current","fullyApplied","setDeclaredTotalBytes","declaredTotalBytes","setClaudeParseContext","claudeHasInit","hasInit","claudeSessionId","sessionId","claudeTurn","turn","claudePendingResult","pendingResult","markStreamClosed","open","state","for","closeReason","reason","closedAt","markTruncated","truncated","pendingClaudeResult","listOpenStreamsByJob","rows","map","getAttemptStreamById","getAttemptStreamByIdInTransaction","setObjectKeyAndDeleteChunks","updated","objectKey","length","delete","listStaleUncompactedStreams","olderThanSeconds","orderBy","limit","listStaleOpenStreams","createdAt","getOpenStreamCount","value","mapWith","BigInt","listExpiredClosedStreams","retentionDays","excludeIds","undefined","skipLocked","deleteExpiredStream","observedObjectKey","deleted","Boolean","accountingHasNoStreams"],"mappings":"AACA,SAAQA,GAAG,EAAEC,GAAG,EAAEC,EAAE,EAAEC,eAAe,EAAEC,MAAM,EAAEC,EAAE,EAAEC,UAAU,EAAEC,GAAG,QAAO,cAAc;AAEvF,SAAQC,wBAAwB,QAAO,kBAAkB;AACzD,SAAQC,EAAE,QAAyB,UAAU;AAC7C,SAAQC,cAAc,EAAEC,eAAe,QAAO,8BAA8B;AAC5E,SAAQC,SAAS,QAAO,qBAAqB;AAiB7C;;;CAGC,GACD,OAAO,eAAeC,iBACpBC,EAAe,EACfC,MAA2B;IAE3B,MAAM,CAACC,IAAI,GAAG,MAAMF,GACjBG,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CACJnB,IACEE,GAAGQ,eAAeU,KAAK,EAAEL,OAAOK,KAAK,GACrClB,GAAGQ,eAAeW,MAAM,EAAEN,OAAOM,MAAM,GACvCnB,GAAGQ,eAAeY,OAAO,EAAEP,OAAOO,OAAO;IAG/C,OAAON,MAAML,gBAAgBK,OAAO;AACtC;AAEA;;;;;CAKC,GACD,OAAO,eAAeO,uBAAuBR,MAG5C;IACC,MAAM,CAACC,IAAI,GAAG,MAAMP,KACjBQ,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CACJnB,IAAIE,GAAGQ,eAAeW,MAAM,EAAEN,OAAOM,MAAM,GAAGnB,GAAGQ,eAAeY,OAAO,EAAEP,OAAOO,OAAO;IAE3F,OAAON,MAAML,gBAAgBK,OAAO;AACtC;AAOA;;;;;CAKC,GACD,OAAO,eAAeQ,mCACpBV,EAAe,EACfW,QAA+B;IAE/B,MAAM,CAACT,IAAI,GAAG,MAAMF,GACjBY,MAAM,CAAChB,gBACPiB,MAAM,CAACF,UACPG,kBAAkB,CAAC;QAClBC,QAAQ;YAACnB,eAAeU,KAAK;YAAEV,eAAeW,MAAM;YAAEX,eAAeY,OAAO;SAAC;QAC7EQ,KAAK;YAACC,WAAWxB,GAAG,CAAC,KAAK,CAAC;QAAA;IAC7B,EACA,uEAAuE;IACvE,wDAAwD;KACvDyB,SAAS,CAAC;QAAC,GAAG7B,gBAAgBO,eAAe;QAAEuB,SAAS1B,GAAY,CAAC,QAAQ,CAAC;IAAA;IAEjF,IAAI,CAACS,KAAK,MAAM,IAAIkB,MAAM;IAC1B,IACElB,IAAImB,WAAW,KAAKV,SAASU,WAAW,IACxCnB,IAAIoB,SAAS,KAAKX,SAASW,SAAS,IACpCpB,IAAIqB,oBAAoB,KAAKZ,SAASY,oBAAoB,EAC1D;QACA,MAAM,IAAI7B;IACZ;IACA,OAAO;QAAC8B,QAAQ3B,gBAAgBK;QAAMiB,SAASjB,IAAIiB,OAAO;IAAA;AAC5D;AAEA,OAAO,eAAeM,yBACpBzB,EAAe,EACfW,QAA+B;IAE/B,MAAMe,SAAS,MAAMhB,mCAAmCV,IAAIW;IAC5D,OAAOe,OAAOF,MAAM;AACtB;AASA;;;;;;;CAOC,GACD,OAAO,eAAeG,yBACpB3B,EAAe,EACf4B,MAA2D;IAE3D,MAAMC,WAAW,MAAM7B,GACpB8B,MAAM,CAAClC,gBACPoB,GAAG,CAAC;QACHe,iBAAiBtC,GAAG,CAAC,EAAEG,eAAemC,eAAe,CAAC,GAAG,EAAEH,OAAOI,OAAO,CAAC,CAAC;QAC3Ef,WAAWxB,GAAG,CAAC,KAAK,CAAC;IACvB,GACCY,KAAK,CACJnB,IACEE,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ,GACrC9C,GAAGQ,eAAemC,eAAe,EAAEH,OAAOO,MAAM,IAGnDjB,SAAS,CAAC;QAACa,iBAAiBnC,eAAemC,eAAe;IAAA;IAE7D,IAAIF,QAAQ,CAAC,EAAE,EAAE,OAAO;QAACO,SAAS;QAAYL,iBAAiBF,QAAQ,CAAC,EAAE,CAACE,eAAe;IAAA;IAE1F,MAAM,CAACM,QAAQ,GAAG,MAAMrC,GACrBG,MAAM,CAAC;QAAC4B,iBAAiBnC,eAAemC,eAAe;IAAA,GACvD3B,IAAI,CAACR,gBACLS,KAAK,CAACjB,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ;IAC9C,MAAMH,kBAAkBM,SAASN,mBAAmB;IAEpD,MAAMO,eAAeV,OAAOO,MAAM,GAAGP,OAAOI,OAAO,IAAID;IACvD,OAAO;QAACK,SAASE,eAAe,UAAU;QAAOP;IAAe;AAClE;AAEA,OAAO,eAAeQ,sBACpBvC,EAAe,EACf4B,MAAsD;IAEtD,MAAM5B,GACH8B,MAAM,CAAClC,gBACPoB,GAAG,CAAC;QAACwB,oBAAoBZ,OAAOY,kBAAkB;QAAEvB,WAAWxB,GAAG,CAAC,KAAK,CAAC;IAAA,GACzEY,KAAK,CAACjB,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ;AAChD;AAEA,OAAO,eAAeO,sBACpBzC,EAAe,EACf4B,MAMC;IAED,MAAM5B,GACH8B,MAAM,CAAClC,gBACPoB,GAAG,CAAC;QACH0B,eAAed,OAAOe,OAAO;QAC7BC,iBAAiBhB,OAAOiB,SAAS;QACjCC,YAAYlB,OAAOmB,IAAI;QACvBC,qBAAqBpB,OAAOqB,aAAa;QACzChC,WAAWxB,GAAG,CAAC,KAAK,CAAC;IACvB,GACCY,KAAK,CAACjB,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ;AAChD;AAEA;;;;;;CAMC,GACD,OAAO,eAAegB,iBACpBlD,EAAe,EACf4B,MAA6E;IAE7E,MAAM,CAACuB,KAAK,GAAG,MAAMnD,GAClBG,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CAACnB,IAAIE,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ,GAAG9C,GAAGQ,eAAewD,KAAK,EAAE,UAC3EC,GAAG,CAAC;IACP,IAAI,CAACF,MAAM,OAAO;IAElB,MAAM,CAACjD,IAAI,GAAG,MAAMF,GACjB8B,MAAM,CAAClC,gBACPoB,GAAG,CAAC;QACHoC,OAAO;QACPE,aAAa1B,OAAO2B,MAAM;QAC1BC,UAAU/D,GAAG,CAAC,KAAK,CAAC;QACpBwB,WAAWxB,GAAG,CAAC,KAAK,CAAC;QACrBuD,qBAAqB;QACrB,GAAIpB,OAAO6B,aAAa,GAAG;YAACC,WAAW;QAAI,IAAI,CAAC,CAAC;IACnD,GACCrD,KAAK,CAACnB,IAAIE,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ,GAAG9C,GAAGQ,eAAewD,KAAK,EAAE,UAC3ElC,SAAS;IAEZ,OAAOhB,MACH;QACEsB,QAAQ3B,gBAAgBK;QACxByD,qBAAqBR,KAAKH,mBAAmB,IAAI;IACnD,IACA;AACN;AAEA,wFAAwF,GACxF,OAAO,eAAeY,qBAAqBtD,KAAa;IACtD,MAAMuD,OAAO,MAAMlE,KAChBQ,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CAACnB,IAAIE,GAAGQ,eAAeU,KAAK,EAAEA,QAAQlB,GAAGQ,eAAewD,KAAK,EAAE;IAEvE,OAAOS,KAAKC,GAAG,CAACjE;AAClB;AAEA,oFAAoF,GACpF,OAAO,eAAekE,qBAAqB7B,QAAgB;IACzD,MAAM,CAAChC,IAAI,GAAG,MAAMP,KAAKQ,MAAM,GAAGC,IAAI,CAACR,gBAAgBS,KAAK,CAACjB,GAAGQ,eAAeqC,EAAE,EAAEC;IACnF,OAAOhC,MAAML,gBAAgBK,OAAO;AACtC;AAEA,OAAO,eAAe8D,kCACpBhE,EAAe,EACfkC,QAAgB;IAEhB,MAAM,CAAChC,IAAI,GAAG,MAAMF,GAAGG,MAAM,GAAGC,IAAI,CAACR,gBAAgBS,KAAK,CAACjB,GAAGQ,eAAeqC,EAAE,EAAEC;IACjF,OAAOhC,MAAML,gBAAgBK,OAAO;AACtC;AAEA;;;;;;;;CAQC,GACD,OAAO,eAAe+D,4BACpBjE,EAAe,EACf4B,MAA6C;IAE7C,MAAMsC,UAAU,MAAMlE,GACnB8B,MAAM,CAAClC,gBACPoB,GAAG,CAAC;QAACmD,WAAWvC,OAAOuC,SAAS;QAAElD,WAAWxB,GAAG,CAAC,KAAK,CAAC;IAAA,GACvDY,KAAK,CACJnB,IACEE,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ,GACrC9C,GAAGQ,eAAewD,KAAK,EAAE,WACzB9D,OAAOM,eAAeuE,SAAS,IAGlCjD,SAAS,CAAC;QAACe,IAAIrC,eAAeqC,EAAE;IAAA;IAEnC,IAAIiC,QAAQE,MAAM,KAAK,GAAG,OAAO;QAACF,SAAS;IAAK;IAEhD,MAAMlE,GAAGqE,MAAM,CAACvE,WAAWO,KAAK,CAACjB,GAAGU,UAAUoC,QAAQ,EAAEN,OAAOM,QAAQ;IACvE,OAAO;QAACgC,SAAS;IAAI;AACvB;AAEA;;;;;CAKC,GACD,OAAO,eAAeI,4BAA4B1C,MAGjD;IACC,MAAMiC,OAAO,MAAMlE,KAChBQ,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CACJnB,IACEE,GAAGQ,eAAewD,KAAK,EAAE,WACzB9D,OAAOM,eAAeuE,SAAS,GAC/B5E,GAAGK,eAAe4D,QAAQ,EAAE/D,GAAG,CAAC,8BAA8B,EAAEmC,OAAO2C,gBAAgB,CAAC,CAAC,CAAC,IAG7FC,OAAO,CAACrF,IAAIS,eAAe4D,QAAQ,GACnCiB,KAAK,CAAC7C,OAAO6C,KAAK;IAErB,OAAOZ,KAAKC,GAAG,CAACjE;AAClB;AAEA;;;;CAIC,GACD,OAAO,eAAe6E,qBAAqB9C,MAG1C;IACC,MAAMiC,OAAO,MAAMlE,KAChBQ,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CACJnB,IACEE,GAAGQ,eAAewD,KAAK,EAAE,SACzB7D,GACEK,eAAe+E,SAAS,EACxBlF,GAAG,CAAC,8BAA8B,EAAEmC,OAAO2C,gBAAgB,CAAC,CAAC,CAAC,IAInEC,OAAO,CAACrF,IAAIS,eAAe+E,SAAS,GACpCF,KAAK,CAAC7C,OAAO6C,KAAK;IAErB,OAAOZ,KAAKC,GAAG,CAACjE;AAClB;AAEA,OAAO,eAAe+E;IACpB,MAAM,CAAC1E,IAAI,GAAG,MAAMP,KACjBQ,MAAM,CAAC;QAAC0E,OAAOpF,GAAW,CAAC,QAAQ,CAAC,CAACqF,OAAO,CAACC;IAAO,GACpD3E,IAAI,CAACR,gBACLS,KAAK,CAACjB,GAAGQ,eAAewD,KAAK,EAAE;IAElC,OAAOlD,KAAK2E,SAAS,EAAE;AACzB;AAEA;;;;;;CAMC,GACD,OAAO,eAAeG,yBAAyBpD,MAI9C;IACC,MAAMiC,OAAO,MAAMlE,KAChBQ,MAAM,GACNC,IAAI,CAACR,gBACLS,KAAK,CACJnB,IACEE,GAAGQ,eAAewD,KAAK,EAAE,WACzB7D,GAAGK,eAAe4D,QAAQ,EAAE/D,GAAG,CAAC,8BAA8B,EAAEmC,OAAOqD,aAAa,CAAC,CAAC,CAAC,GACvFrD,OAAOsD,UAAU,IAAItD,OAAOsD,UAAU,CAACd,MAAM,GAAG,IAC5C5E,WAAWI,eAAeqC,EAAE,EAAEL,OAAOsD,UAAU,IAC/CC,YAGPX,OAAO,CAACrF,IAAIS,eAAe4D,QAAQ,GAAGrE,IAAIS,eAAeqC,EAAE,GAC3DwC,KAAK,CAAC7C,OAAO6C,KAAK,EAClBpB,GAAG,CAAC,UAAU;QAAC+B,YAAY;IAAI;IAElC,OAAOvB,KAAKC,GAAG,CAACjE;AAClB;AAEA;;;CAGC,GACD,OAAO,eAAewF,oBACpBrF,EAAe,EACf4B,MAA4D;IAE5D,MAAM,CAAC1B,IAAI,GAAG,MAAMF,GACjBqE,MAAM,CAACzE,gBACPS,KAAK,CACJnB,IACEE,GAAGQ,eAAeqC,EAAE,EAAEL,OAAOM,QAAQ,GACrCN,OAAO0D,iBAAiB,KAAK,OACzBhG,OAAOM,eAAeuE,SAAS,IAC/B/E,GAAGQ,eAAeuE,SAAS,EAAEvC,OAAO0D,iBAAiB,IAG5DpE,SAAS,CAAC;QAACZ,OAAOV,eAAeU,KAAK;IAAA;IAEzC,OAAO;QAACiF,SAASC,QAAQtF;QAAMI,OAAOJ,KAAKI,SAAS;IAAI;AAC1D;AAEA,OAAO,eAAemF,uBAAuBzF,EAAe,EAAEM,KAAa;IACzE,MAAM,CAACJ,IAAI,GAAG,MAAMF,GACjBG,MAAM,CAAC;QAAC8B,IAAIrC,eAAeqC,EAAE;IAAA,GAC7B7B,IAAI,CAACR,gBACLS,KAAK,CAACjB,GAAGQ,eAAeU,KAAK,EAAEA,QAC/BmE,KAAK,CAAC;IAET,OAAO,CAACvE;AACV"}
|