@shipfox/api-logs 2.0.0

package/src/db/chunks.ts

@@ -0,0 +1,143 @@
+import {Buffer} from 'node:buffer';
+import {and, asc, eq, gt, lte, sql} from 'drizzle-orm';
+import {db, type Transaction} from './db.js';
+import {type ChunkOrigin, logChunks} from './schema/chunks.js';
+
+export interface InsertChunkParams {
+  streamId: string;
+  streamOffset: number;
+  byteLen: number;
+  data: Buffer;
+  origin: ChunkOrigin;
+}
+
+export async function insertChunk(tx: Transaction, params: InsertChunkParams): Promise<void> {
+  await tx.insert(logChunks).values({
+    streamId: params.streamId,
+    streamOffset: params.streamOffset,
+    byteLen: params.byteLen,
+    data: params.data,
+    origin: params.origin,
+  });
+}
+
+export interface ChunkPageRow {
+  seq: number;
+  data: Buffer;
+}
+
+/**
+ * One keyset page of a stream's chunk bytes in `seq` order. Compaction walks pages
+ * with `afterSeq = lastSeqOfPreviousPage`, so memory stays flat regardless of stream
+ * size, and reads run outside any transaction (no tx is pinned for the long upload).
+ * Safe against late writers because a closed stream never accepts another chunk.
+ */
+export async function readChunksKeyset(params: {
+  streamId: string;
+  afterSeq: number;
+  limit: number;
+}): Promise<ChunkPageRow[]> {
+  const rows = await db()
+    .select({seq: logChunks.seq, data: logChunks.data})
+    .from(logChunks)
+    .where(and(eq(logChunks.streamId, params.streamId), gt(logChunks.seq, params.afterSeq)))
+    .orderBy(asc(logChunks.seq))
+    .limit(params.limit);
+  return rows;
+}
+
+export interface ChunkPage {
+  /** Concatenated chunk bytes for the page, in `seq` order: ready-to-serve NDJSON. */
+  data: Buffer;
+  /** The last `seq` included; the next read passes this as `afterSeq`. */
+  nextSeq: number;
+  /** Whether at least one more chunk exists past `nextSeq`. */
+  hasMore: boolean;
+}
+
+// Caps the per-read metadata scan so a stream with pathologically many tiny chunks
+// can't make one read walk the whole stream; the client re-polls (hasMore) past it.
+const CHUNK_PAGE_SCAN_CAP = 4096;
+
+/**
+ * A byte-bounded page of a stream's chunk bytes in `seq` order, for the inline read path.
+ * Walking by `seq` (not the runner byte offset) is what makes server-injected control tombstones
+ * interleave with normalized runner records exactly as compaction concatenates them, so the inline
+ * NDJSON is byte-identical to the compacted object.
+ *
+ * Returns at most ~`maxBytes`, but always at least one whole chunk so the cursor advances
+ * even when a single chunk exceeds `maxBytes`. The cheap `(seq, byte_len)` scan picks the
+ * prefix first, so the heavy `data` column is only materialized for the chunks actually
+ * returned.
+ */
+export async function readChunkPageBySeq(params: {
+  streamId: string;
+  afterSeq: number;
+  maxBytes: number;
+}): Promise<ChunkPage> {
+  const meta = await db()
+    .select({seq: logChunks.seq, byteLen: logChunks.byteLen})
+    .from(logChunks)
+    .where(and(eq(logChunks.streamId, params.streamId), gt(logChunks.seq, params.afterSeq)))
+    .orderBy(asc(logChunks.seq))
+    .limit(CHUNK_PAGE_SCAN_CAP + 1);
+
+  if (meta.length === 0) {
+    return {data: Buffer.alloc(0), nextSeq: params.afterSeq, hasMore: false};
+  }
+
+  let accumulatedBytes = 0;
+  let includedCount = 0;
+  let nextSeq = params.afterSeq;
+  for (const row of meta) {
+    if (includedCount > 0 && accumulatedBytes + row.byteLen > params.maxBytes) break;
+    accumulatedBytes += row.byteLen;
+    nextSeq = row.seq;
+    includedCount += 1;
+    if (includedCount >= CHUNK_PAGE_SCAN_CAP) break;
+  }
+
+  const hasMore = includedCount < meta.length;
+
+  const rows = await db()
+    .select({data: logChunks.data})
+    .from(logChunks)
+    .where(
+      and(
+        eq(logChunks.streamId, params.streamId),
+        gt(logChunks.seq, params.afterSeq),
+        lte(logChunks.seq, nextSeq),
+      ),
+    )
+    .orderBy(asc(logChunks.seq));
+
+  return {data: Buffer.concat(rows.map((row) => row.data)), nextSeq, hasMore};
+}
+
+export interface ChunkStats {
+  count: number;
+  maxSeq: number;
+  uncompressedBytes: number;
+}
+
+/**
+ * Aggregate of a stream's chunks: row count, highest `seq`, and total stored bytes.
+ * Compaction reads this once before uploading, both to stamp object metadata and as the
+ * baseline its streamed totals are checked against before the chunk rows are deleted.
+ */
+export async function chunkStats(streamId: string): Promise<ChunkStats> {
+  const [row] = await db()
+    .select({
+      count: sql<number>`count(*)::int`,
+      maxSeq: sql<number>`coalesce(max(${logChunks.seq}), 0)::int`,
+      uncompressedBytes: sql<string>`coalesce(sum(${logChunks.byteLen}), 0)::text`,
+    })
+    .from(logChunks)
+    .where(eq(logChunks.streamId, streamId));
+
+  return {
+    count: row?.count ?? 0,
+    maxSeq: row?.maxSeq ?? 0,
+    uncompressedBytes: Number(row?.uncompressedBytes ?? '0'),
+  };
+}

package/src/db/db.ts

@@ -0,0 +1,22 @@
+import {drizzle, type NodePgDatabase} from '@shipfox/node-drizzle';
+import {pgClient} from '@shipfox/node-postgres';
+import {attemptStreams} from './schema/attempt-streams.js';
+import {logChunks} from './schema/chunks.js';
+import {jobAccounting} from './schema/job-accounting.js';
+import {logsOutbox} from './schema/outbox.js';
+
+export const schema = {jobAccounting, attemptStreams, logChunks, logsOutbox};
+
+export type Database = NodePgDatabase<typeof schema>;
+export type Transaction = Parameters<Parameters<Database['transaction']>[0]>[0];
+
+let _db: Database | undefined;
+
+export function db() {
+  if (!_db) _db = drizzle(pgClient(), {schema});
+  return _db;
+}
+
+export function closeDb(): void {
+  _db = undefined;
+}

package/src/db/index.ts

@@ -0,0 +1,4 @@
+import {dirname, resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+export const migrationsPath = resolve(dirname(fileURLToPath(import.meta.url)), '../../drizzle');

package/src/db/schema/attempt-streams.ts

@@ -0,0 +1,106 @@
+import {uuidv7PrimaryKey} from '@shipfox/node-drizzle';
+import {sql} from 'drizzle-orm';
+import {
+  bigint,
+  boolean,
+  index,
+  integer,
+  text,
+  timestamp,
+  uniqueIndex,
+  uuid,
+} from 'drizzle-orm/pg-core';
+import type {AttemptStream} from '#core/entities/attempt-stream.js';
+import {pgTable} from './common.js';
+
+/**
+ * One stream per (job, step, attempt). Identity is scoped by `job_id` (from the
+ * lease), so a lease can only ever reach its own job's streams — cross-job log
+ * injection is structurally impossible. `workspace_id`, `project_id`, and `workflow_run_attempt_id`
+ * are stamped from the lease at create time; they are functionally determined by
+ * `job_id` via workflows FKs, so they are denormalized here for self-contained
+ * authorization (per-project read filtering, audit) without joining back to
+ * workflows. `committed_length` is the offset-CAS axis (raw NDJSON spool bytes the
+ * server has durably accepted from the runner).
+ *
+ * `truncated` is an out-of-band terminal flag set when the timeout sweep
+ * force-closes a stream the runner never ended; the `AttemptStream` entity is the
+ * canonical reference for what it means.
+ *
+ * Per-row `committed_length` and `declared_total_bytes` are bounded by the
+ * per-job budget, so `mode: 'number'` is safe on the hot path. Any cross-row
+ * aggregate (workspace or system-wide totals) MUST read as bigint at the
+ * query site — the global sum is unbounded and would silently lose precision
+ * past 2^53 as a JS number.
+ */
+export const attemptStreams = pgTable(
+  'attempt_streams',
+  {
+    id: uuidv7PrimaryKey(),
+    jobId: uuid('job_id').notNull(),
+    stepId: uuid('step_id').notNull(),
+    attempt: integer('attempt').notNull(),
+    workspaceId: uuid('workspace_id').notNull(),
+    projectId: uuid('project_id').notNull(),
+    workflowRunAttemptId: uuid('workflow_run_attempt_id').notNull(),
+    committedLength: bigint('committed_length', {mode: 'number'}).notNull().default(0),
+    state: text('state', {enum: ['open', 'closed']})
+      .notNull()
+      .default('open'),
+    closeReason: text('close_reason', {enum: ['declared', 'timeout']}),
+    declaredTotalBytes: bigint('declared_total_bytes', {mode: 'number'}),
+    truncated: boolean('truncated').notNull().default(false),
+    objectKey: text('object_key'),
+    createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),
+    updatedAt: timestamp('updated_at', {withTimezone: true}).notNull().defaultNow(),
+    closedAt: timestamp('closed_at', {withTimezone: true}),
+  },
+  (table) => [
+    uniqueIndex('logs_attempt_streams_identity_unique').on(
+      table.jobId,
+      table.stepId,
+      table.attempt,
+    ),
+    // The session read path looks up a stream by (step, attempt) without the job id, so the
+    // identity unique (which leads with job_id) can't serve it. This index keeps that lookup
+    // off a sequential scan as the table grows.
+    index('logs_attempt_streams_step_attempt_idx').on(table.stepId, table.attempt),
+    // Timeout-close sweeps an open job's streams by job_id; the partial index keeps
+    // it off the closed tail.
+    index('logs_attempt_streams_open_by_job_idx').on(table.jobId).where(sql`"state" = 'open'`),
+    // Open-age reaping needs an index that stays off the closed tail; created_at is
+    // immutable, so appends do not churn it.
+    index('logs_attempt_streams_open_age_idx').on(table.createdAt).where(sql`"state" = 'open'`),
+    // Retention scans closed streams by close age; partial so it never carries the
+    // open (in-flight) set.
+    index('logs_attempt_streams_retention_idx').on(table.closedAt).where(sql`"state" = 'closed'`),
+    // Reconcile re-drives closed streams that never got an object key.
+    index('logs_attempt_streams_uncompacted_idx')
+      .on(table.closedAt)
+      .where(sql`"state" = 'closed' and "object_key" is null`),
+  ],
+);
+
+export type AttemptStreamDb = typeof attemptStreams.$inferSelect;
+export type AttemptStreamInsertDb = typeof attemptStreams.$inferInsert;
+
+export function toAttemptStream(row: AttemptStreamDb): AttemptStream {
+  return {
+    id: row.id,
+    jobId: row.jobId,
+    stepId: row.stepId,
+    attempt: row.attempt,
+    workspaceId: row.workspaceId,
+    projectId: row.projectId,
+    workflowRunAttemptId: row.workflowRunAttemptId,
+    committedLength: row.committedLength,
+    state: row.state,
+    closeReason: row.closeReason,
+    declaredTotalBytes: row.declaredTotalBytes,
+    truncated: row.truncated,
+    objectKey: row.objectKey,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+    closedAt: row.closedAt,
+  };
+}

package/src/db/schema/chunks.ts

@@ -0,0 +1,43 @@
+import {uuidv7PrimaryKey} from '@shipfox/node-drizzle';
+import {bigint, bigserial, index, integer, text, timestamp, uuid} from 'drizzle-orm/pg-core';
+import {attemptStreams} from './attempt-streams.js';
+import {bytea, pgTable} from './common.js';
+
+/**
+ * Hot, append-only log bytes for open streams, pending compaction.
+ *
+ * Two byte axes meet here and MUST stay distinct:
+ *
+ *   Runner spool (CAS axis)          Stored chunk stream (read axis, by seq)
+ *   offset 0 ┌────────────┐  ───►  seq 1 │ normalized chunk A │ origin=runner
+ *      100   ├─ chunk A ──┤  ───►  seq 2 │ normalized chunk B │ origin=runner
+ *      250   ├─ chunk B ──┤  ───►  seq 3 │ {capped}           │ origin=control ← server-injected;
+ *            └────────────┘                                      does NOT advance committed_length
+ *
+ * `origin` is `runner` for bytes accepted from a runner append after ingest normalization, and
+ * `control` for a server-injected tombstone. `stream_offset` is the runner-axis position of a
+ * runner chunk; for a server `control` chunk it is informational. `seq` (insertion order) is the
+ * read axis the reader walks, so server records interleave correctly with normalized runner
+ * records.
+ */
+export const logChunks = pgTable(
+  'chunks',
+  {
+    id: uuidv7PrimaryKey(),
+    streamId: uuid('stream_id')
+      .notNull()
+      .references(() => attemptStreams.id, {onDelete: 'cascade'}),
+    seq: bigserial('seq', {mode: 'number'}).notNull(),
+    streamOffset: bigint('stream_offset', {mode: 'number'}).notNull(),
+    byteLen: integer('byte_len').notNull(),
+    data: bytea('data').notNull(),
+    origin: text('origin', {enum: ['runner', 'control']}).notNull(),
+    createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),
+  },
+  (table) => [index('logs_chunks_stream_seq_idx').on(table.streamId, table.seq)],
+);
+
+export type LogChunkDb = typeof logChunks.$inferSelect;
+export type LogChunkInsertDb = typeof logChunks.$inferInsert;
+
+export type ChunkOrigin = 'runner' | 'control';

package/src/db/schema/common.ts

@@ -0,0 +1,12 @@
+import type {Buffer} from 'node:buffer';
+import {customType, pgTableCreator} from 'drizzle-orm/pg-core';
+
+export const pgTable = pgTableCreator((name) => `logs_${name}`);
+
+// drizzle-orm 0.45 has no native bytea column; the node-postgres driver returns
+// and accepts Buffer for bytea, so a customType is the supported path.
+export const bytea = customType<{data: Buffer; driverData: Buffer}>({
+  dataType() {
+    return 'bytea';
+  },
+});

package/src/db/schema/job-accounting.ts

@@ -0,0 +1,38 @@
+import {bigint, timestamp, uuid} from 'drizzle-orm/pg-core';
+import type {JobAccounting} from '#core/entities/job-accounting.js';
+import {pgTable} from './common.js';
+
+/**
+ * Accrual-budget state per job run, keyed by the job id carried in the lease.
+ * `stored_bytes_used` counts the normalized NDJSON bytes the server has stored for the job
+ * (envelope and control records included), so the budget bounds exactly what lands in Postgres.
+ * `started_at` is the budget clock origin (first append).
+ * `capped_at`, once set, makes every later append a no-op drop.
+ *
+ * Per-row `stored_bytes_used` is bounded by the per-job budget, so `mode:
+ * 'number'` is safe on the hot path. Any cross-row aggregate (workspace or
+ * system-wide totals) MUST read as bigint at the query site — the global sum
+ * is unbounded and would silently lose precision past 2^53 as a JS number.
+ */
+export const jobAccounting = pgTable('job_accounting', {
+  jobId: uuid('job_id').primaryKey(),
+  workspaceId: uuid('workspace_id').notNull(),
+  storedBytesUsed: bigint('stored_bytes_used', {mode: 'number'}).notNull().default(0),
+  startedAt: timestamp('started_at', {withTimezone: true}).notNull().defaultNow(),
+  cappedAt: timestamp('capped_at', {withTimezone: true}),
+  createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),
+  updatedAt: timestamp('updated_at', {withTimezone: true}).notNull().defaultNow(),
+});
+
+export type JobAccountingDb = typeof jobAccounting.$inferSelect;
+export type JobAccountingInsertDb = typeof jobAccounting.$inferInsert;
+
+export function toJobAccounting(row: JobAccountingDb): JobAccounting {
+  return {
+    jobId: row.jobId,
+    workspaceId: row.workspaceId,
+    storedBytesUsed: row.storedBytesUsed,
+    startedAt: row.startedAt,
+    cappedAt: row.cappedAt,
+  };
+}

package/src/db/schema/outbox.ts

@@ -0,0 +1,4 @@
+import {createOutboxTable} from '@shipfox/node-outbox';
+import {pgTable} from './common.js';
+
+export const logsOutbox = createOutboxTable(pgTable);

package/src/db/streams.test.ts

@@ -0,0 +1,47 @@
+import {appendLogs} from '#core/append-logs.js';
+import {endLine, ndjsonBody, outputLine} from '#test/fixtures/ndjson.js';
+import {getOpenStreamCount} from './streams.js';
+
+interface Ctx {
+  jobId: string;
+  stepId: string;
+  workspaceId: string;
+  projectId: string;
+  workflowRunAttemptId: string;
+}
+
+function newCtx(): Ctx {
+  return {
+    jobId: crypto.randomUUID(),
+    stepId: crypto.randomUUID(),
+    workspaceId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    workflowRunAttemptId: crypto.randomUUID(),
+  };
+}
+
+describe('getOpenStreamCount', () => {
+  it('reports the current open stream count', async () => {
+    const before = await getOpenStreamCount();
+
+    const after = await getOpenStreamCount();
+    expect(after - before).toBe(0n);
+  });
+
+  it('counts newly opened streams without counting streams closed in the same test', async () => {
+    const before = await getOpenStreamCount();
+    const open = newCtx();
+    const closed = newCtx();
+    await appendLogs({...open, attempt: 1, offset: 0, body: ndjsonBody(outputLine('open\n'))});
+    await appendLogs({
+      ...closed,
+      attempt: 1,
+      offset: 0,
+      body: ndjsonBody(outputLine('closed\n'), endLine(7)),
+    });
+
+    const after = await getOpenStreamCount();
+
+    expect(after - before).toBe(1n);
+  });
+});