@shipfox/api-logs 2.0.0

package/src/core/errors.ts

@@ -0,0 +1,37 @@
+/** The append offset is ahead of the committed length: the runner must rewind its spool cursor. */
+export class OffsetGapError extends Error {
+  constructor(public readonly committedLength: number) {
+    super(`Append offset is ahead of the committed length (${committedLength})`);
+    this.name = 'OffsetGapError';
+  }
+}
+
+/**
+ * The append body is not whole, newline-terminated records of the raw log
+ * record contract. `forgedType` is set only for the detectable forgery case — a
+ * line that is a valid server-only record (`capped`/`runner_lost`) under the read
+ * union but is not valid on the raw write path — so the append path can emit a narrowed audit warn
+ * without logging the payload.
+ */
+export class MalformedLogChunkError extends Error {
+  constructor(
+    message: string,
+    public readonly forgedType?: string,
+  ) {
+    super(message);
+    this.name = 'MalformedLogChunkError';
+  }
+}
+
+/**
+ * The lease's `(workspaceId, projectId, workflowRunAttemptId)` does not match the values
+ * stamped on the existing stream row. Since these are functionally determined
+ * by `jobId` via workflows FKs, a mismatch implies a forged token or a
+ * cross-job lease confusion — never a legitimate request.
+ */
+export class LeaseStreamMismatchError extends Error {
+  constructor() {
+    super('Lease identity does not match the existing stream row');
+    this.name = 'LeaseStreamMismatchError';
+  }
+}

package/src/core/finalize-attempt-stream.test.ts

@@ -0,0 +1,90 @@
+import {parseLogRecordLine} from '@shipfox/api-logs-dto';
+import {db} from '#db/db.js';
+import {getOrCreateAttemptStream} from '#db/streams.js';
+import {listChunks, listStreamClosedEvents} from '#test/queries.js';
+import {
+  createFinalizeAttemptLogStream,
+  type FinalizeAttemptLogStreamParams,
+} from './finalize-attempt-stream.js';
+
+const metrics = {
+  recordAppendedAdd: vi.fn(),
+  streamClosedAdd: vi.fn(),
+};
+
+const finalizeAttemptLogStream = createFinalizeAttemptLogStream({
+  recordAppendedCount: {add: metrics.recordAppendedAdd},
+  streamClosedCount: {add: metrics.streamClosedAdd},
+});
+
+function newIdentity(
+  overrides: Partial<FinalizeAttemptLogStreamParams> = {},
+): FinalizeAttemptLogStreamParams {
+  return {
+    jobId: crypto.randomUUID(),
+    stepId: crypto.randomUUID(),
+    attempt: 1,
+    workspaceId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    workflowRunAttemptId: crypto.randomUUID(),
+    logOutcome: 'drained',
+    ...overrides,
+  };
+}
+
+describe('finalizeAttemptLogStream', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('creates a missing drained stream and closes it as declared', async () => {
+    const identity = newIdentity({logOutcome: 'drained'});
+
+    const stream = await finalizeAttemptLogStream(identity);
+
+    expect(stream.state).toBe('closed');
+    expect(stream.closeReason).toBe('declared');
+    expect(stream.truncated).toBe(false);
+    expect(await listChunks(stream.id)).toHaveLength(0);
+    expect(await listStreamClosedEvents(stream.id)).toHaveLength(1);
+    expect(metrics.streamClosedAdd).toHaveBeenCalledWith(1, {reason: 'declared'});
+    expect(metrics.recordAppendedAdd).not.toHaveBeenCalled();
+  });
+
+  it('closes an abandoned open stream with a runner_lost tombstone', async () => {
+    const identity = newIdentity({logOutcome: 'abandoned'});
+    await db().transaction((tx) => getOrCreateAttemptStream(tx, identity));
+
+    const stream = await finalizeAttemptLogStream(identity);
+
+    expect(stream.state).toBe('closed');
+    expect(stream.closeReason).toBe('timeout');
+    expect(stream.truncated).toBe(true);
+    const chunks = await listChunks(stream.id);
+    expect(chunks).toHaveLength(1);
+    expect(chunks[0]?.origin).toBe('control');
+    const records = chunks[0]?.data
+      .toString('utf8')
+      .split('\n')
+      .filter(Boolean)
+      .map(parseLogRecordLine);
+    expect(records).toMatchObject([{type: 'runner_lost'}]);
+    expect(await listStreamClosedEvents(stream.id)).toHaveLength(1);
+    expect(metrics.streamClosedAdd).toHaveBeenCalledWith(1, {reason: 'timeout'});
+    expect(metrics.recordAppendedAdd).toHaveBeenCalledWith(1, {kind: 'runner_lost'});
+  });
+
+  it('does not emit another tombstone or close event when finalized again', async () => {
+    const identity = newIdentity({logOutcome: 'abandoned'});
+
+    const first = await finalizeAttemptLogStream(identity);
+    const second = await finalizeAttemptLogStream(identity);
+
+    expect(second.id).toBe(first.id);
+    expect(second.state).toBe('closed');
+    expect(await listChunks(first.id)).toHaveLength(1);
+    expect(await listStreamClosedEvents(first.id)).toHaveLength(1);
+    expect(metrics.streamClosedAdd).toHaveBeenCalledTimes(1);
+    expect(metrics.recordAppendedAdd).toHaveBeenCalledTimes(1);
+  });
+});

package/src/core/finalize-attempt-stream.ts

@@ -0,0 +1,59 @@
+import type {LogOutcomeDto} from '@shipfox/api-workflows-dto';
+import type {AttemptStream} from '#core/entities/attempt-stream.js';
+import {db} from '#db/db.js';
+import {
+  type AttemptStreamIdentity,
+  getAttemptStreamByIdInTransaction,
+  getOrCreateAttemptStreamWithStatus,
+} from '#db/streams.js';
+import {recordAppendedCount, streamClosedCount} from '#metrics/instance.js';
+import {closeStream} from './close-stream.js';
+
+export interface FinalizeAttemptLogStreamParams extends AttemptStreamIdentity {
+  logOutcome: LogOutcomeDto;
+}
+
+interface FinalizeAttemptLogStreamResult {
+  stream: AttemptStream;
+  closedReason: 'declared' | 'timeout' | null;
+}
+
+interface FinalizeMetrics {
+  recordAppendedCount: {add: (value: number, attributes: {kind: 'runner_lost'}) => void};
+  streamClosedCount: {
+    add: (value: number, attributes: {reason: 'declared' | 'timeout'}) => void;
+  };
+}
+
+const defaultMetrics: FinalizeMetrics = {
+  recordAppendedCount,
+  streamClosedCount,
+};
+
+export function createFinalizeAttemptLogStream(metrics: FinalizeMetrics = defaultMetrics) {
+  return async (params: FinalizeAttemptLogStreamParams): Promise<AttemptStream> => {
+    const result: FinalizeAttemptLogStreamResult = await db().transaction(async (tx) => {
+      const {stream} = await getOrCreateAttemptStreamWithStatus(tx, params);
+      if (stream.state === 'closed') return {stream, closedReason: null};
+
+      const reason = params.logOutcome === 'abandoned' ? 'timeout' : 'declared';
+      const closed = await closeStream(tx, {streamId: stream.id, reason});
+      if (closed) return {stream: closed, closedReason: reason};
+
+      const current = await getAttemptStreamByIdInTransaction(tx, stream.id);
+      if (!current) throw new Error(`Log stream disappeared during finalization: ${stream.id}`);
+      return {stream: current, closedReason: null};
+    });
+
+    if (result.closedReason) {
+      if (result.closedReason === 'timeout') {
+        metrics.recordAppendedCount.add(1, {kind: 'runner_lost'});
+      }
+      metrics.streamClosedCount.add(1, {reason: result.closedReason});
+    }
+
+    return result.stream;
+  };
+}
+
+export const finalizeAttemptLogStream = createFinalizeAttemptLogStream();

package/src/core/read-logs.ts

@@ -0,0 +1,77 @@
+import {presignedGetUrl} from '#api/object-storage.js';
+import {config} from '#config.js';
+import type {AttemptStream, StreamState} from '#core/entities/attempt-stream.js';
+import {readChunkPageBySeq} from '#db/chunks.js';
+import {getAttemptStreamById} from '#db/streams.js';
+
+export interface InlineLogRead {
+  mode: 'inline';
+  ndjson: string;
+  nextCursor: number;
+  hasMore: boolean;
+  state: StreamState;
+  truncated: boolean;
+}
+
+export interface PresignedLogRead {
+  mode: 'presigned';
+  url: string;
+  state: 'closed';
+  expiresAt: Date;
+  totalBytes: number;
+  truncated: boolean;
+}
+
+export type LogReadResult = InlineLogRead | PresignedLogRead;
+
+async function presignedRead(stream: AttemptStream, objectKey: string): Promise<PresignedLogRead> {
+  const {url, expiresAt} = await presignedGetUrl(objectKey);
+  return {
+    mode: 'presigned',
+    url,
+    state: 'closed',
+    expiresAt,
+    totalBytes: stream.committedLength,
+    truncated: stream.truncated,
+  };
+}
+
+/**
+ * Compaction-boundary guard: a stream can be loaded with no `objectKey`, then compaction
+ * publishes the key and deletes the chunks before the chunk read runs, which would yield a
+ * spurious empty inline frame on a closed stream. On that exact shape (closed stream, empty
+ * page) the row is re-read once; if it has since compacted, the presigned object is served
+ * instead.
+ *
+ * The re-read is scoped to a closed snapshot on purpose. An open stream returns empty pages
+ * on every idle tail poll, so re-reading each would add a row read to the hot path; and an
+ * open snapshot cannot have compacted (compaction runs only after close, seconds later via
+ * Temporal), so the open-empty-compacted window is too small to hit. If it ever did, the
+ * client's next poll reloads the now-compacted row and self-corrects.
+ */
+export async function buildLogReadResult(
+  stream: AttemptStream,
+  cursor: number,
+): Promise<LogReadResult> {
+  if (stream.objectKey) return presignedRead(stream, stream.objectKey);
+
+  const page = await readChunkPageBySeq({
+    streamId: stream.id,
+    afterSeq: cursor,
+    maxBytes: config.LOG_READ_INLINE_MAX_BYTES,
+  });
+
+  if (page.data.length === 0 && stream.state === 'closed') {
+    const refreshed = await getAttemptStreamById(stream.id);
+    if (refreshed?.objectKey) return presignedRead(refreshed, refreshed.objectKey);
+  }
+
+  return {
+    mode: 'inline',
+    ndjson: page.data.toString('utf8'),
+    nextCursor: page.nextSeq,
+    hasMore: page.hasMore,
+    state: stream.state,
+    truncated: stream.truncated,
+  };
+}

package/src/core/reap-stale-open-streams.test.ts

@@ -0,0 +1,137 @@
+import {eq, sql} from 'drizzle-orm';
+import {appendLogs} from '#core/append-logs.js';
+import {db, type Transaction} from '#db/db.js';
+import {attemptStreams} from '#db/schema/attempt-streams.js';
+import {endLine, ndjsonBody, outputLine} from '#test/fixtures/ndjson.js';
+import {findStream, listChunks, listStreamClosedEvents} from '#test/queries.js';
+import type {CloseStreamParams} from './close-stream.js';
+import * as closeStreamModule from './close-stream.js';
+import {reapStaleOpenStreams} from './reap-stale-open-streams.js';
+
+interface Ctx {
+  jobId: string;
+  stepId: string;
+  workspaceId: string;
+  projectId: string;
+  workflowRunAttemptId: string;
+}
+
+function newCtx(): Ctx {
+  return {
+    jobId: crypto.randomUUID(),
+    stepId: crypto.randomUUID(),
+    workspaceId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    workflowRunAttemptId: crypto.randomUUID(),
+  };
+}
+
+// Default LOG_STREAM_REAP_AFTER_SECONDS is 7200 (2h); a 3h backdate makes a stream stale.
+function reap(overrides: Partial<Parameters<typeof reapStaleOpenStreams>[0]> = {}) {
+  return reapStaleOpenStreams({olderThanSeconds: 7200, batchLimit: 100, ...overrides});
+}
+
+async function backdateCreatedAtPastReapWindow(streamId: string): Promise<void> {
+  await db()
+    .update(attemptStreams)
+    .set({createdAt: sql`now() - interval '3 hours'`})
+    .where(eq(attemptStreams.id, streamId));
+}
+
+describe('reapStaleOpenStreams', () => {
+  it('force-closes a stale open stream with a runner_lost tombstone and one event', async () => {
+    const ctx = newCtx();
+    await appendLogs({...ctx, attempt: 1, offset: 0, body: ndjsonBody(outputLine('partial\n'))});
+    const open = await findStream({...ctx, attempt: 1});
+    await backdateCreatedAtPastReapWindow(open?.id as string);
+
+    const {reaped} = await reap();
+
+    // Counters are whole-table tallies; row state isolates this test from unrelated stale rows
+    // in the shared test DB.
+    expect(reaped).toBeGreaterThanOrEqual(1);
+    const after = await findStream({...ctx, attempt: 1});
+    expect(after?.state).toBe('closed');
+    expect(after?.closeReason).toBe('timeout');
+    expect(after?.truncated).toBe(true);
+    expect(after?.committedLength).toBe(open?.committedLength);
+    expect((await listChunks(after?.id as string)).map((c) => c.origin)).toEqual([
+      'runner',
+      'control',
+    ]);
+    expect(await listStreamClosedEvents(after?.id as string)).toHaveLength(1);
+  });
+
+  it('leaves a recently-created open stream untouched', async () => {
+    const ctx = newCtx();
+    await appendLogs({...ctx, attempt: 1, offset: 0, body: ndjsonBody(outputLine('fresh\n'))});
+
+    await reap();
+
+    const after = await findStream({...ctx, attempt: 1});
+    expect(after?.state).toBe('open');
+  });
+
+  it('does not select an already declared-closed stream, even an old one', async () => {
+    const ctx = newCtx();
+    await appendLogs({
+      ...ctx,
+      attempt: 1,
+      offset: 0,
+      body: ndjsonBody(outputLine('done\n'), endLine(5)),
+    });
+    const closed = await findStream({...ctx, attempt: 1});
+    await backdateCreatedAtPastReapWindow(closed?.id as string);
+
+    await reap();
+
+    const after = await findStream({...ctx, attempt: 1});
+    expect(after?.closeReason).toBe('declared');
+    expect(after?.truncated).toBe(false);
+    expect(await listStreamClosedEvents(after?.id as string)).toHaveLength(1);
+  });
+
+  it('logs and skips one stream whose close fails, still reaping the rest of the batch', async () => {
+    const poisonCtx = newCtx();
+    const healthyCtx = newCtx();
+    await appendLogs({
+      ...poisonCtx,
+      attempt: 1,
+      offset: 0,
+      body: ndjsonBody(outputLine('poison\n')),
+    });
+    await appendLogs({
+      ...healthyCtx,
+      attempt: 1,
+      offset: 0,
+      body: ndjsonBody(outputLine('healthy\n')),
+    });
+    const poison = await findStream({...poisonCtx, attempt: 1});
+    const healthy = await findStream({...healthyCtx, attempt: 1});
+    await backdateCreatedAtPastReapWindow(poison?.id as string);
+    await backdateCreatedAtPastReapWindow(healthy?.id as string);
+    const realCloseStream = closeStreamModule.closeStream;
+    vi.spyOn(closeStreamModule, 'closeStream').mockImplementation(
+      (tx: Transaction, params: CloseStreamParams) =>
+        params.streamId === poison?.id
+          ? Promise.reject(new Error('induced close failure'))
+          : realCloseStream(tx, params),
+    );
+
+    const {reaped, failed} = await reap();
+
+    expect(failed).toBeGreaterThanOrEqual(1);
+    expect(reaped).toBeGreaterThanOrEqual(1);
+    const healthyAfter = await findStream({...healthyCtx, attempt: 1});
+    expect(healthyAfter?.state).toBe('closed');
+    const poisonAfter = await findStream({...poisonCtx, attempt: 1});
+    expect(poisonAfter?.state).toBe('open');
+
+    // The mocked failure leaves this row open by design. Mark it closed through the DB so later
+    // reaper tests do not inherit a stale open row.
+    await db()
+      .update(attemptStreams)
+      .set({state: 'closed'})
+      .where(eq(attemptStreams.id, poison?.id as string));
+  });
+});

package/src/core/reap-stale-open-streams.ts

@@ -0,0 +1,60 @@
+import {logger} from '@shipfox/node-opentelemetry';
+import {db} from '#db/db.js';
+import {listStaleOpenStreams} from '#db/streams.js';
+import {recordAppendedCount, streamClosedCount} from '#metrics/instance.js';
+import {closeStream} from './close-stream.js';
+
+export interface ReapStaleOpenStreamsResult {
+  /** Streams force-closed by this run. */
+  reaped: number;
+  /** Streams whose guarded close threw; logged, skipped, retried next run. */
+  failed: number;
+}
+
+export interface ReapStaleOpenStreamsParams {
+  /**
+   * A stream open longer than this since `created_at` can no longer be appended to: its job
+   * holds one lease minted at-or-before the stream's creation, so once the lease TTL has
+   * elapsed no append can land and the close is safe. Set above `AUTH_JOB_LEASE_TOKEN_EXPIRES_IN`.
+   */
+  olderThanSeconds: number;
+  batchLimit: number;
+}
+
+/**
+ * Force-closes open streams the one-shot job-terminal sweep missed: a stream whose first append
+ * landed after that sweep's snapshot would otherwise stay `open` forever, invisible to
+ * compaction and retention (both partial on `state = 'closed'`). Reusing the guarded
+ * `closeStream` re-drives each into the normal closed -> compact -> retention lifecycle.
+ *
+ * Each row closes in its own transaction, so a tick/retry overlap is idempotent (the guarded
+ * `markStreamClosed` lets only one close win) and one failed row is logged and skipped instead
+ * of aborting the batch.
+ */
+export async function reapStaleOpenStreams(
+  params: ReapStaleOpenStreamsParams,
+): Promise<ReapStaleOpenStreamsResult> {
+  const stale = await listStaleOpenStreams({
+    olderThanSeconds: params.olderThanSeconds,
+    limit: params.batchLimit,
+  });
+
+  const result: ReapStaleOpenStreamsResult = {reaped: 0, failed: 0};
+  for (const stream of stale) {
+    try {
+      const closed = await db().transaction((tx) =>
+        closeStream(tx, {streamId: stream.id, reason: 'timeout'}),
+      );
+      if (closed) {
+        result.reaped += 1;
+        recordAppendedCount.add(1, {kind: 'runner_lost'});
+        streamClosedCount.add(1, {reason: 'timeout'});
+      }
+    } catch (error) {
+      result.failed += 1;
+      logger().error({err: error, streamId: stream.id}, 'Failed to reap stale open log stream');
+    }
+  }
+
+  return result;
+}