@shipfox/api-logs 2.0.0

package/src/temporal/activities/compaction-reconcile.ts

@@ -0,0 +1,48 @@
+import {logger} from '@shipfox/node-opentelemetry';
+import {temporalClient} from '@shipfox/node-temporal';
+import {config} from '#config.js';
+import {listStaleUncompactedStreams} from '#db/streams.js';
+import {LOGS_COMPACTION_TASK_QUEUE} from '#temporal/constants.js';
+
+// Bounded per tick; remaining stale streams are picked up on the next cron run.
+const RECONCILE_BATCH_LIMIT = 100;
+
+/**
+ * Backstop for the event-triggered path: finds closed streams that never got an object
+ * key past the stale window and re-starts compaction for each. Starting by the same
+ * `logs-compact:{streamId}` workflow id re-drives a stream whose bounded-retry run already
+ * failed and closed, while skipping one whose run is still RUNNING (AlreadyStarted). Starts
+ * independent workflows, not children, so a re-driven run is not tied to this cron tick.
+ *
+ * One stream's unexpected start failure is logged and skipped, never thrown: a single poison
+ * stream must not abort the batch and leave the rest of the backlog un-re-driven (the next
+ * tick retries it). The activity only fails on the initial DB query.
+ */
+export async function compactionReconcileActivity(): Promise<{restarted: number; failed: number}> {
+  const stale = await listStaleUncompactedStreams({
+    olderThanSeconds: config.LOG_COMPACTION_RECONCILE_STALE_SECONDS,
+    limit: RECONCILE_BATCH_LIMIT,
+  });
+
+  let restarted = 0;
+  let failed = 0;
+  for (const stream of stale) {
+    try {
+      await temporalClient().workflow.start('compactStream', {
+        taskQueue: LOGS_COMPACTION_TASK_QUEUE,
+        workflowId: `logs-compact:${stream.id}`,
+        args: [{streamId: stream.id}],
+      });
+      restarted += 1;
+    } catch (error) {
+      if (error instanceof Error && error.name === 'WorkflowExecutionAlreadyStartedError') continue;
+      failed += 1;
+      logger().error(
+        {err: error, streamId: stream.id},
+        'Failed to re-drive stale stream compaction',
+      );
+    }
+  }
+
+  return {restarted, failed};
+}

package/src/temporal/activities/index.ts

@@ -0,0 +1,15 @@
+import {closeAbandonedStreamsActivity} from './close-abandoned-streams.js';
+import {compactStreamActivity} from './compact-stream.js';
+import {compactionReconcileActivity} from './compaction-reconcile.js';
+import {reapStaleOpenStreamsActivity} from './reap-stale-open-streams.js';
+import {retentionSweepActivity} from './retention-sweep.js';
+
+export function createLogsActivities() {
+  return {
+    closeAbandonedStreamsActivity,
+    compactStreamActivity,
+    compactionReconcileActivity,
+    reapStaleOpenStreamsActivity,
+    retentionSweepActivity,
+  };
+}

package/src/temporal/activities/reap-stale-open-streams.test.ts

@@ -0,0 +1,48 @@
+import {MockActivityEnvironment} from '@temporalio/testing';
+import {eq, sql} from 'drizzle-orm';
+import {appendLogs} from '#core/append-logs.js';
+import {db} from '#db/db.js';
+import {attemptStreams} from '#db/schema/attempt-streams.js';
+import {ndjsonBody, outputLine} from '#test/fixtures/ndjson.js';
+import {findStream} from '#test/queries.js';
+import {reapStaleOpenStreamsActivity} from './reap-stale-open-streams.js';
+
+interface Ctx {
+  jobId: string;
+  stepId: string;
+  workspaceId: string;
+  projectId: string;
+  workflowRunAttemptId: string;
+}
+
+function newCtx(): Ctx {
+  return {
+    jobId: crypto.randomUUID(),
+    stepId: crypto.randomUUID(),
+    workspaceId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    workflowRunAttemptId: crypto.randomUUID(),
+  };
+}
+
+describe('reapStaleOpenStreamsActivity', () => {
+  it('reaps a stale open stream end to end (config wiring under a real activity context)', async () => {
+    const ctx = newCtx();
+    await appendLogs({...ctx, attempt: 1, offset: 0, body: ndjsonBody(outputLine('partial\n'))});
+    const open = await findStream({...ctx, attempt: 1});
+    // Past the default LOG_STREAM_REAP_AFTER_SECONDS (7200) the activity reads from config.
+    await db()
+      .update(attemptStreams)
+      .set({createdAt: sql`now() - interval '3 hours'`})
+      .where(eq(attemptStreams.id, open?.id as string));
+
+    const result = (await new MockActivityEnvironment().run(
+      reapStaleOpenStreamsActivity,
+    )) as Awaited<ReturnType<typeof reapStaleOpenStreamsActivity>>;
+
+    expect(result.reaped).toBeGreaterThanOrEqual(1);
+    const after = await findStream({...ctx, attempt: 1});
+    expect(after?.state).toBe('closed');
+    expect(after?.closeReason).toBe('timeout');
+  });
+});

package/src/temporal/activities/reap-stale-open-streams.ts

@@ -0,0 +1,14 @@
+import {config} from '#config.js';
+import {
+  type ReapStaleOpenStreamsResult,
+  reapStaleOpenStreams,
+} from '#core/reap-stale-open-streams.js';
+import {REAP_BATCH_LIMIT} from '#temporal/constants.js';
+
+/** Cron-driven backstop that force-closes open streams the one-shot job-terminal close missed. */
+export function reapStaleOpenStreamsActivity(): Promise<ReapStaleOpenStreamsResult> {
+  return reapStaleOpenStreams({
+    olderThanSeconds: config.LOG_STREAM_REAP_AFTER_SECONDS,
+    batchLimit: REAP_BATCH_LIMIT,
+  });
+}

package/src/temporal/activities/retention-sweep.test.ts

@@ -0,0 +1,38 @@
+import {MockActivityEnvironment} from '@temporalio/testing';
+import {eq, sql} from 'drizzle-orm';
+import {db} from '#db/db.js';
+import {attemptStreams} from '#db/schema/attempt-streams.js';
+import {arrangeClosedStream, type ClosedStreamIdentity} from '#test/fixtures/closed-stream.js';
+import {ndjsonBody, outputLine} from '#test/fixtures/ndjson.js';
+import {findStream} from '#test/queries.js';
+import {retentionSweepActivity} from './retention-sweep.js';
+
+function newIdentity(): ClosedStreamIdentity {
+  return {
+    jobId: crypto.randomUUID(),
+    stepId: crypto.randomUUID(),
+    attempt: 1,
+    workspaceId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    workflowRunAttemptId: crypto.randomUUID(),
+  };
+}
+
+describe('retentionSweepActivity', () => {
+  it('sweeps an expired stream end to end (config wiring + heartbeat under a real activity context)', async () => {
+    const id = newIdentity();
+    const stream = await arrangeClosedStream(id, {chunks: [ndjsonBody(outputLine('x'))]});
+    // Past the default LOG_RETENTION_DAYS (90) the activity reads from config.
+    await db()
+      .update(attemptStreams)
+      .set({closedAt: sql`now() - interval '100 days'`})
+      .where(eq(attemptStreams.id, stream.id));
+
+    const result = (await new MockActivityEnvironment().run(retentionSweepActivity)) as Awaited<
+      ReturnType<typeof retentionSweepActivity>
+    >;
+
+    expect(await findStream(id)).toBeNull();
+    expect(result.timedOut).toBe(false);
+  });
+});

package/src/temporal/activities/retention-sweep.ts

@@ -0,0 +1,23 @@
+import {Context} from '@temporalio/activity';
+import {config} from '#config.js';
+import {type RetentionSweepResult, runRetentionSweep} from '#core/retention.js';
+import {
+  RETENTION_BATCH_LIMIT,
+  RETENTION_MAX_ITERATIONS,
+  RETENTION_TIME_BUDGET_MS,
+} from '#temporal/constants.js';
+
+/**
+ * Cron-driven sweep for expired closed log streams. Heartbeats per stream; the core loop owns
+ * the wall-clock budget because Temporal timeouts do not stop already-running JS.
+ */
+export function retentionSweepActivity(): Promise<RetentionSweepResult> {
+  const ctx = Context.current();
+  return runRetentionSweep({
+    retentionDays: config.LOG_RETENTION_DAYS,
+    batchLimit: RETENTION_BATCH_LIMIT,
+    timeBudgetMs: RETENTION_TIME_BUDGET_MS,
+    maxIterations: RETENTION_MAX_ITERATIONS,
+    onProgress: () => ctx.heartbeat(),
+  });
+}

package/src/temporal/constants.ts

@@ -0,0 +1,14 @@
+export const LOGS_LIFECYCLE_TASK_QUEUE = 'logs-lifecycle';
+
+// Compaction uploads can run for minutes; keeping them off the lifecycle queue stops a
+// burst of large compactions from starving the short, time-sensitive close-abandoned sweep.
+export const LOGS_COMPACTION_TASK_QUEUE = 'logs-compaction';
+
+// The wall-clock budget is the real sweep bound; Temporal's timeout does not stop JS already
+// running in the worker.
+export const RETENTION_BATCH_LIMIT = 200;
+export const RETENTION_TIME_BUDGET_MS = 4 * 60_000;
+export const RETENTION_MAX_ITERATIONS = 1_000;
+
+// Bounded per tick; remaining stale open streams are picked up on the next cron run.
+export const REAP_BATCH_LIMIT = 100;

package/src/temporal/workflows/close-abandoned-streams.ts

@@ -0,0 +1,25 @@
+import {log, proxyActivities, sleep} from '@temporalio/workflow';
+import type {createLogsActivities} from '../activities/index.js';
+
+const {closeAbandonedStreamsActivity} = proxyActivities<ReturnType<typeof createLogsActivities>>({
+  startToCloseTimeout: '5 minutes',
+});
+
+export interface CloseAbandonedStreamsInput {
+  jobId: string;
+  graceSeconds: number;
+}
+
+/**
+ * Started by the `WORKFLOWS_JOB_TERMINATED` subscriber, deduped per job. Waits out
+ * the grace period (so a last in-flight append can land and the runner's own
+ * declared close can win), then force-closes whatever is still open.
+ */
+export async function closeAbandonedStreams(input: CloseAbandonedStreamsInput): Promise<void> {
+  await sleep(input.graceSeconds * 1000);
+
+  const {closed} = await closeAbandonedStreamsActivity({jobId: input.jobId});
+  if (closed > 0) {
+    log.info('Force-closed abandoned log streams', {jobId: input.jobId, closed});
+  }
+}

package/src/temporal/workflows/compact-stream.ts

@@ -0,0 +1,26 @@
+import {log, proxyActivities} from '@temporalio/workflow';
+import type {createLogsActivities} from '../activities/index.js';
+
+// Bounded generous retry (not unlimited): a never-finishing RUNNING workflow would reject
+// the reconcile cron's restart of the same workflow id. After the attempts are spent the
+// run fails and closes, so the cron can re-drive it.
+const {compactStreamActivity} = proxyActivities<ReturnType<typeof createLogsActivities>>({
+  startToCloseTimeout: '1 hour',
+  heartbeatTimeout: '2 minutes',
+  retry: {
+    initialInterval: '10 seconds',
+    backoffCoefficient: 2,
+    maximumInterval: '5 minutes',
+    maximumAttempts: 12,
+  },
+});
+
+export interface CompactStreamInput {
+  streamId: string;
+}
+
+/** Started per closed stream (deduped by `logs-compact:{streamId}`) by the closed-event subscriber and the reconcile cron. */
+export async function compactStream(input: CompactStreamInput): Promise<void> {
+  const result = await compactStreamActivity({streamId: input.streamId});
+  log.info('Compacted log stream', {streamId: input.streamId, ...result});
+}

package/src/temporal/workflows/compaction-reconcile-cron.ts

@@ -0,0 +1,14 @@
+import {log, proxyActivities} from '@temporalio/workflow';
+import type {createLogsActivities} from '../activities/index.js';
+
+const {compactionReconcileActivity} = proxyActivities<ReturnType<typeof createLogsActivities>>({
+  startToCloseTimeout: '5 minutes',
+});
+
+/** Cron-scheduled backstop that re-drives closed streams whose compaction never started or permanently failed. */
+export async function compactionReconcileCron(): Promise<void> {
+  const {restarted, failed} = await compactionReconcileActivity();
+  if (restarted > 0 || failed > 0) {
+    log.info('Re-drove stale uncompacted log streams', {restarted, failed});
+  }
+}

package/src/temporal/workflows/index.ts

@@ -0,0 +1,5 @@
+export {closeAbandonedStreams} from './close-abandoned-streams.js';
+export {compactStream} from './compact-stream.js';
+export {compactionReconcileCron} from './compaction-reconcile-cron.js';
+export {reapStaleOpenStreamsCron} from './reap-stale-open-streams-cron.js';
+export {retentionSweepCron} from './retention-sweep-cron.js';

package/src/temporal/workflows/reap-stale-open-streams-cron.ts

@@ -0,0 +1,13 @@
+import {log, proxyActivities} from '@temporalio/workflow';
+import type {createLogsActivities} from '../activities/index.js';
+
+const {reapStaleOpenStreamsActivity} = proxyActivities<ReturnType<typeof createLogsActivities>>({
+  startToCloseTimeout: '5 minutes',
+});
+
+export async function reapStaleOpenStreamsCron(): Promise<void> {
+  const {reaped, failed} = await reapStaleOpenStreamsActivity();
+  if (reaped > 0 || failed > 0) {
+    log.info('Reaped stale open log streams', {reaped, failed});
+  }
+}

package/src/temporal/workflows/retention-sweep-cron.ts

@@ -0,0 +1,23 @@
+import {log, proxyActivities} from '@temporalio/workflow';
+import type {createLogsActivities} from '../activities/index.js';
+
+const {retentionSweepActivity} = proxyActivities<ReturnType<typeof createLogsActivities>>({
+  startToCloseTimeout: '5 minutes',
+  heartbeatTimeout: '1 minute',
+  // Retry on the next cron tick so a timed-out attempt cannot overlap its own still-running loop.
+  retry: {maximumAttempts: 1},
+});
+
+export async function retentionSweepCron(): Promise<void> {
+  const {deleted, raced, failed, accountingPruned, iterations, timedOut} =
+    await retentionSweepActivity();
+  // Keep a workflow-level audit trail for this destructive sweep.
+  log.info('Retention sweep complete', {
+    deleted,
+    raced,
+    failed,
+    accountingPruned,
+    iterations,
+    timedOut,
+  });
+}

package/test/env.ts

@@ -0,0 +1,37 @@
+process.env.POSTGRES_HOST ??= 'localhost';
+process.env.POSTGRES_PORT ??= '5432';
+process.env.POSTGRES_USERNAME ??= 'shipfox';
+process.env.POSTGRES_PASSWORD ??= 'password';
+process.env.POSTGRES_DATABASE = 'api_test';
+process.env.POSTGRES_MAX_CONNECTIONS ??= '5';
+process.env.AUTH_JWT_SECRET = 'test-secret';
+process.env.AUTH_JOB_LEASE_TOKEN_SECRET = 'test-lease-secret';
+process.env.AUTH_RUNNER_SESSION_TOKEN_SECRET = 'test-runner-session-secret';
+process.env.SECRETS_ENCRYPTION_KEK = 'ZmVkY2JhOTg3NjU0MzIxMGZlZGNiYTk4NzY1NDMyMTA=';
+
+// Small accrual budget so cap/budget tests use tiny payloads: base 100 bytes,
+// rate 60 bytes/min (1 byte/second).
+process.env.LOG_BUDGET_BASE_BYTES = '100';
+process.env.LOG_BUDGET_RATE_BYTES_PER_MINUTE = '60';
+
+// Modest body limit (64 KiB) so the route 413 test uses a small payload.
+process.env.LOG_APPEND_BODY_LIMIT_BYTES = '65536';
+
+// Small agent_session line cap so the over-cap test uses a tiny payload. Must stay
+// <= LOG_APPEND_BODY_LIMIT_BYTES for the startup invariant.
+process.env.LOG_MAX_SESSION_LINE_BYTES = '512';
+
+// Tiny inline read page so the read-path drain/has_more test pages with small fixtures.
+process.env.LOG_READ_INLINE_MAX_BYTES = '256';
+
+// Real Garage dev credentials (bootstrap.sh creates the shipfox-logs-test bucket and grants
+// this key). Compaction tests upload to and read back from live Garage from compose.yml.
+process.env.LOG_STORAGE_S3_ENDPOINT ??= 'http://localhost:3900';
+process.env.LOG_STORAGE_S3_REGION = 'garage';
+process.env.LOG_STORAGE_S3_BUCKET = 'shipfox-logs-test';
+process.env.LOG_STORAGE_S3_ACCESS_KEY_ID = 'GK000000000000000000000000';
+process.env.LOG_STORAGE_S3_SECRET_ACCESS_KEY =
+  '0000000000000000000000000000000000000000000000000000000000000000';
+process.env.LOG_STORAGE_S3_FORCE_PATH_STYLE = 'true';
+
+process.env.TZ = 'UTC';

package/test/factories/job-accounting.ts

@@ -0,0 +1,38 @@
+import {Factory} from 'fishery';
+import {db} from '#db/db.js';
+import {jobAccounting} from '#db/schema/job-accounting.js';
+
+interface JobAccountingAttrs {
+  jobId: string;
+  workspaceId: string;
+  storedBytesUsed: number;
+  startedAt: Date;
+  cappedAt: Date | null;
+}
+
+// Arranges a pre-existing accounting row so a test can control the budget clock
+// origin (`startedAt`) or pre-cap a job. `appendLogs` creates the row with
+// `INSERT ... ON CONFLICT DO NOTHING`, so a pre-arranged row is preserved.
+export const jobAccountingFactory = Factory.define<JobAccountingAttrs>(({onCreate}) => {
+  onCreate(async (attrs) => {
+    await db()
+      .insert(jobAccounting)
+      .values({
+        jobId: attrs.jobId,
+        workspaceId: attrs.workspaceId,
+        storedBytesUsed: attrs.storedBytesUsed,
+        startedAt: attrs.startedAt,
+        cappedAt: attrs.cappedAt,
+      })
+      .onConflictDoNothing({target: jobAccounting.jobId});
+    return attrs;
+  });
+
+  return {
+    jobId: crypto.randomUUID(),
+    workspaceId: crypto.randomUUID(),
+    storedBytesUsed: 0,
+    startedAt: new Date(),
+    cappedAt: null,
+  };
+});

package/test/fixtures/closed-stream.ts

@@ -0,0 +1,53 @@
+import type {Buffer} from 'node:buffer';
+import {closeStream} from '#core/close-stream.js';
+import type {AttemptStream} from '#core/entities/attempt-stream.js';
+import {insertChunk} from '#db/chunks.js';
+import {db} from '#db/db.js';
+import {getOrCreateAttemptStream} from '#db/streams.js';
+
+export interface ClosedStreamIdentity {
+  jobId: string;
+  stepId: string;
+  attempt: number;
+  workspaceId: string;
+  projectId: string;
+  workflowRunAttemptId: string;
+}
+
+export interface ArrangeClosedStreamOptions {
+  /** Runner chunk bodies, inserted in order with increasing `stream_offset`. */
+  chunks?: Buffer[];
+  /** Close via the timeout path, appending a `runner_lost` control chunk (no runner chunks). */
+  tombstone?: boolean;
+}
+
+/** Arranges a closed stream with the given chunks, the precondition compaction runs against. */
+export function arrangeClosedStream(
+  identity: ClosedStreamIdentity,
+  options: ArrangeClosedStreamOptions = {},
+): Promise<AttemptStream> {
+  const chunks = options.chunks ?? [];
+
+  return db().transaction(async (tx) => {
+    const stream = await getOrCreateAttemptStream(tx, identity);
+
+    let offset = 0;
+    for (const data of chunks) {
+      await insertChunk(tx, {
+        streamId: stream.id,
+        streamOffset: offset,
+        byteLen: data.length,
+        data,
+        origin: 'runner',
+      });
+      offset += data.length;
+    }
+
+    const closed = await closeStream(tx, {
+      streamId: stream.id,
+      reason: options.tombstone ? 'timeout' : 'declared',
+    });
+
+    return closed ?? stream;
+  });
+}

package/test/fixtures/lease-token.ts

@@ -0,0 +1,42 @@
+import {JOB_LEASE_TOKEN_AUDIENCE} from '@shipfox/api-auth-dto';
+import {signHs256} from '@shipfox/node-jwt';
+
+// Matches test/env.ts; the lease-token auth method reads this same value from config.
+const SECRET = process.env.AUTH_JOB_LEASE_TOKEN_SECRET ?? 'test-lease-secret';
+
+export interface MintLeaseTokenParams {
+  jobId: string;
+  jobExecutionId: string;
+  workspaceId?: string;
+  projectId?: string;
+  workflowRunId?: string;
+  workflowRunAttemptId?: string;
+  currentStepId?: string;
+  currentStepAttempt?: number;
+  secret?: string;
+  expiresIn?: string;
+  audience?: string;
+}
+
+export function mintLeaseToken(params: MintLeaseTokenParams): Promise<string> {
+  return signHs256({
+    payload: {
+      jobId: params.jobId,
+      jobExecutionId: params.jobExecutionId,
+      workflowRunId: params.workflowRunId ?? crypto.randomUUID(),
+      workflowRunAttemptId: params.workflowRunAttemptId ?? crypto.randomUUID(),
+      projectId: params.projectId ?? crypto.randomUUID(),
+      workspaceId: params.workspaceId ?? crypto.randomUUID(),
+      runnerSessionId: crypto.randomUUID(),
+      ...(params.currentStepId && params.currentStepAttempt !== undefined
+        ? {
+            currentStepId: params.currentStepId,
+            currentStepAttempt: params.currentStepAttempt,
+          }
+        : {}),
+    },
+    secret: params.secret ?? SECRET,
+    expiresIn: params.expiresIn ?? '90m',
+    audience: params.audience ?? JOB_LEASE_TOKEN_AUDIENCE,
+  });
+}

package/test/fixtures/ndjson.ts

@@ -0,0 +1,34 @@
+import {Buffer} from 'node:buffer';
+
+export function outputLine(data: string, stream: 'stdout' | 'stderr' = 'stdout'): string {
+  return `${JSON.stringify({v: 1, ts: 1, type: 'output', stream, data})}\n`;
+}
+
+export function recordLine(fields: Record<string, unknown>): string {
+  return `${JSON.stringify({v: 1, ts: 1, ...fields})}\n`;
+}
+
+export function endLine(totalBytes: number): string {
+  return recordLine({type: 'end', total_bytes: totalBytes});
+}
+
+export function sessionLine(data: string): string {
+  return recordLine({type: 'agent_session', data});
+}
+
+export function groupStartLine(
+  groupId: string,
+  name: string,
+  parentGroupId: string | null = null,
+): string {
+  return recordLine({type: 'group_start', group_id: groupId, parent_group_id: parentGroupId, name});
+}
+
+export function ndjsonBody(...lines: string[]): Buffer {
+  return Buffer.from(lines.join(''), 'utf8');
+}
+
+/** A single output record whose decoded `data` payload is exactly `payloadBytes` ASCII bytes. */
+export function outputOfBytes(payloadBytes: number): Buffer {
+  return ndjsonBody(outputLine('x'.repeat(payloadBytes)));
+}

package/test/globalSetup.ts

@@ -0,0 +1,44 @@
+import './env.js';
+import {runMigrations} from '@shipfox/node-drizzle';
+import {closePostgresClient, createPostgresClient} from '@shipfox/node-postgres';
+import {sql} from 'drizzle-orm';
+import {checkBucketReachable, closeS3Client} from '#api/object-storage.js';
+import {closeDb, db} from '#db/db.js';
+import {migrationsPath} from '#db/index.js';
+
+const BUCKET_READY_TIMEOUT_MS = 30_000;
+const BUCKET_READY_POLL_INTERVAL_MS = 500;
+
+// `garage-init` exits 0 once it has posted the key/bucket ACLs, but Garage does not guarantee
+// those are immediately usable for S3 uploads. Poll HeadBucket so the suite never starts before
+// the store will accept the configured credentials — without this, the compaction tests race the
+// ACL propagation and fail with `AccessDenied: No such key`.
+async function waitForBucketReachable(timeoutMs: number): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (await checkBucketReachable()) return;
+    await new Promise((resolve) => setTimeout(resolve, BUCKET_READY_POLL_INTERVAL_MS));
+  }
+  if (!(await checkBucketReachable())) {
+    throw new Error(
+      `Garage bucket not reachable after ${timeoutMs / 1000}s; ` +
+        'check that the garage + garage-init compose services are up and the access key is provisioned.',
+    );
+  }
+}
+
+export async function setup() {
+  createPostgresClient();
+
+  await runMigrations(db(), migrationsPath, '__drizzle_migrations_logs');
+  await db().execute(sql`TRUNCATE logs_chunks CASCADE`);
+  await db().execute(sql`TRUNCATE logs_attempt_streams CASCADE`);
+  await db().execute(sql`TRUNCATE logs_job_accounting CASCADE`);
+  await db().execute(sql`TRUNCATE logs_outbox CASCADE`);
+
+  closeDb();
+  await closePostgresClient();
+
+  await waitForBucketReachable(BUCKET_READY_TIMEOUT_MS);
+  closeS3Client();
+}

package/test/index.ts

@@ -0,0 +1,9 @@
+export {jobAccountingFactory} from './factories/job-accounting.js';
+export {mintLeaseToken} from './fixtures/lease-token.js';
+export {
+  findAccounting,
+  findStream,
+  listChunks,
+  listStreamClosedEvents,
+  type StreamIdentity,
+} from './queries.js';

package/test/queries.ts

@@ -0,0 +1,54 @@
+import {LOG_STREAM_CLOSED, type LogStreamClosedEvent} from '@shipfox/api-logs-dto';
+import {and, asc, eq} from 'drizzle-orm';
+import type {AttemptStream} from '#core/entities/attempt-stream.js';
+import type {JobAccounting} from '#core/entities/job-accounting.js';
+import {db} from '#db/db.js';
+import {attemptStreams, toAttemptStream} from '#db/schema/attempt-streams.js';
+import {type LogChunkDb, logChunks} from '#db/schema/chunks.js';
+import {jobAccounting, toJobAccounting} from '#db/schema/job-accounting.js';
+import {logsOutbox} from '#db/schema/outbox.js';
+
+export interface StreamIdentity {
+  jobId: string;
+  stepId: string;
+  attempt: number;
+}
+
+export async function findStream(identity: StreamIdentity): Promise<AttemptStream | null> {
+  const [row] = await db()
+    .select()
+    .from(attemptStreams)
+    .where(
+      and(
+        eq(attemptStreams.jobId, identity.jobId),
+        eq(attemptStreams.stepId, identity.stepId),
+        eq(attemptStreams.attempt, identity.attempt),
+      ),
+    );
+  return row ? toAttemptStream(row) : null;
+}
+
+export async function findAccounting(jobId: string): Promise<JobAccounting | null> {
+  const [row] = await db().select().from(jobAccounting).where(eq(jobAccounting.jobId, jobId));
+  return row ? toJobAccounting(row) : null;
+}
+
+export async function listChunks(streamId: string): Promise<LogChunkDb[]> {
+  return await db()
+    .select()
+    .from(logChunks)
+    .where(eq(logChunks.streamId, streamId))
+    .orderBy(asc(logChunks.seq));
+}
+
+/** The `logs.stream.closed` outbox events written for a given stream (filtered in memory; the table is not truncated between tests). */
+export async function listStreamClosedEvents(streamId: string): Promise<LogStreamClosedEvent[]> {
+  const rows = await db()
+    .select()
+    .from(logsOutbox)
+    .where(eq(logsOutbox.eventType, LOG_STREAM_CLOSED));
+
+  return rows
+    .map((row) => row.payload as LogStreamClosedEvent)
+    .filter((payload) => payload.streamId === streamId);
+}

package/test/setup.ts

@@ -0,0 +1,17 @@
+import './env.js';
+import {closePostgresClient, createPostgresClient} from '@shipfox/node-postgres';
+import {afterAll, afterEach, beforeAll, vi} from '@shipfox/vitest/vi';
+import {closeDb} from '#db/db.js';
+
+beforeAll(() => {
+  createPostgresClient();
+});
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+afterAll(async () => {
+  closeDb();
+  await closePostgresClient();
+});