@shipfox/api-logs 2.0.0

package/src/presentation/subscribers/on-log-stream-closed.ts

@@ -0,0 +1,26 @@
+import type {LogStreamClosedEvent} from '@shipfox/api-logs-dto';
+import {logger} from '@shipfox/node-opentelemetry';
+import {temporalClient} from '@shipfox/node-temporal';
+import {LOGS_COMPACTION_TASK_QUEUE} from '#temporal/constants.js';
+
+/**
+ * A stream reached `closed` (declared end or timeout sweep). Start its compaction workflow
+ * and return; no S3 work runs in the dispatcher. Deduped by `logs-compact:{streamId}`, so a
+ * redelivered event is a no-op. A start that fails for any other reason is left to the
+ * reconcile cron, which re-drives any closed-but-uncompacted stream.
+ */
+export async function onLogStreamClosed(payload: LogStreamClosedEvent): Promise<void> {
+  try {
+    await temporalClient().workflow.start('compactStream', {
+      taskQueue: LOGS_COMPACTION_TASK_QUEUE,
+      workflowId: `logs-compact:${payload.streamId}`,
+      args: [{streamId: payload.streamId}],
+    });
+  } catch (error) {
+    if (error instanceof Error && error.name === 'WorkflowExecutionAlreadyStartedError') {
+      logger().debug({streamId: payload.streamId}, 'Compaction workflow already started');
+      return;
+    }
+    throw error;
+  }
+}

package/src/presentation/subscribers/on-step-attempt-terminated.ts

@@ -0,0 +1,16 @@
+import type {WorkflowsStepAttemptTerminatedEventDto} from '@shipfox/api-workflows-dto';
+import {finalizeAttemptLogStream} from '#core/finalize-attempt-stream.js';
+
+export async function onStepAttemptTerminated(
+  payload: WorkflowsStepAttemptTerminatedEventDto,
+): Promise<void> {
+  await finalizeAttemptLogStream({
+    jobId: payload.jobId,
+    workflowRunAttemptId: payload.workflowRunAttemptId,
+    workspaceId: payload.workspaceId,
+    projectId: payload.projectId,
+    stepId: payload.stepId,
+    attempt: payload.attempt,
+    logOutcome: payload.logOutcome,
+  });
+}

package/src/temporal/activities/close-abandoned-streams.test.ts

@@ -0,0 +1,84 @@
+import {appendLogs} from '#core/append-logs.js';
+import {closeAbandonedStreamsActivity} from '#temporal/activities/close-abandoned-streams.js';
+import {endLine, ndjsonBody, outputLine} from '#test/fixtures/ndjson.js';
+import {findStream, listChunks, listStreamClosedEvents} from '#test/queries.js';
+
+interface Ctx {
+  jobId: string;
+  stepId: string;
+  workspaceId: string;
+  projectId: string;
+  workflowRunAttemptId: string;
+}
+
+function newCtx(): Ctx {
+  return {
+    jobId: crypto.randomUUID(),
+    stepId: crypto.randomUUID(),
+    workspaceId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    workflowRunAttemptId: crypto.randomUUID(),
+  };
+}
+
+describe('closeAbandonedStreamsActivity', () => {
+  it('force-closes an open stream with a runner_lost tombstone and one event', async () => {
+    const ctx = newCtx();
+    await appendLogs({...ctx, attempt: 1, offset: 0, body: ndjsonBody(outputLine('partial\n'))});
+    const open = await findStream({...ctx, attempt: 1});
+
+    const {closed} = await closeAbandonedStreamsActivity({jobId: ctx.jobId});
+
+    expect(closed).toBe(1);
+    const after = await findStream({...ctx, attempt: 1});
+    expect(after?.state).toBe('closed');
+    expect(after?.closeReason).toBe('timeout');
+    expect(after?.truncated).toBe(true);
+    expect(after?.committedLength).toBe(open?.committedLength);
+    expect((await listChunks(after?.id as string)).map((c) => c.origin)).toEqual([
+      'runner',
+      'control',
+    ]);
+    expect(await listStreamClosedEvents(after?.id as string)).toHaveLength(1);
+  });
+
+  it('closes only the still-open streams, skipping ones already declared-closed', async () => {
+    const ctx = newCtx();
+    const stepOpen = crypto.randomUUID();
+    const stepDone = crypto.randomUUID();
+    await appendLogs({
+      ...ctx,
+      stepId: stepDone,
+      attempt: 1,
+      offset: 0,
+      body: ndjsonBody(outputLine('done\n'), endLine(4)),
+    });
+    await appendLogs({
+      ...ctx,
+      stepId: stepOpen,
+      attempt: 1,
+      offset: 0,
+      body: ndjsonBody(outputLine('partial\n')),
+    });
+    const doneStream = await findStream({jobId: ctx.jobId, stepId: stepDone, attempt: 1});
+
+    const {closed} = await closeAbandonedStreamsActivity({jobId: ctx.jobId});
+
+    expect(closed).toBe(1);
+    const openAfter = await findStream({jobId: ctx.jobId, stepId: stepOpen, attempt: 1});
+    expect(openAfter?.closeReason).toBe('timeout');
+    expect(openAfter?.truncated).toBe(true);
+    const doneAfter = await findStream({jobId: ctx.jobId, stepId: stepDone, attempt: 1});
+    expect(doneAfter?.closeReason).toBe('declared');
+    expect(doneAfter?.truncated).toBe(false);
+    expect(await listStreamClosedEvents(doneStream?.id as string)).toHaveLength(1);
+  });
+
+  it('is a no-op for a job with no open streams', async () => {
+    const ctx = newCtx();
+
+    const {closed} = await closeAbandonedStreamsActivity({jobId: ctx.jobId});
+
+    expect(closed).toBe(0);
+  });
+});

package/src/temporal/activities/close-abandoned-streams.ts

@@ -0,0 +1,30 @@
+import {closeStream} from '#core/close-stream.js';
+import {db} from '#db/db.js';
+import {listOpenStreamsByJob} from '#db/streams.js';
+import {recordAppendedCount, streamClosedCount} from '#metrics/instance.js';
+
+/**
+ * Job termination does not guarantee the runner flushed an end record: it may have
+ * died, been capped, or lost its spool. Each stream closes in its own transaction
+ * through the guarded `closeStream`, so a declared-close race is skipped instead of
+ * writing a duplicate event or tombstone.
+ */
+export async function closeAbandonedStreamsActivity(params: {
+  jobId: string;
+}): Promise<{closed: number}> {
+  const open = await listOpenStreamsByJob(params.jobId);
+
+  let closed = 0;
+  for (const stream of open) {
+    const result = await db().transaction((tx) =>
+      closeStream(tx, {streamId: stream.id, reason: 'timeout'}),
+    );
+    if (result) {
+      closed += 1;
+      recordAppendedCount.add(1, {kind: 'runner_lost'});
+      streamClosedCount.add(1, {reason: 'timeout'});
+    }
+  }
+
+  return {closed};
+}

package/src/temporal/activities/compact-stream.test.ts

@@ -0,0 +1,221 @@
+import {Buffer} from 'node:buffer';
+import {Readable} from 'node:stream';
+import {createGzip, gunzipSync} from 'node:zlib';
+import {GetObjectCommand, HeadObjectCommand, ListObjectsV2Command} from '@aws-sdk/client-s3';
+import {MockActivityEnvironment} from '@temporalio/testing';
+import {eq} from 'drizzle-orm';
+import {deleteObject, s3Client} from '#api/object-storage.js';
+import {config} from '#config.js';
+import {compactedGzipStream} from '#core/compaction.js';
+import {logObjectKey} from '#core/entities/log-object.js';
+import {db} from '#db/db.js';
+import {attemptStreams} from '#db/schema/attempt-streams.js';
+import {getAttemptStreamById, setObjectKeyAndDeleteChunks} from '#db/streams.js';
+import {arrangeClosedStream, type ClosedStreamIdentity} from '#test/fixtures/closed-stream.js';
+import {ndjsonBody, outputLine} from '#test/fixtures/ndjson.js';
+import {listChunks} from '#test/queries.js';
+import {
+  type CompactStreamResult,
+  compactStreamActivity,
+  createCompactStreamActivity,
+} from './compact-stream.js';
+
+const compactedGzipStreamMock = vi.fn<typeof compactedGzipStream>();
+const setObjectKeyAndDeleteChunksMock = vi.fn<typeof setObjectKeyAndDeleteChunks>();
+const compactStreamActivityWithMocks = createCompactStreamActivity({
+  compactedGzipStream: compactedGzipStreamMock,
+  setObjectKeyAndDeleteChunks: setObjectKeyAndDeleteChunksMock,
+});
+
+function newIdentity(): ClosedStreamIdentity {
+  return {
+    jobId: crypto.randomUUID(),
+    stepId: crypto.randomUUID(),
+    attempt: 1,
+    workspaceId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    workflowRunAttemptId: crypto.randomUUID(),
+  };
+}
+
+function runCompaction(
+  streamId: string,
+  activity = compactStreamActivity,
+): Promise<CompactStreamResult> {
+  return new MockActivityEnvironment().run(activity, {streamId});
+}
+
+async function getObjectBytes(key: string): Promise<Buffer> {
+  const res = await s3Client().send(
+    new GetObjectCommand({Bucket: config.LOG_STORAGE_S3_BUCKET, Key: key}),
+  );
+  if (!res.Body) throw new Error('object has no body');
+  return Buffer.from(await res.Body.transformToByteArray());
+}
+
+function headObject(key: string) {
+  return s3Client().send(new HeadObjectCommand({Bucket: config.LOG_STORAGE_S3_BUCKET, Key: key}));
+}
+
+// Every attempt's object lives under the stream's stable prefix; listing it proves both the
+// winner's object and that losing/failed attempts left nothing behind.
+async function listKeysUnderStream(identity: ClosedStreamIdentity): Promise<string[]> {
+  const res = await s3Client().send(
+    new ListObjectsV2Command({
+      Bucket: config.LOG_STORAGE_S3_BUCKET,
+      Prefix: `${logObjectKey(config.LOG_STORAGE_S3_PREFIX, identity)}/`,
+    }),
+  );
+  return (res.Contents ?? []).map((object) => object.Key ?? '');
+}
+
+function compactedKey(result: CompactStreamResult): string {
+  if (result.outcome !== 'compacted') throw new Error(`expected compacted, got ${result.outcome}`);
+  return result.objectKey;
+}
+
+describe('compactStreamActivity', () => {
+  beforeEach(() => {
+    compactedGzipStreamMock.mockReset();
+    setObjectKeyAndDeleteChunksMock.mockReset();
+    compactedGzipStreamMock.mockImplementation(compactedGzipStream);
+    setObjectKeyAndDeleteChunksMock.mockImplementation(setObjectKeyAndDeleteChunks);
+  });
+
+  it('compacts many chunks into one gzip object and deletes the chunk rows', async () => {
+    const chunks = [outputLine('one\n'), outputLine('two\n'), outputLine('three\n')].map((l) =>
+      ndjsonBody(l),
+    );
+    const identity = newIdentity();
+    const stream = await arrangeClosedStream(identity, {chunks});
+
+    const result = await runCompaction(stream.id);
+
+    const key = compactedKey(result);
+    expect(key.startsWith(`${logObjectKey(config.LOG_STORAGE_S3_PREFIX, identity)}/`)).toBe(true);
+    const after = await getAttemptStreamById(stream.id);
+    expect(after?.objectKey).toBe(key);
+    expect(await listChunks(stream.id)).toHaveLength(0);
+    expect(await listKeysUnderStream(identity)).toEqual([key]);
+
+    const head = await headObject(key);
+    expect(head.ContentEncoding).toBe('gzip');
+    expect(head.ContentType).toBe('application/x-ndjson');
+    expect(head.Metadata?.stream_id).toBe(stream.id);
+    expect(head.Metadata?.chunk_count).toBe('3');
+
+    expect(gunzipSync(await getObjectBytes(key))).toEqual(Buffer.concat(chunks));
+
+    await deleteObject(key);
+  });
+
+  it('preserves seq order across many keyset pages (more chunks than one page)', async () => {
+    // CHUNK_PAGE_SIZE is 64; 150 chunks forces the keyset loop across three page seams, so a
+    // wrong afterSeq advance or page-boundary off-by-one would drop, dup, or reorder bytes.
+    const chunks = Array.from({length: 150}, (_, i) => ndjsonBody(outputLine(`line-${i}\n`)));
+    const identity = newIdentity();
+    const stream = await arrangeClosedStream(identity, {chunks});
+
+    const result = await runCompaction(stream.id);
+
+    const key = compactedKey(result);
+    expect(result.outcome === 'compacted' && result.chunkCount).toBe(150);
+    expect(gunzipSync(await getObjectBytes(key))).toEqual(Buffer.concat(chunks));
+    expect((await headObject(key)).Metadata?.chunk_count).toBe('150');
+    expect(await listChunks(stream.id)).toHaveLength(0);
+
+    await deleteObject(key);
+  });
+
+  it('is a no-op on re-run once the object key is set (idempotent / crash-safe)', async () => {
+    const stream = await arrangeClosedStream(newIdentity(), {
+      chunks: [ndjsonBody(outputLine('x\n'))],
+    });
+    const key = compactedKey(await runCompaction(stream.id));
+
+    const result = await runCompaction(stream.id);
+
+    expect(result.outcome).toBe('already-compacted');
+    await deleteObject(key);
+  });
+
+  it('produces a valid empty object for a stream with no chunks', async () => {
+    const stream = await arrangeClosedStream(newIdentity(), {chunks: []});
+
+    const result = await runCompaction(stream.id);
+
+    const key = compactedKey(result);
+    expect(gunzipSync(await getObjectBytes(key))).toHaveLength(0);
+    expect((await headObject(key)).Metadata?.chunk_count).toBe('0');
+
+    await deleteObject(key);
+  });
+
+  it('compacts a tombstone-only (timeout-closed) stream', async () => {
+    const stream = await arrangeClosedStream(newIdentity(), {tombstone: true});
+
+    const result = await runCompaction(stream.id);
+
+    const key = compactedKey(result);
+    expect(gunzipSync(await getObjectBytes(key)).toString('utf8')).toContain(
+      '"type":"runner_lost"',
+    );
+
+    await deleteObject(key);
+  });
+
+  it('returns gone when the stream row no longer exists', async () => {
+    const result = await runCompaction(crypto.randomUUID());
+
+    expect(result.outcome).toBe('gone');
+  });
+
+  it('throws, deletes its upload, and keeps the chunks when streamed totals disagree', async () => {
+    const identity = newIdentity();
+    const stream = await arrangeClosedStream(identity, {
+      chunks: [ndjsonBody(outputLine('a\n')), ndjsonBody(outputLine('b\n'))],
+    });
+    // Upload a (wrong) empty body whose stats claim zero chunks; the table has two.
+    compactedGzipStreamMock.mockReturnValueOnce({
+      body: Readable.from([]).pipe(createGzip()),
+      stats: {chunkCount: 0, lastSeq: 0, uncompressedBytes: 0},
+    });
+
+    await expect(runCompaction(stream.id, compactStreamActivityWithMocks)).rejects.toThrow(
+      'integrity check',
+    );
+
+    const after = await getAttemptStreamById(stream.id);
+    expect(after?.objectKey).toBeNull();
+    expect(await listChunks(stream.id)).toHaveLength(2);
+    expect(await listKeysUnderStream(identity)).toEqual([]);
+  });
+
+  it('deletes its own upload and reports superseded when another attempt won the publish', async () => {
+    const identity = newIdentity();
+    const stream = await arrangeClosedStream(identity, {chunks: [ndjsonBody(outputLine('x\n'))]});
+    // Simulate a concurrent attempt publishing first: the guarded update matches 0 rows while
+    // the row still exists.
+    setObjectKeyAndDeleteChunksMock.mockResolvedValueOnce({updated: false});
+
+    const result = await runCompaction(stream.id, compactStreamActivityWithMocks);
+
+    expect(result.outcome).toBe('superseded');
+    expect(await listKeysUnderStream(identity)).toEqual([]);
+  });
+
+  it('deletes the orphaned object and reports retention-raced when the row vanished mid-upload', async () => {
+    const identity = newIdentity();
+    const stream = await arrangeClosedStream(identity, {chunks: [ndjsonBody(outputLine('x\n'))]});
+    // Simulate retention hard-deleting the row mid-upload: the guarded update finds 0 rows.
+    setObjectKeyAndDeleteChunksMock.mockImplementationOnce(async (_tx, params) => {
+      await db().delete(attemptStreams).where(eq(attemptStreams.id, params.streamId));
+      return {updated: false};
+    });
+
+    const result = await runCompaction(stream.id, compactStreamActivityWithMocks);
+
+    expect(result.outcome).toBe('retention-raced');
+    expect(await listKeysUnderStream(identity)).toEqual([]);
+  });
+});

package/src/temporal/activities/compact-stream.ts

@@ -0,0 +1,123 @@
+import {Context} from '@temporalio/activity';
+import {compactedObjectKey, deleteObject, putCompactedObject} from '#api/object-storage.js';
+import {compactedGzipStream} from '#core/compaction.js';
+import {chunkStats} from '#db/chunks.js';
+import {db, type Transaction} from '#db/db.js';
+import {getAttemptStreamById, setObjectKeyAndDeleteChunks} from '#db/streams.js';
+import {type CompactionMetricOutcome, compactionCount} from '#metrics/instance.js';
+
+export type CompactStreamResult =
+  | {outcome: 'gone'}
+  | {outcome: 'already-compacted'}
+  | {outcome: 'superseded'}
+  | {outcome: 'retention-raced'}
+  | {outcome: 'compacted'; objectKey: string; chunkCount: number; uncompressedBytes: number};
+
+interface CompactStreamDependencies {
+  compactedGzipStream: typeof compactedGzipStream;
+  setObjectKeyAndDeleteChunks: (
+    tx: Transaction,
+    params: {streamId: string; objectKey: string},
+  ) => Promise<{updated: boolean}>;
+}
+
+const defaultDependencies: CompactStreamDependencies = {
+  compactedGzipStream,
+  setObjectKeyAndDeleteChunks,
+};
+
+/**
+ * Compacts one closed stream into a single gzip NDJSON object, then deletes its chunk rows.
+ *
+ *   load stream ──┬─ gone ─────────────► no-op (retention raced)
+ *                 ├─ object_key set ───► no-op (idempotent re-run)
+ *                 └─ else: upload to a per-attempt key ─► gzip ─► multipart (abort-aware, heartbeat)
+ *                          │
+ *                 verify streamed count/maxSeq/bytes == table  (mismatch ─► delete upload, throw, retry)
+ *                          │
+ *                 tx: set object_key (state='closed' AND object_key IS NULL) + delete chunks
+ *                          └─ 0 rows ─► delete this attempt's upload, then re-read the row:
+ *                                         gone ─► retention raced · keyed ─► superseded by another attempt
+ *
+ * Each attempt uploads to its own `compactedObjectKey(stream, uuid)`, so a slow or zombie
+ * attempt can never overwrite a published object. The single-winner publish (object_key set
+ * + chunk delete, atomic) drops chunks only once a complete object is durable; the integrity
+ * check (count, maxSeq, and byte total) guards a read bug from publishing a truncated object
+ * before the only copy of the source is gone (S3 part checksums cover byte transfer).
+ */
+async function compactStream(
+  params: {streamId: string},
+  dependencies: CompactStreamDependencies,
+): Promise<CompactStreamResult> {
+  const stream = await getAttemptStreamById(params.streamId);
+  if (!stream) return {outcome: 'gone'};
+  if (stream.objectKey) return {outcome: 'already-compacted'};
+
+  const ctx = Context.current();
+  const uploadKey = compactedObjectKey(stream, crypto.randomUUID());
+  const expected = await chunkStats(stream.id);
+  const {body, stats} = dependencies.compactedGzipStream({
+    streamId: stream.id,
+    onPage: () => ctx.heartbeat(),
+  });
+
+  await putCompactedObject({
+    key: uploadKey,
+    body,
+    signal: ctx.cancellationSignal,
+    onProgress: () => ctx.heartbeat(),
+    metadata: {
+      stream_id: stream.id,
+      chunk_count: String(expected.count),
+      uncompressed_bytes: String(expected.uncompressedBytes),
+      last_seq: String(expected.maxSeq),
+    },
+  });
+
+  if (
+    stats.chunkCount !== expected.count ||
+    stats.lastSeq !== expected.maxSeq ||
+    stats.uncompressedBytes !== expected.uncompressedBytes
+  ) {
+    await deleteObject(uploadKey).catch(() => undefined);
+    throw new Error(
+      `Compaction integrity check failed for stream ${stream.id}: streamed ${stats.chunkCount} chunks / ${stats.uncompressedBytes} bytes up to seq ${stats.lastSeq}, table holds ${expected.count} / ${expected.uncompressedBytes} bytes up to seq ${expected.maxSeq}`,
+    );
+  }
+
+  const {updated} = await db().transaction((tx) =>
+    dependencies.setObjectKeyAndDeleteChunks(tx, {
+      streamId: stream.id,
+      objectKey: uploadKey,
+    }),
+  );
+  if (!updated) {
+    await deleteObject(uploadKey);
+    const current = await getAttemptStreamById(stream.id);
+    return {outcome: current ? 'superseded' : 'retention-raced'};
+  }
+
+  return {
+    outcome: 'compacted',
+    objectKey: uploadKey,
+    chunkCount: stats.chunkCount,
+    uncompressedBytes: stats.uncompressedBytes,
+  };
+}
+
+export function createCompactStreamActivity(
+  dependencies: CompactStreamDependencies = defaultDependencies,
+): (params: {streamId: string}) => Promise<CompactStreamResult> {
+  return async (params) => {
+    let outcome: CompactionMetricOutcome = 'failed';
+    try {
+      const result = await compactStream(params, dependencies);
+      outcome = result.outcome;
+      return result;
+    } finally {
+      compactionCount.add(1, {outcome});
+    }
+  };
+}
+
+export const compactStreamActivity = createCompactStreamActivity();

package/src/temporal/activities/compaction-reconcile.test.ts

@@ -0,0 +1,143 @@
+import {eq, sql} from 'drizzle-orm';
+import {db} from '#db/db.js';
+import {attemptStreams} from '#db/schema/attempt-streams.js';
+import {LOGS_COMPACTION_TASK_QUEUE} from '#temporal/constants.js';
+import {arrangeClosedStream, type ClosedStreamIdentity} from '#test/fixtures/closed-stream.js';
+import {ndjsonBody, outputLine} from '#test/fixtures/ndjson.js';
+import {compactionReconcileActivity} from './compaction-reconcile.js';
+
+const startMock = vi.fn();
+
+vi.mock('@shipfox/node-temporal', () => ({
+  temporalClient: () => ({
+    workflow: {
+      start: startMock,
+    },
+  }),
+}));
+
+function newIdentity(): ClosedStreamIdentity {
+  return {
+    jobId: crypto.randomUUID(),
+    stepId: crypto.randomUUID(),
+    attempt: 1,
+    workspaceId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    workflowRunAttemptId: crypto.randomUUID(),
+  };
+}
+
+async function backdateClosedAt(streamId: string): Promise<void> {
+  await db()
+    .update(attemptStreams)
+    .set({closedAt: sql`now() - interval '1 hour'`})
+    .where(eq(attemptStreams.id, streamId));
+}
+
+async function markCompacted(streamId: string): Promise<void> {
+  await db()
+    .update(attemptStreams)
+    .set({objectKey: `logs/test/${streamId}`})
+    .where(eq(attemptStreams.id, streamId));
+}
+
+function alreadyStartedError(): Error {
+  const error = new Error('Workflow execution already started');
+  error.name = 'WorkflowExecutionAlreadyStartedError';
+  return error;
+}
+
+describe('compactionReconcileActivity', () => {
+  beforeEach(() => {
+    startMock.mockReset();
+    startMock.mockResolvedValue({});
+  });
+
+  it('re-starts compaction for a closed, uncompacted stream past the stale window', async () => {
+    const stream = await arrangeClosedStream(newIdentity(), {
+      chunks: [ndjsonBody(outputLine('x\n'))],
+    });
+    await backdateClosedAt(stream.id);
+
+    await compactionReconcileActivity();
+
+    expect(startMock).toHaveBeenCalledWith('compactStream', {
+      taskQueue: LOGS_COMPACTION_TASK_QUEUE,
+      workflowId: `logs-compact:${stream.id}`,
+      args: [{streamId: stream.id}],
+    });
+  });
+
+  it('does not re-start a stream closed too recently to be stale', async () => {
+    const stream = await arrangeClosedStream(newIdentity(), {
+      chunks: [ndjsonBody(outputLine('x\n'))],
+    });
+
+    await compactionReconcileActivity();
+
+    expect(startMock).not.toHaveBeenCalledWith(
+      'compactStream',
+      expect.objectContaining({workflowId: `logs-compact:${stream.id}`}),
+    );
+  });
+
+  it('does not re-start an already-compacted stream', async () => {
+    const stream = await arrangeClosedStream(newIdentity(), {
+      chunks: [ndjsonBody(outputLine('x\n'))],
+    });
+    await backdateClosedAt(stream.id);
+    await markCompacted(stream.id);
+
+    await compactionReconcileActivity();
+
+    expect(startMock).not.toHaveBeenCalledWith(
+      'compactStream',
+      expect.objectContaining({workflowId: `logs-compact:${stream.id}`}),
+    );
+  });
+
+  it('swallows an already-started workflow so a still-running compaction is left alone', async () => {
+    const stream = await arrangeClosedStream(newIdentity(), {
+      chunks: [ndjsonBody(outputLine('x\n'))],
+    });
+    await backdateClosedAt(stream.id);
+    startMock.mockRejectedValue(alreadyStartedError());
+
+    await expect(compactionReconcileActivity()).resolves.toEqual(
+      expect.objectContaining({restarted: expect.any(Number)}),
+    );
+  });
+
+  it('logs and skips one stream whose start fails, still re-driving the rest of the batch', async () => {
+    const poison = await arrangeClosedStream(newIdentity(), {
+      chunks: [ndjsonBody(outputLine('a\n'))],
+    });
+    const healthy = await arrangeClosedStream(newIdentity(), {
+      chunks: [ndjsonBody(outputLine('b\n'))],
+    });
+    // Poison sorts first (oldest closed_at); if the loop aborted on its failure, healthy would
+    // never be attempted, which is exactly the regression this guards.
+    await db()
+      .update(attemptStreams)
+      .set({closedAt: sql`now() - interval '2 hours'`})
+      .where(eq(attemptStreams.id, poison.id));
+    await backdateClosedAt(healthy.id);
+    startMock.mockImplementation((_name, opts) =>
+      opts.workflowId === `logs-compact:${poison.id}`
+        ? Promise.reject(new Error('temporal namespace rate-limited'))
+        : Promise.resolve({}),
+    );
+
+    const result = await compactionReconcileActivity();
+
+    expect(startMock).toHaveBeenCalledWith(
+      'compactStream',
+      expect.objectContaining({workflowId: `logs-compact:${poison.id}`}),
+    );
+    expect(startMock).toHaveBeenCalledWith(
+      'compactStream',
+      expect.objectContaining({workflowId: `logs-compact:${healthy.id}`}),
+    );
+    expect(result.failed).toBeGreaterThanOrEqual(1);
+  });
+});