@shipfox/api-logs 2.0.0

package/src/config.ts

@@ -0,0 +1,130 @@
+import {config as authConfig} from '@shipfox/api-auth/config';
+import {bool, createConfig, num, str, url} from '@shipfox/config';
+import {durationToSeconds} from '@shipfox/node-jwt';
+
+export const config = createConfig({
+  LOG_STORAGE_S3_ENDPOINT: url({
+    desc: 'Endpoint URL of the S3-compatible object store that holds compacted logs. Defaults to the bundled local-development Garage (http://localhost:3900); set it to your object store endpoint for production.',
+    default: 'http://localhost:3900',
+  }),
+  LOG_STORAGE_S3_REGION: str({
+    desc: 'Region passed to the S3 client. Any value works for Garage; set the real region for AWS S3. Defaults to garage for local development.',
+    default: 'garage',
+  }),
+  LOG_STORAGE_S3_BUCKET: str({
+    desc: 'Name of the bucket that stores compacted log objects. Defaults to shipfox-logs (created by dev/garage/bootstrap.sh); create the bucket and set this for production.',
+    default: 'shipfox-logs',
+  }),
+  LOG_STORAGE_S3_PREFIX: str({
+    desc: 'Key prefix under which compacted log objects are stored in the bucket. Set this to host several modules in one bucket, each under its own prefix. Use a value without a leading or trailing slash. Defaults to logs.',
+    default: 'logs',
+  }),
+  LOG_STORAGE_S3_ACCESS_KEY_ID: str({
+    desc: 'Optional access key ID used to authenticate to the object store. Set it together with LOG_STORAGE_S3_SECRET_ACCESS_KEY for an explicit credential pair, or leave both unset to use the standard AWS SDK credential provider chain.',
+    default: undefined,
+  }),
+  LOG_STORAGE_S3_SECRET_ACCESS_KEY: str({
+    desc: 'Optional secret access key used to authenticate to the object store. Set it together with LOG_STORAGE_S3_ACCESS_KEY_ID for an explicit credential pair, or leave both unset to use the standard AWS SDK credential provider chain.',
+    default: undefined,
+  }),
+  LOG_STORAGE_S3_FORCE_PATH_STYLE: bool({
+    desc: 'Whether to address the bucket as a path (endpoint/bucket) instead of a subdomain. Set it to true for Garage and MinIO; false works for AWS S3.',
+    default: true,
+  }),
+  LOG_BUDGET_BASE_BYTES: num({
+    desc: 'Base of the per-job log accrual budget, in stored bytes (normalized NDJSON the server keeps, framing included). A job may always store this much before the time-based rate is added. Sized to hold a few inline agent_session entries (such as base64 images) before the shared per-job cap trips. Defaults to 32 MiB.',
+    default: 33_554_432,
+  }),
+  LOG_BUDGET_RATE_BYTES_PER_MINUTE: num({
+    desc: 'Stored bytes added to the per-job log budget for each minute since the first log append. Defaults to 1 MiB per minute.',
+    default: 1_048_576,
+  }),
+  LOG_STREAM_CLOSE_GRACE_SECONDS: num({
+    desc: 'How long to wait after a job reaches a terminal state before force-closing any of its log streams the runner never ended itself (it died, was capped, or its log spool failed). The wait lets a last in-flight chunk land before the stream is marked truncated. Defaults to 120 seconds.',
+    default: 120,
+  }),
+  LOG_COMPACTION_RECONCILE_STALE_SECONDS: num({
+    desc: 'How long a closed log stream may stay uncompacted before the reconcile cron re-drives it. The event-triggered compaction normally runs within seconds of close; this backstop only re-drives streams whose compaction never started or permanently failed. Defaults to 900 seconds (15 minutes).',
+    default: 900,
+  }),
+  LOG_STREAM_REAP_AFTER_SECONDS: num({
+    desc: 'How long a log stream may stay open before the reaper cron force-closes it as abandoned. This is the backstop for a stream a runner started after its job already went terminal (the one-shot close sweep had already run); the reaper marks it truncated so it re-enters the compaction and retention lifecycle. Must be greater than AUTH_JOB_LEASE_TOKEN_EXPIRES_IN (the job lease lifetime, default 90 minutes) so a still-valid lease can never be force-closed mid-append, which would silently truncate live logs. Defaults to 7200 seconds (2 hours).',
+    default: 7200,
+  }),
+  LOG_APPEND_BODY_LIMIT_BYTES: num({
+    desc: 'Maximum size of a single log append request body, in bytes. Output records cap at 16 KiB each, but one agent_session record carries a whole session entry (which can inline base64 images), so the body must hold a full LOG_MAX_SESSION_LINE_BYTES line plus framing. This also bounds how far a job can overshoot its log budget: the one append that crosses the budget is stored in full before the job is capped. Defaults to 8 MiB.',
+    default: 8_388_608,
+  }),
+  LOG_MAX_SESSION_LINE_BYTES: num({
+    desc: 'Maximum size of one agent_session log line (a verbatim agent session entry), in bytes. A larger line is rejected with 400, and the runner drops it with a gap marker instead of sending it. Sized generously for entries that inline large content such as base64 images. Must be at most LOG_APPEND_BODY_LIMIT_BYTES, since a line that cannot fit in one request body could never be accepted. Defaults to 4 MiB.',
+    default: 4_194_304,
+  }),
+  LOG_READ_URL_TTL_SECONDS: num({
+    desc: 'Lifetime, in seconds, of a presigned GET URL handed out for a compacted (cold) log read. The browser must finish fetching the object before it expires; raise it for slow links or large logs. Must be between 1 and 604800 (7 days, the longest a presigned URL can live). Defaults to 3600 (one hour).',
+    default: 3600,
+  }),
+  LOG_READ_INLINE_MAX_BYTES: num({
+    desc: 'Maximum number of stored log bytes returned in a single inline (hot) read response. A long-running step can buffer far more than this in Postgres before compaction, so a read is capped here and the client re-polls (following has_more) to drain the backlog before it tails. Bounds per-request memory. Must be at least 1. Defaults to 1 MiB.',
+    default: 1_048_576,
+  }),
+  LOG_RETENTION_DAYS: num({
+    desc: 'How many days a closed log stream is kept before the retention cron hard-deletes its stored object and database row. Our own worker enforces this (not bucket lifecycle rules), so behavior is identical across object stores. Must be a whole number of days, 1 or greater. Defaults to 90 days.',
+    default: 90,
+  }),
+});
+
+const hasS3AccessKeyId = Boolean(config.LOG_STORAGE_S3_ACCESS_KEY_ID);
+const hasS3SecretAccessKey = Boolean(config.LOG_STORAGE_S3_SECRET_ACCESS_KEY);
+if (hasS3AccessKeyId !== hasS3SecretAccessKey) {
+  throw new Error(
+    'LOG_STORAGE_S3_ACCESS_KEY_ID and LOG_STORAGE_S3_SECRET_ACCESS_KEY must be set together or both left unset.',
+  );
+}
+
+// SigV4 caps a presigned URL's lifetime at 7 days, so a larger TTL would fail at signing
+// time on every cold read. Reject out-of-range values at startup rather than per request.
+const MAX_PRESIGN_TTL_SECONDS = 604_800;
+if (
+  config.LOG_READ_URL_TTL_SECONDS < 1 ||
+  config.LOG_READ_URL_TTL_SECONDS > MAX_PRESIGN_TTL_SECONDS
+) {
+  throw new Error(
+    `LOG_READ_URL_TTL_SECONDS must be between 1 and ${MAX_PRESIGN_TTL_SECONDS}; got ${config.LOG_READ_URL_TTL_SECONDS}`,
+  );
+}
+if (config.LOG_READ_INLINE_MAX_BYTES < 1) {
+  throw new Error(
+    `LOG_READ_INLINE_MAX_BYTES must be at least 1; got ${config.LOG_READ_INLINE_MAX_BYTES}`,
+  );
+}
+
+// A whole agent_session line is one record in one body. If the body limit is below the
+// line cap, a legitimate large session line could never fit in one request body — Fastify
+// would 413 it before the per-line validator runs. Fail fast at startup instead.
+if (config.LOG_APPEND_BODY_LIMIT_BYTES < config.LOG_MAX_SESSION_LINE_BYTES) {
+  throw new Error(
+    `LOG_APPEND_BODY_LIMIT_BYTES (${config.LOG_APPEND_BODY_LIMIT_BYTES}) must be >= LOG_MAX_SESSION_LINE_BYTES (${config.LOG_MAX_SESSION_LINE_BYTES}).`,
+  );
+}
+
+// `num` accepts zero, negative, and fractional values; reject them before they reach
+// `make_interval(days => ...)` and hard-delete the wrong streams.
+if (!Number.isInteger(config.LOG_RETENTION_DAYS) || config.LOG_RETENTION_DAYS < 1) {
+  throw new Error(
+    `LOG_RETENTION_DAYS (${config.LOG_RETENTION_DAYS}) must be a whole number of days >= 1.`,
+  );
+}
+
+// A stream still accepts appends until its job lease expires, so reaping it any sooner would
+// truncate live logs. Validate against the real lease TTL read from auth's config (not a
+// hardcoded floor), so raising AUTH_JOB_LEASE_TOKEN_EXPIRES_IN can never silently outrun the
+// reaper window.
+const jobLeaseTtlSeconds = durationToSeconds(authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN);
+if (
+  !Number.isFinite(config.LOG_STREAM_REAP_AFTER_SECONDS) ||
+  config.LOG_STREAM_REAP_AFTER_SECONDS <= jobLeaseTtlSeconds
+) {
+  throw new Error(
+    `LOG_STREAM_REAP_AFTER_SECONDS (${config.LOG_STREAM_REAP_AFTER_SECONDS}) must be greater than the job lease TTL (${jobLeaseTtlSeconds}s, from AUTH_JOB_LEASE_TOKEN_EXPIRES_IN=${authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN}); a smaller value would let the reaper force-close a stream a still-valid lease is appending to and silently truncate live logs.`,
+  );
+}

package/src/core/append-logs.test.ts

@@ -0,0 +1,563 @@
+import {type LogRecord, parseLogRecordLine} from '@shipfox/api-logs-dto';
+import {appendLogs, setStepLookupForTesting} from '#core/append-logs.js';
+import {LeaseStreamMismatchError, MalformedLogChunkError, OffsetGapError} from '#core/errors.js';
+import {jobAccountingFactory} from '#test/factories/job-accounting.js';
+import {
+  endLine,
+  groupStartLine,
+  ndjsonBody,
+  outputLine,
+  outputOfBytes,
+  recordLine,
+  sessionLine,
+} from '#test/fixtures/ndjson.js';
+import {findAccounting, findStream, listChunks, listStreamClosedEvents} from '#test/queries.js';
+
+const mockSteps = new Map<string, {config: Record<string, unknown>}>();
+
+interface Ctx {
+  jobId: string;
+  stepId: string;
+  workspaceId: string;
+  projectId: string;
+  workflowRunAttemptId: string;
+}
+
+function newCtx(): Ctx {
+  return {
+    jobId: crypto.randomUUID(),
+    stepId: crypto.randomUUID(),
+    workspaceId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    workflowRunAttemptId: crypto.randomUUID(),
+  };
+}
+
+async function allowLargeLogBudget(ctx: Ctx): Promise<void> {
+  await jobAccountingFactory.create({
+    jobId: ctx.jobId,
+    workspaceId: ctx.workspaceId,
+    startedAt: new Date(Date.now() - 5 * 60_000),
+  });
+}
+
+function recordsFromChunks(chunks: Awaited<ReturnType<typeof listChunks>>): LogRecord[] {
+  return chunks.flatMap((chunk) =>
+    chunk.data.toString('utf8').split('\n').filter(Boolean).map(parseLogRecordLine),
+  );
+}
+
+describe('appendLogs', () => {
+  let restoreStepLookup: () => void;
+
+  beforeAll(() => {
+    restoreStepLookup = setStepLookupForTesting((stepId) => Promise.resolve(mockSteps.get(stepId)));
+  });
+
+  afterAll(() => {
+    restoreStepLookup();
+  });
+
+  beforeEach(() => {
+    mockSteps.clear();
+  });
+
+  describe('offset-CAS', () => {
+    it('extends committed_length and stores one runner chunk on an in-order append', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(outputLine('hello\n'));
+
+      const result = await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      expect(result).toEqual({committedLength: body.length, capped: false});
+      const stream = await findStream({...ctx, attempt: 1});
+      expect(stream?.committedLength).toBe(body.length);
+      const chunks = await listChunks(stream?.id as string);
+      expect(chunks).toHaveLength(1);
+      expect(chunks[0]?.origin).toBe('runner');
+    });
+
+    it('acks a re-sent (offset < committed) append without storing a new chunk', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(outputLine('hello\n'));
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      const result = await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      expect(result).toEqual({committedLength: body.length, capped: false});
+      const stream = await findStream({...ctx, attempt: 1});
+      expect(await listChunks(stream?.id as string)).toHaveLength(1);
+    });
+
+    it('rejects a gap (offset > committed) with the committed length', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(outputLine('hello\n'));
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      const error = await appendLogs({
+        ...ctx,
+        attempt: 1,
+        offset: body.length + 5,
+        body: ndjsonBody(outputLine('more\n')),
+      }).catch((e: unknown) => e);
+
+      expect(error).toBeInstanceOf(OffsetGapError);
+      expect((error as OffsetGapError).committedLength).toBe(body.length);
+    });
+
+    it('rejects a straddling append (offset before committed but extending past it)', async () => {
+      const ctx = newCtx();
+      const first = ndjsonBody(outputLine('hello\n'));
+      await appendLogs({...ctx, attempt: 1, offset: 0, body: first});
+
+      const error = await appendLogs({
+        ...ctx,
+        attempt: 1,
+        offset: first.length - 2,
+        body: ndjsonBody(outputLine('more\n')),
+      }).catch((e: unknown) => e);
+
+      expect(error).toBeInstanceOf(OffsetGapError);
+      expect((error as OffsetGapError).committedLength).toBe(first.length);
+    });
+
+    it('treats an empty body as a heartbeat that creates no stream', async () => {
+      const ctx = newCtx();
+
+      const result = await appendLogs({...ctx, attempt: 1, offset: 0, body: ndjsonBody()});
+
+      expect(result).toEqual({committedLength: 0, capped: false});
+      expect(await findStream({...ctx, attempt: 1})).toBeNull();
+    });
+  });
+
+  describe('agent_session', () => {
+    it('stores a normalized session line as one runner chunk and leaves the stream open', async () => {
+      const ctx = newCtx();
+      await allowLargeLogBudget(ctx);
+      const body = ndjsonBody(sessionLine('{"type":"x"}'));
+
+      const result = await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      expect(result).toEqual({committedLength: body.length, capped: false});
+      const stream = await findStream({...ctx, attempt: 1});
+      expect(stream?.state).toBe('open');
+      const chunks = await listChunks(stream?.id as string);
+      expect(chunks).toHaveLength(1);
+      expect(chunks[0]?.origin).toBe('runner');
+      expect(recordsFromChunks(chunks)).toEqual([
+        {
+          v: 1,
+          ts: 1,
+          type: 'agent_session',
+          row: {
+            kind: 'raw',
+            timestamp: 1,
+            label: 'Unknown session entry: x',
+            raw: '{"type":"x"}',
+          },
+        },
+      ]);
+    });
+
+    it('stores parsed pi session rows in append order', async () => {
+      const ctx = newCtx();
+      await allowLargeLogBudget(ctx);
+      const first = sessionLine(
+        JSON.stringify({type: 'session', id: 'session-1', cwd: '/workspace'}),
+      );
+      const second = sessionLine(
+        JSON.stringify({type: 'message', message: {role: 'assistant', content: 'Done.'}}),
+      );
+      const body = ndjsonBody(first, second);
+
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      const stream = await findStream({...ctx, attempt: 1});
+      const rows = recordsFromChunks(await listChunks(stream?.id as string)).map((record) => {
+        expect(record.type).toBe('agent_session');
+        return record.type === 'agent_session' ? record.row : null;
+      });
+      expect(rows).toEqual([
+        {
+          kind: 'lifecycle',
+          timestamp: 1,
+          label: 'Session started',
+          detail: 'session-1 · /workspace',
+          meta: [],
+          tone: 'default',
+          terminalFailure: false,
+        },
+        {
+          kind: 'message',
+          timestamp: 1,
+          role: 'assistant',
+          label: 'assistant',
+          meta: [],
+          text: 'Done.',
+          terminalFailure: false,
+        },
+      ]);
+    });
+
+    it('selects the Claude parser from the step harness', async () => {
+      const ctx = newCtx();
+      await allowLargeLogBudget(ctx);
+      mockSteps.set(ctx.stepId, {config: {harness: 'claude'}});
+      const body = ndjsonBody(
+        sessionLine(
+          JSON.stringify({
+            type: 'assistant',
+            message: {
+              role: 'assistant',
+              content: [{type: 'tool_use', id: 'tool-1', name: 'Read', input: {path: 'a.ts'}}],
+            },
+          }),
+        ),
+      );
+
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      const stream = await findStream({...ctx, attempt: 1});
+      const rows = recordsFromChunks(await listChunks(stream?.id as string)).map((record) => {
+        expect(record.type).toBe('agent_session');
+        return record.type === 'agent_session' ? record.row : null;
+      });
+      expect(rows).toEqual([
+        {
+          kind: 'tool-call',
+          timestamp: 1,
+          id: 'tool-1',
+          name: 'Read',
+          input: '{\n  "path": "a.ts"\n}',
+        },
+      ]);
+    });
+
+    it('does not duplicate parsed rows on a retried append', async () => {
+      const ctx = newCtx();
+      await allowLargeLogBudget(ctx);
+      const body = ndjsonBody(sessionLine(JSON.stringify({type: 'session', id: 'session-1'})));
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      const stream = await findStream({...ctx, attempt: 1});
+      expect(recordsFromChunks(await listChunks(stream?.id as string))).toHaveLength(1);
+    });
+
+    it('does not store normalized session records when a capped append is dropped', async () => {
+      const ctx = newCtx();
+      const first = outputOfBytes(150);
+      await appendLogs({...ctx, attempt: 1, offset: 0, body: first});
+      const straggler = ndjsonBody(sessionLine(JSON.stringify({type: 'session', id: 'dropped'})));
+
+      await appendLogs({...ctx, attempt: 1, offset: first.length, body: straggler});
+
+      const stream = await findStream({...ctx, attempt: 1});
+      expect(recordsFromChunks(await listChunks(stream?.id as string))).not.toContainEqual(
+        expect.objectContaining({type: 'agent_session'}),
+      );
+    });
+
+    it('rejects a session line over LOG_MAX_SESSION_LINE_BYTES before any stream is created', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(sessionLine('x'.repeat(600)));
+
+      const error = await appendLogs({...ctx, attempt: 1, offset: 0, body}).catch(
+        (e: unknown) => e,
+      );
+
+      expect(error).toBeInstanceOf(MalformedLogChunkError);
+      expect(await findStream({...ctx, attempt: 1})).toBeNull();
+    });
+  });
+
+  describe('budget accounting', () => {
+    it('charges stored bytes, envelope and control records included', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(outputLine('abc'), groupStartLine('g1', 'Build'));
+
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      const accounting = await findAccounting(ctx.jobId);
+      expect(accounting?.storedBytesUsed).toBe(body.length);
+    });
+
+    it('stays under cap when accrual from elapsed time covers the payload', async () => {
+      const ctx = newCtx();
+      await jobAccountingFactory.create({
+        jobId: ctx.jobId,
+        workspaceId: ctx.workspaceId,
+        startedAt: new Date(Date.now() - 5 * 60_000),
+      });
+
+      const result = await appendLogs({...ctx, attempt: 1, offset: 0, body: outputOfBytes(150)});
+
+      expect(result.capped).toBe(false);
+      expect((await findAccounting(ctx.jobId))?.cappedAt).toBeNull();
+    });
+  });
+
+  describe('cap', () => {
+    it('caps when the payload crosses the budget, injecting a control tombstone', async () => {
+      const ctx = newCtx();
+
+      const result = await appendLogs({...ctx, attempt: 1, offset: 0, body: outputOfBytes(150)});
+
+      expect(result.capped).toBe(true);
+      expect((await findAccounting(ctx.jobId))?.cappedAt).not.toBeNull();
+      const stream = await findStream({...ctx, attempt: 1});
+      const chunks = await listChunks(stream?.id as string);
+      expect(chunks.map((c) => c.origin)).toEqual(['runner', 'control']);
+    });
+
+    it('drops a post-cap straggler but still advances committed_length', async () => {
+      const ctx = newCtx();
+      const first = outputOfBytes(150);
+      await appendLogs({...ctx, attempt: 1, offset: 0, body: first});
+      const straggler = ndjsonBody(outputLine('late\n'));
+
+      const result = await appendLogs({...ctx, attempt: 1, offset: first.length, body: straggler});
+
+      expect(result).toEqual({committedLength: first.length + straggler.length, capped: true});
+      const stream = await findStream({...ctx, attempt: 1});
+      const chunks = await listChunks(stream?.id as string);
+      expect(chunks.map((c) => c.origin)).toEqual(['runner', 'control']);
+    });
+
+    it('does not cap when stored bytes land exactly on the budget', async () => {
+      const ctx = newCtx();
+      const body = outputOfBytes(9);
+      // Test env base budget is 100 bytes; pre-fill so this append lands used == allowed.
+      await jobAccountingFactory.create({
+        jobId: ctx.jobId,
+        workspaceId: ctx.workspaceId,
+        storedBytesUsed: 100 - body.length,
+        startedAt: new Date(),
+      });
+
+      const result = await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      expect(result.capped).toBe(false);
+      expect((await findAccounting(ctx.jobId))?.cappedAt).toBeNull();
+    });
+  });
+
+  describe('stream lifecycle', () => {
+    it('declared-closes the stream and emits one stream-closed event on an end record', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(outputLine('done\n'), endLine(12345));
+
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      const stream = await findStream({...ctx, attempt: 1});
+      expect(stream?.state).toBe('closed');
+      expect(stream?.closeReason).toBe('declared');
+      expect(stream?.truncated).toBe(false);
+      expect(stream?.declaredTotalBytes).toBe(12345);
+      expect(stream?.closedAt).not.toBeNull();
+      expect(await listStreamClosedEvents(stream?.id as string)).toHaveLength(1);
+    });
+
+    it('is idempotent: a re-sent end body neither re-closes nor emits a second event', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(outputLine('done\n'), endLine(10));
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+      const stream = await findStream({...ctx, attempt: 1});
+
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      expect(await listStreamClosedEvents(stream?.id as string)).toHaveLength(1);
+    });
+
+    it('drops further output once declared-closed (no new chunk, committed_length frozen)', async () => {
+      const ctx = newCtx();
+      // End-only body so the single stored chunk stays under the 100-byte test budget
+      // (an extra output line would trip the cap and add a tombstone chunk).
+      const end = ndjsonBody(endLine(4));
+      await appendLogs({...ctx, attempt: 1, offset: 0, body: end});
+      const closed = await findStream({...ctx, attempt: 1});
+
+      const result = await appendLogs({
+        ...ctx,
+        attempt: 1,
+        offset: end.length,
+        body: ndjsonBody(outputLine('late\n')),
+      });
+
+      expect(result.committedLength).toBe(end.length);
+      const after = await findStream({...ctx, attempt: 1});
+      expect(after?.committedLength).toBe(closed?.committedLength);
+      expect(await listChunks(after?.id as string)).toHaveLength(1);
+    });
+
+    it('declared-closes when one body both crosses the budget and ends', async () => {
+      const ctx = newCtx();
+      // 150 payload bytes cross the 100-byte test budget, but the crossing chunk is still
+      // stored in full; the same body carries the end, so the stream declared-closes.
+      const body = ndjsonBody(outputLine('x'.repeat(150)), endLine(4));
+
+      const result = await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      expect(result.capped).toBe(true);
+      const stream = await findStream({...ctx, attempt: 1});
+      expect(stream?.state).toBe('closed');
+      expect(stream?.closeReason).toBe('declared');
+      expect(stream?.truncated).toBe(false);
+      expect((await listChunks(stream?.id as string)).map((c) => c.origin)).toEqual([
+        'runner',
+        'control',
+      ]);
+      expect(await listStreamClosedEvents(stream?.id as string)).toHaveLength(1);
+    });
+
+    it('does not declared-close when an already-capped job drops the end body', async () => {
+      const ctx = newCtx();
+      // Cap the job first (150 payload bytes cross the 100-byte budget), then send the end
+      // body: it is dropped, so the stream is not whole and must stay open for the sweep.
+      const first = outputOfBytes(150);
+      await appendLogs({...ctx, attempt: 1, offset: 0, body: first});
+      const end = ndjsonBody(endLine(4));
+
+      const result = await appendLogs({...ctx, attempt: 1, offset: first.length, body: end});
+
+      expect(result.capped).toBe(true);
+      const stream = await findStream({...ctx, attempt: 1});
+      expect(stream?.state).toBe('open');
+      expect(stream?.closeReason).toBeNull();
+      expect(stream?.declaredTotalBytes).toBeNull();
+      // The dropped end body persisted nothing: still just the runner chunk + cap tombstone.
+      expect(await listChunks(stream?.id as string)).toHaveLength(2);
+      expect(await listStreamClosedEvents(stream?.id as string)).toHaveLength(0);
+    });
+
+    it('keeps attempts of the same step on independent streams', async () => {
+      const ctx = newCtx();
+      const a1 = ndjsonBody(outputLine('one\n'));
+      const a2 = ndjsonBody(outputLine('two-two\n'));
+
+      await appendLogs({...ctx, attempt: 1, offset: 0, body: a1});
+      await appendLogs({...ctx, attempt: 2, offset: 0, body: a2});
+
+      expect((await findStream({...ctx, attempt: 1}))?.committedLength).toBe(a1.length);
+      expect((await findStream({...ctx, attempt: 2}))?.committedLength).toBe(a2.length);
+    });
+  });
+
+  describe('concurrency and isolation', () => {
+    it('serializes two concurrent first appends at offset 0 into one chunk', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(outputLine('race\n'));
+
+      const results = await Promise.all([
+        appendLogs({...ctx, attempt: 1, offset: 0, body}),
+        appendLogs({...ctx, attempt: 1, offset: 0, body}),
+      ]);
+
+      expect(results.every((r) => r.committedLength === body.length)).toBe(true);
+      const stream = await findStream({...ctx, attempt: 1});
+      expect(await listChunks(stream?.id as string)).toHaveLength(1);
+    });
+
+    it('claims the cap once when two steps of one job cross the budget concurrently', async () => {
+      const ctx = newCtx();
+      const stepA = crypto.randomUUID();
+      const stepB = crypto.randomUUID();
+
+      await Promise.all([
+        appendLogs({...ctx, stepId: stepA, attempt: 1, offset: 0, body: outputOfBytes(80)}),
+        appendLogs({...ctx, stepId: stepB, attempt: 1, offset: 0, body: outputOfBytes(80)}),
+      ]);
+
+      expect((await findAccounting(ctx.jobId))?.cappedAt).not.toBeNull();
+      const streamA = await findStream({jobId: ctx.jobId, stepId: stepA, attempt: 1});
+      const streamB = await findStream({jobId: ctx.jobId, stepId: stepB, attempt: 1});
+      const controls = [
+        ...(await listChunks(streamA?.id as string)),
+        ...(await listChunks(streamB?.id as string)),
+      ].filter((c) => c.origin === 'control');
+      expect(controls).toHaveLength(1);
+    });
+
+    it('rejects a second append whose lease workspace/project/run does not match the stamped row', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(outputLine('first\n'));
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+      const before = await findStream({...ctx, attempt: 1});
+
+      const error = await appendLogs({
+        ...ctx,
+        projectId: crypto.randomUUID(),
+        attempt: 1,
+        offset: body.length,
+        body: ndjsonBody(outputLine('more\n')),
+      }).catch((e: unknown) => e);
+
+      expect(error).toBeInstanceOf(LeaseStreamMismatchError);
+      // The wrapping transaction rolls back: updated_at and committed_length on
+      // the stamped row are untouched, so a stale lease cannot tick the
+      // stream's freshness or advance its CAS axis.
+      const after = await findStream({...ctx, attempt: 1});
+      expect(after?.updatedAt.getTime()).toBe(before?.updatedAt.getTime());
+      expect(after?.committedLength).toBe(before?.committedLength);
+    });
+
+    it('isolates streams by job: a different job with the same stepId gets its own stream', async () => {
+      const stepId = crypto.randomUUID();
+      const jobA = crypto.randomUUID();
+      const jobB = crypto.randomUUID();
+      const workspaceId = crypto.randomUUID();
+      const projectId = crypto.randomUUID();
+      const workflowRunAttemptId = crypto.randomUUID();
+      const bodyA = ndjsonBody(outputLine('a\n'));
+      const bodyB = ndjsonBody(outputLine('bbbb\n'));
+
+      const common = {
+        workspaceId,
+        projectId,
+        workflowRunAttemptId,
+        stepId,
+        attempt: 1,
+        offset: 0,
+      };
+      await appendLogs({...common, jobId: jobA, body: bodyA});
+      await appendLogs({...common, jobId: jobB, body: bodyB});
+
+      const streamA = await findStream({jobId: jobA, stepId, attempt: 1});
+      const streamB = await findStream({jobId: jobB, stepId, attempt: 1});
+      expect(streamA?.id).not.toBe(streamB?.id);
+      expect(streamA?.committedLength).toBe(bodyA.length);
+      expect(streamB?.committedLength).toBe(bodyB.length);
+    });
+  });
+
+  describe('write-path enforcement', () => {
+    it.each([
+      'capped',
+      'runner_lost',
+    ])('rejects a forged server-only %s tombstone before any row is created', async (type) => {
+      const ctx = newCtx();
+      const body = ndjsonBody(recordLine({type}));
+
+      const error = await appendLogs({...ctx, attempt: 1, offset: 0, body}).catch(
+        (e: unknown) => e,
+      );
+
+      expect(error).toBeInstanceOf(MalformedLogChunkError);
+      expect(await findStream({...ctx, attempt: 1})).toBeNull();
+    });
+
+    it('rejects an invalid NDJSON record', async () => {
+      const ctx = newCtx();
+      const recordWithoutData = recordLine({type: 'output', stream: 'stdout'});
+      const body = ndjsonBody(recordWithoutData);
+
+      const error = await appendLogs({...ctx, attempt: 1, offset: 0, body}).catch(
+        (e: unknown) => e,
+      );
+
+      expect(error).toBeInstanceOf(MalformedLogChunkError);
+    });
+  });
+});