@shipfox/api-logs 13.0.0 → 14.0.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shipfox/api-logs",
   "license": "MIT",
-  "version": "13.0.0",
+  "version": "14.0.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ShipfoxHQ/shipfox.git",
@@ -27,15 +27,15 @@
     "zod": "^4.4.3",
     "@shipfox/api-auth-context": "12.2.0",
     "@shipfox/api-logs-dto": "12.0.0",
-    "@shipfox/api-workflows-dto": "13.0.0",
-    "@shipfox/workflow-document": "3.0.1",
+    "@shipfox/api-workflows-dto": "14.0.0",
     "@shipfox/config": "1.2.4",
-    "@shipfox/node-drizzle": "0.3.5",
-    "@shipfox/node-error-monitoring": "0.3.0",
+    "@shipfox/workflow-document": "3.1.0",
     "@shipfox/node-fastify": "0.4.2",
-    "@shipfox/node-module": "1.0.6",
     "@shipfox/node-opentelemetry": "0.6.4",
     "@shipfox/node-outbox": "0.2.6",
+    "@shipfox/node-module": "1.0.6",
+    "@shipfox/node-error-monitoring": "0.3.0",
+    "@shipfox/node-drizzle": "0.3.5",
     "@shipfox/node-postgres": "0.5.0",
     "@shipfox/node-temporal": "0.4.5"
   },

package/src/core/append-logs.test.ts

@@ -16,6 +16,24 @@ import {
 } from '#test/fixtures/ndjson.js';
 import {findAccounting, findStream, listChunks, listStreamClosedEvents} from '#test/queries.js';
 
+const metricsMocks = vi.hoisted(() => {
+  const counters = new Map<string, {add: ReturnType<typeof vi.fn>}>();
+  const add = (name: string) => {
+    const counter = {add: vi.fn()};
+    counters.set(name, counter);
+    return counter;
+  };
+  return {counters, add};
+});
+
+vi.mock('#metrics/instance.js', () => ({
+  bytesIngestedCount: metricsMocks.add('bytesIngestedCount'),
+  bytesStoredCount: metricsMocks.add('bytesStoredCount'),
+  recordAppendedCount: metricsMocks.add('recordAppendedCount'),
+  streamClosedCount: metricsMocks.add('streamClosedCount'),
+  streamOpenedCount: metricsMocks.add('streamOpenedCount'),
+}));
+
 interface Ctx {
   jobId: string;
   stepId: string;
@@ -727,6 +745,141 @@ describe('appendLogs', () => {
     });
   });
 
+  describe('byte volume metrics', () => {
+    beforeEach(() => {
+      for (const counter of metricsMocks.counters.values()) counter.add.mockClear();
+    });
+
+    function ingestedAdd() {
+      return metricsMocks.counters.get('bytesIngestedCount')?.add;
+    }
+
+    function storedAdd() {
+      return metricsMocks.counters.get('bytesStoredCount')?.add;
+    }
+
+    it('counts raw ingested and normalized stored bytes on an in-order append', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(outputLine('hello\n'));
+
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      expect(ingestedAdd()).toHaveBeenCalledTimes(1);
+      expect(ingestedAdd()).toHaveBeenCalledWith(body.length);
+      expect(storedAdd()).toHaveBeenCalledTimes(1);
+      expect(storedAdd()).toHaveBeenCalledWith(body.length);
+    });
+
+    it('counts normalized stored bytes separately from raw ingested bytes', async () => {
+      const ctx = newCtx();
+      await allowLargeLogBudget(ctx);
+      // A session line is parsed into a view row before storage, so the durable chunk is
+      // byte-different from the raw body: the exact point of the raw-vs-normalized split.
+      const body = ndjsonBody(sessionLine('{"type":"x"}'));
+
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      expect(ingestedAdd()).toHaveBeenCalledTimes(1);
+      expect(ingestedAdd()).toHaveBeenCalledWith(body.length);
+      const storedAddMock = storedAdd();
+      if (!storedAddMock) throw new Error('Expected bytesStoredCount mock');
+      const storedBytes = storedAddMock.mock.calls[0]?.[0];
+      expect(typeof storedBytes).toBe('number');
+      expect(storedBytes).not.toBe(body.length);
+      const stream = await findStream({...ctx, attempt: 1});
+      const chunks = await listChunks(stream?.id as string);
+      expect(storedBytes).toBe(chunks[0]?.byteLen);
+    });
+
+    it('does not re-count bytes on a retried append', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(outputLine('hello\n'));
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      expect(ingestedAdd()).toHaveBeenCalledTimes(1);
+      expect(ingestedAdd()).toHaveBeenCalledWith(body.length);
+      expect(storedAdd()).toHaveBeenCalledTimes(1);
+    });
+
+    it('does not count bytes for a rejected gap append', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(outputLine('hello\n'));
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      await appendLogs({
+        ...ctx,
+        attempt: 1,
+        offset: body.length + 5,
+        body: ndjsonBody(outputLine('more\n')),
+      }).catch(() => undefined);
+
+      expect(ingestedAdd()).toHaveBeenCalledTimes(1);
+      expect(storedAdd()).toHaveBeenCalledTimes(1);
+    });
+
+    it('does not count bytes dropped on a closed stream', async () => {
+      const ctx = newCtx();
+      // End-only body so the single stored chunk stays under the 100-byte test budget.
+      const end = ndjsonBody(endLine(4));
+      await appendLogs({...ctx, attempt: 1, offset: 0, body: end});
+
+      await appendLogs({
+        ...ctx,
+        attempt: 1,
+        offset: end.length,
+        body: ndjsonBody(outputLine('late\n')),
+      });
+
+      expect(ingestedAdd()).toHaveBeenCalledTimes(1);
+      expect(storedAdd()).toHaveBeenCalledTimes(1);
+    });
+
+    it('counts control records in both ingested and stored byte totals', async () => {
+      const ctx = newCtx();
+      const body = ndjsonBody(outputLine('hello\n'), groupStartLine('g1', 'Build'), endLine(42));
+
+      await appendLogs({...ctx, attempt: 1, offset: 0, body});
+
+      expect(ingestedAdd()).toHaveBeenCalledTimes(1);
+      expect(ingestedAdd()).toHaveBeenCalledWith(body.length);
+      expect(storedAdd()).toHaveBeenCalledTimes(1);
+      expect(storedAdd()).toHaveBeenCalledWith(body.length);
+    });
+
+    it('counts a cap-crossing append once and a dropped straggler as ingested only', async () => {
+      const ctx = newCtx();
+      // 150 payload bytes cross the 100-byte test budget, but the crossing chunk is stored in
+      // full; the straggler is accepted-and-dropped, so it must not count as stored.
+      const crossing = outputOfBytes(150);
+      await appendLogs({...ctx, attempt: 1, offset: 0, body: crossing});
+      const straggler = ndjsonBody(outputLine('late\n'));
+
+      await appendLogs({...ctx, attempt: 1, offset: crossing.length, body: straggler});
+
+      expect(ingestedAdd()).toHaveBeenCalledTimes(2);
+      expect(ingestedAdd()).toHaveBeenCalledWith(crossing.length);
+      expect(ingestedAdd()).toHaveBeenCalledWith(straggler.length);
+      // The server-injected `capped` tombstone chunk never counts as stored bytes either.
+      expect(storedAdd()).toHaveBeenCalledTimes(1);
+      expect(storedAdd()).toHaveBeenCalledWith(crossing.length);
+    });
+
+    it('does not re-count a capped-job straggler when the runner retries it', async () => {
+      const ctx = newCtx();
+      const crossing = outputOfBytes(150);
+      await appendLogs({...ctx, attempt: 1, offset: 0, body: crossing});
+      const straggler = ndjsonBody(outputLine('late\n'));
+      await appendLogs({...ctx, attempt: 1, offset: crossing.length, body: straggler});
+
+      await appendLogs({...ctx, attempt: 1, offset: crossing.length, body: straggler});
+
+      expect(ingestedAdd()).toHaveBeenCalledTimes(2);
+      expect(storedAdd()).toHaveBeenCalledTimes(1);
+    });
+  });
+
   describe('write-path enforcement', () => {
     it.each([
       'capped',

package/src/core/append-logs.ts

@@ -22,6 +22,8 @@ import {
   setDeclaredTotalBytes,
 } from '#db/streams.js';
 import {
+  bytesIngestedCount,
+  bytesStoredCount,
   type LogRecordMetricKind,
   recordAppendedCount,
   streamClosedCount,
@@ -427,6 +429,12 @@ export async function appendLogs(
     recordCounts: {} as Partial<Record<LogRecordMetricKind, number>>,
     streamClosedReason: undefined as 'declared' | undefined,
     streamOpened: false,
+    // Raw runner body bytes accepted by the in-order CAS; normalized durable bytes written
+    // to chunk rows. Both are accumulated inside the transaction and recorded only after it
+    // commits, so a rolled-back append never counts. See the semantics on the metric
+    // definitions in `#metrics/instance.js`.
+    ingestedBytes: 0,
+    storedBytes: 0,
   };
 
   const result = await db().transaction(async (tx) => {
@@ -458,6 +466,9 @@ export async function appendLogs(
     if (cas.outcome === 'retry') {
       return {committedLength: cas.committedLength, capped: await isJobCapped(tx, params.jobId)};
     }
+    // In-order CAS extension: the raw body is accepted. Retries and gaps returned above, and
+    // closed streams / empty heartbeats never reach the CAS, so each body is counted once.
+    metrics.ingestedBytes += commitByteLen;
 
     const parseHarness =
       sessionHarness === 'claude' ||
@@ -491,6 +502,8 @@ export async function appendLogs(
       declaredTotalBytes,
     });
     if (chunkStored) {
+      // Normalized durable bytes; a cap-dropped straggler never reaches this branch.
+      metrics.storedBytes += stored.body.length;
       addRecordCounts(metrics.recordCounts, stored.recordCounts);
       if (stored.claudeParseContext !== undefined) {
         await setClaudeParseContext(tx, {
@@ -520,6 +533,8 @@ export async function appendLogs(
   });
 
   if (metrics.streamOpened) streamOpenedCount.add(1);
+  if (metrics.ingestedBytes > 0) bytesIngestedCount.add(metrics.ingestedBytes);
+  if (metrics.storedBytes > 0) bytesStoredCount.add(metrics.storedBytes);
   for (const [kind, count] of Object.entries(metrics.recordCounts)) {
     if (count > 0) recordAppendedCount.add(count, {kind: kind as LogRecordMetricKind});
   }

package/src/db/chunks.test.ts

@@ -0,0 +1,59 @@
+import {appendLogs} from '#core/append-logs.js';
+import {
+  endLine,
+  groupStartLine,
+  ndjsonBody,
+  outputLine,
+  outputOfBytes,
+} from '#test/fixtures/ndjson.js';
+import {findStream, listChunks} from '#test/queries.js';
+import {getUncompactedChunkBytes} from './chunks.js';
+
+interface Ctx {
+  jobId: string;
+  stepId: string;
+  workspaceId: string;
+  projectId: string;
+  workflowRunAttemptId: string;
+}
+
+function newCtx(): Ctx {
+  return {
+    jobId: crypto.randomUUID(),
+    stepId: crypto.randomUUID(),
+    workspaceId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    workflowRunAttemptId: crypto.randomUUID(),
+  };
+}
+
+function chunkBytesOf(streamId: string): Promise<number> {
+  return listChunks(streamId).then((chunks) =>
+    chunks.reduce((total, chunk) => total + chunk.byteLen, 0),
+  );
+}
+
+describe('getUncompactedChunkBytes', () => {
+  it('sums chunk bytes across open and declared-closed streams, control chunks included', async () => {
+    const before = await getUncompactedChunkBytes();
+    const open = newCtx();
+    const closed = newCtx();
+    // 150 payload bytes cross the 100-byte test budget, so this stream also gets a `capped`
+    // control chunk: the gauge must count it, since compaction will move it to storage too.
+    const crossingBody = outputOfBytes(150);
+    await appendLogs({...open, attempt: 1, offset: 0, body: crossingBody});
+    const closedBody = ndjsonBody(outputLine('done\n'), groupStartLine('g1', 'Build'), endLine(4));
+    await appendLogs({...closed, attempt: 1, offset: 0, body: closedBody});
+
+    const openStream = await findStream({...open, attempt: 1});
+    const closedStream = await findStream({...closed, attempt: 1});
+    const expected =
+      (await chunkBytesOf(openStream?.id as string)) +
+      (await chunkBytesOf(closedStream?.id as string));
+
+    const after = await getUncompactedChunkBytes();
+
+    expect(after - before).toBe(BigInt(expected));
+    expect(closedStream?.state).toBe('closed');
+  });
+});

package/src/db/chunks.ts

@@ -114,6 +114,20 @@ export async function readChunkPageBySeq(params: {
   return {data: Buffer.concat(rows.map((row) => row.data)), nextSeq, hasMore};
 }
 
+/**
+ * Total bytes held in hot chunk rows across all streams, runner and server-injected
+ * control chunks alike. Chunks are only deleted by compaction, so this is exactly the
+ * un-compacted hot volume the service gauge reports (open streams plus closed streams
+ * still awaiting compaction).
+ */
+export async function getUncompactedChunkBytes(): Promise<bigint> {
+  const [row] = await db()
+    .select({value: sql<bigint>`coalesce(sum(${logChunks.byteLen}), 0)`.mapWith(BigInt)})
+    .from(logChunks);
+
+  return row?.value ?? 0n;
+}
+
 export interface ChunkStats {
   count: number;
   maxSeq: number;

package/src/metrics/instance.ts

@@ -14,6 +14,38 @@ export const streamOpenedCount = meter.createCounter<Record<string, never>>('log
   description: 'Log streams opened by first append',
 });
 
+// Byte-volume counters use the OpenTelemetry byte unit so Prometheus appends `_bytes` to the
+// metric name. The two ingest axes MUST stay distinct:
+//
+//   raw ingested (CAS axis)      normalized stored (read axis)
+//   runner body bytes accepted   durable chunk bytes written after ingest
+//   by the offset-CAS, before    normalization (agent_session records are parsed
+//   normalization                into view rows), excluding server tombstones
+//
+// `logs_bytes_ingested` counts what the protocol accepted: every in-order CAS extension,
+// including cap-crossing appends and post-cap accept-and-drop stragglers (their
+// `committed_length` advances, so they are accepted even though nothing is stored).
+// Retries, gaps, closed-stream appends, and empty heartbeats never extend the CAS and are
+// never counted, so the same bytes are never double-counted.
+//
+// `logs_bytes_stored` counts only normalized bodies durably written as chunk rows, so a
+// capped job's dropped straggler and server-injected `capped`/`runner_lost` tombstones do
+// not inflate it. ingested - stored is the normalization delta plus cap/close drops.
+export const bytesIngestedCount = meter.createCounter<Record<string, never>>(
+  'logs_bytes_ingested',
+  {
+    description:
+      'Raw runner bytes accepted after offset validation (in-order CAS extension; retries, gaps, closed-stream and cap-dropped bodies excluded)',
+    unit: 'By',
+  },
+);
+
+export const bytesStoredCount = meter.createCounter<Record<string, never>>('logs_bytes_stored', {
+  description:
+    'Normalized durable bytes written to log chunks from runner appends (server tombstones and cap-dropped bodies excluded)',
+  unit: 'By',
+});
+
 export const streamClosedCount = meter.createCounter<{reason: 'declared' | 'timeout'}>(
   'logs_stream_closed',
   {description: 'Log streams closed by reason'},
@@ -31,3 +63,15 @@ export const compactionCount = meter.createCounter<{outcome: CompactionMetricOut
   'logs_compaction',
   {description: 'Log stream compaction attempts by outcome'},
 );
+
+// Uncompressed NDJSON log bytes (not the gzip object size), matching the byte axis of the
+// ingest/storage counters and the `uncompressed_bytes` object metadata. Recorded only once
+// per stream on the single-winner publish, so an idempotent re-run (`already-compacted`) or
+// a failed attempt never double-counts.
+export const compactedBytesCount = meter.createCounter<Record<string, never>>(
+  'logs_compacted_bytes',
+  {
+    description: 'Uncompressed log bytes successfully compacted to object storage',
+    unit: 'By',
+  },
+);

package/src/metrics/service.ts

@@ -1,4 +1,5 @@
 import {getServiceMetricsProvider} from '@shipfox/node-opentelemetry';
+import {getUncompactedChunkBytes} from '#db/chunks.js';
 import {getOpenStreamCount} from '#db/streams.js';
 
 export function registerLogsServiceMetrics(): void {
@@ -8,11 +9,20 @@ export function registerLogsServiceMetrics(): void {
     description: 'Log streams currently open for appends',
   });
 
+  // Hot-storage volume on the service plane: the chunk rows live in shared Postgres, so every
+  // pod would report the same sum and Prometheus must not add them together.
+  const openChunkBytes = meter.createObservableGauge('logs_open_chunk_bytes', {
+    description:
+      'Bytes in un-compacted hot log chunks (open streams plus closed streams awaiting compaction)',
+    unit: 'By',
+  });
+
   meter.addBatchObservableCallback(
     async (observer) => {
       observer.observe(openStreams, toSafeGaugeNumber(await getOpenStreamCount()));
+      observer.observe(openChunkBytes, toSafeGaugeNumber(await getUncompactedChunkBytes()));
     },
-    [openStreams],
+    [openStreams, openChunkBytes],
   );
 }
 

package/src/temporal/activities/compact-stream.ts

@@ -6,7 +6,11 @@ import {compactedGzipStream} from '#core/compaction.js';
 import {chunkStats} from '#db/chunks.js';
 import {db, type Transaction} from '#db/db.js';
 import {getAttemptStreamById, setObjectKeyAndDeleteChunks} from '#db/streams.js';
-import {type CompactionMetricOutcome, compactionCount} from '#metrics/instance.js';
+import {
+  type CompactionMetricOutcome,
+  compactedBytesCount,
+  compactionCount,
+} from '#metrics/instance.js';
 
 export type CompactStreamResult =
   | {outcome: 'gone'}
@@ -125,6 +129,12 @@ export function createCompactStreamActivity(
     try {
       const result = await compactStream(params, dependencies);
       outcome = result.outcome;
+      // Count uncompressed log bytes only on the single-winner publish: idempotent re-runs
+      // (`already-compacted`) and failed attempts never reach this branch, so the counter
+      // tracks exactly the bytes durably moved to object storage.
+      if (result.outcome === 'compacted') {
+        compactedBytesCount.add(result.uncompressedBytes);
+      }
       return result;
     } finally {
       compactionCount.add(1, {outcome});