@shipfox/api-logs 6.0.0 → 8.0.0

package/.turbo/turbo-build.log

@@ -1,21 +1,21 @@
 $ shipfox-swc && shipfox-temporal-bundle dist/temporal/workflows/index.js
-Successfully compiled: 59 files with swc (510.82ms)
-2026-07-21T08:11:10.748Z [INFO] asset workflow-bundle-2850a9a125f53b77d607.js 1.46 MiB [emitted] [immutable] (name: main)
-runtime modules 670 bytes 3 modules
-modules by path ../../../node_modules/.pnpm/ 424 KiB
-  modules by path ../../../node_modules/.pnpm/@temporalio+common@1.18.1/node_modules/@temporal...(truncated) 155 KiB 34 modules
-  modules by path ../../../node_modules/.pnpm/@temporalio+workflow@1.18.1/node_modules/@tempor...(truncated) 201 KiB 26 modules
-  modules by path ../../../node_modules/.pnpm/nexus-rpc@0.0.2/node_modules/nexus-rpc/lib/ 22.3 KiB 13 modules
-  + 2 modules
-modules by path ./dist/temporal/workflows/ 4.17 KiB
-  ./dist/temporal/workflows/index-autogenerated-entrypoint.cjs 523 bytes [built] [code generated]
+Successfully compiled: 59 files with swc (410.67ms)
+2026-07-22T14:51:57.748Z [INFO] asset workflow-bundle-17839eee3b21fbae5b8c.js 3.18 MiB [emitted] [immutable] (name: main)
+orphan modules 33.5 KiB [orphan] 21 modules
+runtime modules 1.13 KiB 5 modules
+modules by path ../../../node_modules/.pnpm/ 970 KiB 196 modules
+modules by path ./dist/temporal/workflows/ 4.57 KiB
+  ./dist/temporal/workflows/index-autogenerated-entrypoint.cjs 929 bytes [built] [code generated]
   ./dist/temporal/workflows/index.js 340 bytes [built] [code generated]
   ./dist/temporal/workflows/close-abandoned-streams.js 753 bytes [built] [code generated]
   ./dist/temporal/workflows/compact-stream.js 941 bytes [built] [code generated]
-  + 3 modules
+  ./dist/temporal/workflows/compaction-reconcile-cron.js 559 bytes [built] [code generated]
+  ./dist/temporal/workflows/reap-stale-open-streams-cron.js 429 bytes [built] [code generated]
+  ./dist/temporal/workflows/retention-sweep-cron.js 727 bytes [built] [code generated]
 optional modules 30 bytes [optional]
   __temporal_custom_payload_converter (ignored) 15 bytes [optional] [built] [code generated]
   __temporal_custom_failure_converter (ignored) 15 bytes [optional] [built] [code generated]
-webpack 5.107.2 compiled successfully in 3800 ms
-2026-07-21T08:11:10.761Z [INFO] Workflow bundle created { size: '1.46MB' }
-/home/runner/work/shipfox/shipfox/libs/api/logs/dist/temporal/workflows/index.js: 1530555 bytes
+../../shared/node/temporal/dist/workflow-error-interceptor.js 1.21 KiB [built] [code generated]
+webpack 5.107.2 compiled successfully in 7153 ms
+2026-07-22T14:51:57.766Z [INFO] Workflow bundle created { size: '3.18MB' }
+/home/runner/work/shipfox/shipfox/libs/api/logs/dist/temporal/workflows/index.js: 3335222 bytes

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @shipfox/api-logs
 
+## 8.0.0
+
+### Patch Changes
+
+- de559bb: Moves Agent validation policy behind a versioned inter-module catalog and injects it into Definitions normalization.
+- b15f3a7: Removes Auth implementation dependencies from consumer test boundaries.
+  - @shipfox/api-workflows-dto@8.0.0
+
+## 7.1.0
+
+### Patch Changes
+
+- ac42c96: Adds boundary-owned reporting for unexpected API runtime failures while preserving expected client and provider outcomes.
+- Updated dependencies [ac42c96]
+- Updated dependencies [6ce08c0]
+  - @shipfox/node-error-monitoring@0.2.0
+  - @shipfox/node-fastify@0.3.0
+  - @shipfox/node-module@0.5.0
+  - @shipfox/node-temporal@0.4.0
+  - @shipfox/node-opentelemetry@0.6.0
+  - @shipfox/api-auth-context@7.1.0
+
 ## 6.0.0
 
 ### Major Changes

package/dist/api/object-storage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"object-storage.d.ts","sourceRoot":"","sources":["../../src/api/object-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,EAML,QAAQ,EACT,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAC,KAAK,kBAAkB,EAAe,MAAM,8BAA8B,CAAC;AAYnF,oFAAoF;AACpF,wBAAgB,QAAQ,IAAI,QAAQ,CAcnC;AAED,wBAAgB,aAAa,IAAI,IAAI,CAGpC;AAED,6HAA6H;AAC7H,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,OAAO,CAAC,CAO7D;AAID;;;;;GAKG;AACH,wBAAgB,cAAc,IAAI,QAAQ,CAYzC;AAED,wBAAgB,mBAAmB,IAAI,IAAI,CAG1C;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAE5F;AAOD,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,QAAQ,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,IAAI,CAAC;CACzB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiCxF;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE7D;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAC,CAAC,CAYhG;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCzE"}
+{"version":3,"file":"object-storage.d.ts","sourceRoot":"","sources":["../../src/api/object-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,EAML,QAAQ,EACT,MAAM,oBAAoB,CAAC;AAM5B,OAAO,EAAC,KAAK,kBAAkB,EAAe,MAAM,8BAA8B,CAAC;AAYnF,oFAAoF;AACpF,wBAAgB,QAAQ,IAAI,QAAQ,CAcnC;AAED,wBAAgB,aAAa,IAAI,IAAI,CAGpC;AAED,6HAA6H;AAC7H,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,OAAO,CAAC,CAO7D;AAID;;;;;GAKG;AACH,wBAAgB,cAAc,IAAI,QAAQ,CAYzC;AAED,wBAAgB,mBAAmB,IAAI,IAAI,CAG1C;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAE5F;AAOD,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,QAAQ,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,IAAI,CAAC;CACzB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAkDxF;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE7D;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAC,CAAC,CAYhG;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCzE"}

package/dist/api/object-storage.js

@@ -1,6 +1,8 @@
 import { DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, HeadBucketCommand, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import { reportError } from '@shipfox/node-error-monitoring';
+import { logger } from '@shipfox/node-opentelemetry';
 import { config } from '#config.js';
 import { logObjectKey } from '#core/entities/log-object.js';
 const explicitS3Credentials = config.LOG_STORAGE_S3_ACCESS_KEY_ID && config.LOG_STORAGE_S3_SECRET_ACCESS_KEY ? {
@@ -114,12 +116,36 @@ const UPLOAD_QUEUE_SIZE = 2;
         // Subscribe before the aborted-check so a signal firing between them cannot be missed,
         // then handle the already-aborted case eagerly (an already-aborted signal never fires 'abort').
         params.signal.addEventListener('abort', ()=>{
-            void upload.abort().catch(()=>undefined);
+            void upload.abort().catch((error)=>{
+                logger().error({
+                    err: error,
+                    objectKey: params.key
+                }, 'Failed to abort multipart log upload');
+                reportError(error, {
+                    boundary: 'logs.cleanup',
+                    operation: 'abort-multipart-upload',
+                    extra: {
+                        objectKey: params.key
+                    }
+                });
+            });
         }, {
             once: true
         });
         if (params.signal.aborted) {
-            await upload.abort().catch(()=>undefined);
+            await upload.abort().catch((error)=>{
+                logger().error({
+                    err: error,
+                    objectKey: params.key
+                }, 'Failed to abort multipart log upload');
+                reportError(error, {
+                    boundary: 'logs.cleanup',
+                    operation: 'abort-multipart-upload',
+                    extra: {
+                        objectKey: params.key
+                    }
+                });
+            });
             throw new Error('Compaction upload aborted before it started');
         }
     }

package/dist/api/object-storage.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/api/object-storage.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  DeleteObjectCommand,\n  DeleteObjectsCommand,\n  GetObjectCommand,\n  HeadBucketCommand,\n  ListObjectsV2Command,\n  S3Client,\n} from '@aws-sdk/client-s3';\nimport {Upload} from '@aws-sdk/lib-storage';\nimport {getSignedUrl} from '@aws-sdk/s3-request-presigner';\nimport {config} from '#config.js';\nimport {type LogObjectKeyParams, logObjectKey} from '#core/entities/log-object.js';\n\nconst explicitS3Credentials =\n  config.LOG_STORAGE_S3_ACCESS_KEY_ID && config.LOG_STORAGE_S3_SECRET_ACCESS_KEY\n    ? {\n        accessKeyId: config.LOG_STORAGE_S3_ACCESS_KEY_ID,\n        secretAccessKey: config.LOG_STORAGE_S3_SECRET_ACCESS_KEY,\n      }\n    : undefined;\n\nlet _client: S3Client | undefined;\n\n/** Lazily-built S3 client targeting the configured object store (Garage in dev). */\nexport function s3Client(): S3Client {\n  if (!_client) {\n    _client = new S3Client({\n      endpoint: config.LOG_STORAGE_S3_ENDPOINT,\n      region: config.LOG_STORAGE_S3_REGION,\n      forcePathStyle: config.LOG_STORAGE_S3_FORCE_PATH_STYLE,\n      ...(explicitS3Credentials ? {credentials: explicitS3Credentials} : {}),\n      // Fail fast: a slow or black-holed endpoint must not hang callers behind\n      // SDK backoff. One retry, short connect/request timeouts.\n      maxAttempts: 2,\n      requestHandler: {connectionTimeout: 1_000, requestTimeout: 3_000},\n    });\n  }\n  return _client;\n}\n\nexport function closeS3Client(): void {\n  _client?.destroy();\n  _client = undefined;\n}\n\n/** Readiness probe: true when the configured bucket answers a HEAD. Used by the app readiness check, not at process boot. */\nexport async function checkBucketReachable(): Promise<boolean> {\n  try {\n    await s3Client().send(new HeadBucketCommand({Bucket: config.LOG_STORAGE_S3_BUCKET}));\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nlet _uploadClient: S3Client | undefined;\n\n/**\n * S3 client for streaming compaction uploads. `s3Client()` is tuned fail-fast for the\n * readiness HEAD, but each multipart part upload can take as long as the link needs, so\n * the per-request timeout is disabled here. Temporal owns the retry budget, so the SDK\n * keeps only a small attempt count for transient connection blips.\n */\nexport function uploadS3Client(): S3Client {\n  if (!_uploadClient) {\n    _uploadClient = new S3Client({\n      endpoint: config.LOG_STORAGE_S3_ENDPOINT,\n      region: config.LOG_STORAGE_S3_REGION,\n      forcePathStyle: config.LOG_STORAGE_S3_FORCE_PATH_STYLE,\n      ...(explicitS3Credentials ? {credentials: explicitS3Credentials} : {}),\n      maxAttempts: 3,\n      requestHandler: {connectionTimeout: 5_000, requestTimeout: 0},\n    });\n  }\n  return _uploadClient;\n}\n\nexport function closeUploadS3Client(): void {\n  _uploadClient?.destroy();\n  _uploadClient = undefined;\n}\n\n/**\n * Per-attempt object key for a compacted stream: the stream's stable prefix\n * (`logObjectKey` -> `{prefix}/{workspaceId}/{jobId}/{stepId}/{attempt}`, shared with retention\n * and workspace prefix deletes) plus a unique `uploadToken` leaf. Each compaction attempt\n * uploads to its own key and the winner records it atomically, so a slow or zombie attempt\n * can never overwrite a published object. Losing and crashed attempts leave a leaf under the\n * same prefix, reclaimed by the bucket's incomplete-multipart-abort rule and prefix-scoped\n * retention.\n */\nexport function compactedObjectKey(identity: LogObjectKeyParams, uploadToken: string): string {\n  return `${logObjectKey(config.LOG_STORAGE_S3_PREFIX, identity)}/${uploadToken}`;\n}\n\n// lib-storage defaults (5MB parts, queue of 4); we pin a small queue so peak buffer memory\n// stays bounded under concurrent compactions (queueSize x partSize per active upload).\nconst UPLOAD_PART_SIZE = 5 * 1024 * 1024;\nconst UPLOAD_QUEUE_SIZE = 2;\n\nexport interface PutCompactedObjectParams {\n  key: string;\n  body: Readable;\n  metadata: Record<string, string>;\n  signal?: AbortSignal;\n  onProgress?: () => void;\n}\n\n/**\n * Streams `body` (gzip-compressed NDJSON) to the compacted object via a multipart upload.\n * `Content-Encoding: gzip` so a browser-direct presigned GET auto-decompresses. Aborts the\n * multipart upload when `signal` fires, so a cancelled activity leaves no dangling parts.\n *\n * A hard crash (no graceful abort) can still strand incomplete multipart parts, which on\n * pay-per-storage providers (S3, R2) bill until aborted. The worker never sets bucket\n * policy, so the deployed bucket must carry an `AbortIncompleteMultipartUpload` lifecycle\n * rule. `dev/garage/bootstrap.sh` provisions it for local Garage; self-hosters set the same\n * rule on their bucket.\n */\nexport async function putCompactedObject(params: PutCompactedObjectParams): Promise<void> {\n  const upload = new Upload({\n    client: uploadS3Client(),\n    partSize: UPLOAD_PART_SIZE,\n    queueSize: UPLOAD_QUEUE_SIZE,\n    params: {\n      Bucket: config.LOG_STORAGE_S3_BUCKET,\n      Key: params.key,\n      Body: params.body,\n      ContentType: 'application/x-ndjson',\n      ContentEncoding: 'gzip',\n      Metadata: params.metadata,\n    },\n  });\n\n  if (params.onProgress) upload.on('httpUploadProgress', params.onProgress);\n  if (params.signal) {\n    // Subscribe before the aborted-check so a signal firing between them cannot be missed,\n    // then handle the already-aborted case eagerly (an already-aborted signal never fires 'abort').\n    params.signal.addEventListener(\n      'abort',\n      () => {\n        void upload.abort().catch(() => undefined);\n      },\n      {once: true},\n    );\n    if (params.signal.aborted) {\n      await upload.abort().catch(() => undefined);\n      throw new Error('Compaction upload aborted before it started');\n    }\n  }\n\n  await upload.done();\n}\n\n/**\n * Deletes a compacted object. S3 `DeleteObject` is idempotent (a missing key is a success),\n * so retention and the publish orphan guard can call it freely without a pre-check.\n */\nexport async function deleteObject(key: string): Promise<void> {\n  await s3Client().send(new DeleteObjectCommand({Bucket: config.LOG_STORAGE_S3_BUCKET, Key: key}));\n}\n\n/**\n * Presigns a GET for a compacted log object so the browser can fetch it directly,\n * bypassing API egress. `getSignedUrl` only computes a signature locally (no network\n * call), so the fail-fast `s3Client()` is fine here. `expiresAt` mirrors the URL's\n * `LOG_READ_URL_TTL_SECONDS` lifetime so the caller can hand the client an absolute\n * deadline without re-deriving it.\n */\nexport async function presignedGetUrl(objectKey: string): Promise<{url: string; expiresAt: Date}> {\n  const ttlSeconds = config.LOG_READ_URL_TTL_SECONDS;\n  // Stamp the deadline from before signing: getSignedUrl bakes its expiry from the clock at\n  // sign time, so reading Date.now() after the await could report a deadline slightly past the\n  // real one. Computing it first keeps expiresAt a conservative (never-late) bound.\n  const expiresAt = new Date(Date.now() + ttlSeconds * 1000);\n  const url = await getSignedUrl(\n    s3Client(),\n    new GetObjectCommand({Bucket: config.LOG_STORAGE_S3_BUCKET, Key: objectKey}),\n    {expiresIn: ttlSeconds},\n  );\n  return {url, expiresAt};\n}\n\n/**\n * Deletes all objects under a per-attempt prefix. Retention deletes by prefix, not only the\n * recorded `object_key`, to reclaim orphan leaves from failed compaction attempts.\n *\n * Pass a trailing slash so attempt `1` never matches attempt `10`.\n */\nexport async function deleteObjectsByPrefix(prefix: string): Promise<void> {\n  if (prefix === '') {\n    throw new Error(\n      'deleteObjectsByPrefix refuses an empty prefix (it would target the whole bucket)',\n    );\n  }\n  const client = s3Client();\n  let continuationToken: string | undefined;\n  do {\n    const listed = await client.send(\n      new ListObjectsV2Command({\n        Bucket: config.LOG_STORAGE_S3_BUCKET,\n        Prefix: prefix,\n        ContinuationToken: continuationToken,\n      }),\n    );\n    const objects = (listed.Contents ?? []).flatMap((object) =>\n      object.Key ? [{Key: object.Key}] : [],\n    );\n    if (objects.length > 0) {\n      const deleted = await client.send(\n        new DeleteObjectsCommand({\n          Bucket: config.LOG_STORAGE_S3_BUCKET,\n          Delete: {Objects: objects, Quiet: true},\n        }),\n      );\n      if (deleted.Errors && deleted.Errors.length > 0) {\n        const [first] = deleted.Errors;\n        throw new Error(\n          `Failed to delete ${deleted.Errors.length} object(s) under ${prefix}: ${first?.Key} ${first?.Message}`,\n        );\n      }\n    }\n    continuationToken = listed.IsTruncated ? listed.NextContinuationToken : undefined;\n  } while (continuationToken);\n}\n"],"names":["DeleteObjectCommand","DeleteObjectsCommand","GetObjectCommand","HeadBucketCommand","ListObjectsV2Command","S3Client","Upload","getSignedUrl","config","logObjectKey","explicitS3Credentials","LOG_STORAGE_S3_ACCESS_KEY_ID","LOG_STORAGE_S3_SECRET_ACCESS_KEY","accessKeyId","secretAccessKey","undefined","_client","s3Client","endpoint","LOG_STORAGE_S3_ENDPOINT","region","LOG_STORAGE_S3_REGION","forcePathStyle","LOG_STORAGE_S3_FORCE_PATH_STYLE","credentials","maxAttempts","requestHandler","connectionTimeout","requestTimeout","closeS3Client","destroy","checkBucketReachable","send","Bucket","LOG_STORAGE_S3_BUCKET","_uploadClient","uploadS3Client","closeUploadS3Client","compactedObjectKey","identity","uploadToken","LOG_STORAGE_S3_PREFIX","UPLOAD_PART_SIZE","UPLOAD_QUEUE_SIZE","putCompactedObject","params","upload","client","partSize","queueSize","Key","key","Body","body","ContentType","ContentEncoding","Metadata","metadata","onProgress","on","signal","addEventListener","abort","catch","once","aborted","Error","done","deleteObject","presignedGetUrl","objectKey","ttlSeconds","LOG_READ_URL_TTL_SECONDS","expiresAt","Date","now","url","expiresIn","deleteObjectsByPrefix","prefix","continuationToken","listed","Prefix","ContinuationToken","objects","Contents","flatMap","object","length","deleted","Delete","Objects","Quiet","Errors","first","Message","IsTruncated","NextContinuationToken"],"mappings":"AACA,SACEA,mBAAmB,EACnBC,oBAAoB,EACpBC,gBAAgB,EAChBC,iBAAiB,EACjBC,oBAAoB,EACpBC,QAAQ,QACH,qBAAqB;AAC5B,SAAQC,MAAM,QAAO,uBAAuB;AAC5C,SAAQC,YAAY,QAAO,gCAAgC;AAC3D,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAiCC,YAAY,QAAO,+BAA+B;AAEnF,MAAMC,wBACJF,OAAOG,4BAA4B,IAAIH,OAAOI,gCAAgC,GAC1E;IACEC,aAAaL,OAAOG,4BAA4B;IAChDG,iBAAiBN,OAAOI,gCAAgC;AAC1D,IACAG;AAEN,IAAIC;AAEJ,kFAAkF,GAClF,OAAO,SAASC;IACd,IAAI,CAACD,SAAS;QACZA,UAAU,IAAIX,SAAS;YACrBa,UAAUV,OAAOW,uBAAuB;YACxCC,QAAQZ,OAAOa,qBAAqB;YACpCC,gBAAgBd,OAAOe,+BAA+B;YACtD,GAAIb,wBAAwB;gBAACc,aAAad;YAAqB,IAAI,CAAC,CAAC;YACrE,yEAAyE;YACzE,0DAA0D;YAC1De,aAAa;YACbC,gBAAgB;gBAACC,mBAAmB;gBAAOC,gBAAgB;YAAK;QAClE;IACF;IACA,OAAOZ;AACT;AAEA,OAAO,SAASa;IACdb,SAASc;IACTd,UAAUD;AACZ;AAEA,2HAA2H,GAC3H,OAAO,eAAegB;IACpB,IAAI;QACF,MAAMd,WAAWe,IAAI,CAAC,IAAI7B,kBAAkB;YAAC8B,QAAQzB,OAAO0B,qBAAqB;QAAA;QACjF,OAAO;IACT,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,IAAIC;AAEJ;;;;;CAKC,GACD,OAAO,SAASC;IACd,IAAI,CAACD,eAAe;QAClBA,gBAAgB,IAAI9B,SAAS;YAC3Ba,UAAUV,OAAOW,uBAAuB;YACxCC,QAAQZ,OAAOa,qBAAqB;YACpCC,gBAAgBd,OAAOe,+BAA+B;YACtD,GAAIb,wBAAwB;gBAACc,aAAad;YAAqB,IAAI,CAAC,CAAC;YACrEe,aAAa;YACbC,gBAAgB;gBAACC,mBAAmB;gBAAOC,gBAAgB;YAAC;QAC9D;IACF;IACA,OAAOO;AACT;AAEA,OAAO,SAASE;IACdF,eAAeL;IACfK,gBAAgBpB;AAClB;AAEA;;;;;;;;CAQC,GACD,OAAO,SAASuB,mBAAmBC,QAA4B,EAAEC,WAAmB;IAClF,OAAO,GAAG/B,aAAaD,OAAOiC,qBAAqB,EAAEF,UAAU,CAAC,EAAEC,aAAa;AACjF;AAEA,2FAA2F;AAC3F,uFAAuF;AACvF,MAAME,mBAAmB,IAAI,OAAO;AACpC,MAAMC,oBAAoB;AAU1B;;;;;;;;;;CAUC,GACD,OAAO,eAAeC,mBAAmBC,MAAgC;IACvE,MAAMC,SAAS,IAAIxC,OAAO;QACxByC,QAAQX;QACRY,UAAUN;QACVO,WAAWN;QACXE,QAAQ;YACNZ,QAAQzB,OAAO0B,qBAAqB;YACpCgB,KAAKL,OAAOM,GAAG;YACfC,MAAMP,OAAOQ,IAAI;YACjBC,aAAa;YACbC,iBAAiB;YACjBC,UAAUX,OAAOY,QAAQ;QAC3B;IACF;IAEA,IAAIZ,OAAOa,UAAU,EAAEZ,OAAOa,EAAE,CAAC,sBAAsBd,OAAOa,UAAU;IACxE,IAAIb,OAAOe,MAAM,EAAE;QACjB,uFAAuF;QACvF,gGAAgG;QAChGf,OAAOe,MAAM,CAACC,gBAAgB,CAC5B,SACA;YACE,KAAKf,OAAOgB,KAAK,GAAGC,KAAK,CAAC,IAAMhD;QAClC,GACA;YAACiD,MAAM;QAAI;QAEb,IAAInB,OAAOe,MAAM,CAACK,OAAO,EAAE;YACzB,MAAMnB,OAAOgB,KAAK,GAAGC,KAAK,CAAC,IAAMhD;YACjC,MAAM,IAAImD,MAAM;QAClB;IACF;IAEA,MAAMpB,OAAOqB,IAAI;AACnB;AAEA;;;CAGC,GACD,OAAO,eAAeC,aAAajB,GAAW;IAC5C,MAAMlC,WAAWe,IAAI,CAAC,IAAIhC,oBAAoB;QAACiC,QAAQzB,OAAO0B,qBAAqB;QAAEgB,KAAKC;IAAG;AAC/F;AAEA;;;;;;CAMC,GACD,OAAO,eAAekB,gBAAgBC,SAAiB;IACrD,MAAMC,aAAa/D,OAAOgE,wBAAwB;IAClD,0FAA0F;IAC1F,6FAA6F;IAC7F,kFAAkF;IAClF,MAAMC,YAAY,IAAIC,KAAKA,KAAKC,GAAG,KAAKJ,aAAa;IACrD,MAAMK,MAAM,MAAMrE,aAChBU,YACA,IAAIf,iBAAiB;QAAC+B,QAAQzB,OAAO0B,qBAAqB;QAAEgB,KAAKoB;IAAS,IAC1E;QAACO,WAAWN;IAAU;IAExB,OAAO;QAACK;QAAKH;IAAS;AACxB;AAEA;;;;;CAKC,GACD,OAAO,eAAeK,sBAAsBC,MAAc;IACxD,IAAIA,WAAW,IAAI;QACjB,MAAM,IAAIb,MACR;IAEJ;IACA,MAAMnB,SAAS9B;IACf,IAAI+D;IACJ,GAAG;QACD,MAAMC,SAAS,MAAMlC,OAAOf,IAAI,CAC9B,IAAI5B,qBAAqB;YACvB6B,QAAQzB,OAAO0B,qBAAqB;YACpCgD,QAAQH;YACRI,mBAAmBH;QACrB;QAEF,MAAMI,UAAU,AAACH,CAAAA,OAAOI,QAAQ,IAAI,EAAE,AAAD,EAAGC,OAAO,CAAC,CAACC,SAC/CA,OAAOrC,GAAG,GAAG;gBAAC;oBAACA,KAAKqC,OAAOrC,GAAG;gBAAA;aAAE,GAAG,EAAE;QAEvC,IAAIkC,QAAQI,MAAM,GAAG,GAAG;YACtB,MAAMC,UAAU,MAAM1C,OAAOf,IAAI,CAC/B,IAAI/B,qBAAqB;gBACvBgC,QAAQzB,OAAO0B,qBAAqB;gBACpCwD,QAAQ;oBAACC,SAASP;oBAASQ,OAAO;gBAAI;YACxC;YAEF,IAAIH,QAAQI,MAAM,IAAIJ,QAAQI,MAAM,CAACL,MAAM,GAAG,GAAG;gBAC/C,MAAM,CAACM,MAAM,GAAGL,QAAQI,MAAM;gBAC9B,MAAM,IAAI3B,MACR,CAAC,iBAAiB,EAAEuB,QAAQI,MAAM,CAACL,MAAM,CAAC,iBAAiB,EAAET,OAAO,EAAE,EAAEe,OAAO5C,IAAI,CAAC,EAAE4C,OAAOC,SAAS;YAE1G;QACF;QACAf,oBAAoBC,OAAOe,WAAW,GAAGf,OAAOgB,qBAAqB,GAAGlF;IAC1E,QAASiE,kBAAmB;AAC9B"}
+{"version":3,"sources":["../../src/api/object-storage.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  DeleteObjectCommand,\n  DeleteObjectsCommand,\n  GetObjectCommand,\n  HeadBucketCommand,\n  ListObjectsV2Command,\n  S3Client,\n} from '@aws-sdk/client-s3';\nimport {Upload} from '@aws-sdk/lib-storage';\nimport {getSignedUrl} from '@aws-sdk/s3-request-presigner';\nimport {reportError} from '@shipfox/node-error-monitoring';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {config} from '#config.js';\nimport {type LogObjectKeyParams, logObjectKey} from '#core/entities/log-object.js';\n\nconst explicitS3Credentials =\n  config.LOG_STORAGE_S3_ACCESS_KEY_ID && config.LOG_STORAGE_S3_SECRET_ACCESS_KEY\n    ? {\n        accessKeyId: config.LOG_STORAGE_S3_ACCESS_KEY_ID,\n        secretAccessKey: config.LOG_STORAGE_S3_SECRET_ACCESS_KEY,\n      }\n    : undefined;\n\nlet _client: S3Client | undefined;\n\n/** Lazily-built S3 client targeting the configured object store (Garage in dev). */\nexport function s3Client(): S3Client {\n  if (!_client) {\n    _client = new S3Client({\n      endpoint: config.LOG_STORAGE_S3_ENDPOINT,\n      region: config.LOG_STORAGE_S3_REGION,\n      forcePathStyle: config.LOG_STORAGE_S3_FORCE_PATH_STYLE,\n      ...(explicitS3Credentials ? {credentials: explicitS3Credentials} : {}),\n      // Fail fast: a slow or black-holed endpoint must not hang callers behind\n      // SDK backoff. One retry, short connect/request timeouts.\n      maxAttempts: 2,\n      requestHandler: {connectionTimeout: 1_000, requestTimeout: 3_000},\n    });\n  }\n  return _client;\n}\n\nexport function closeS3Client(): void {\n  _client?.destroy();\n  _client = undefined;\n}\n\n/** Readiness probe: true when the configured bucket answers a HEAD. Used by the app readiness check, not at process boot. */\nexport async function checkBucketReachable(): Promise<boolean> {\n  try {\n    await s3Client().send(new HeadBucketCommand({Bucket: config.LOG_STORAGE_S3_BUCKET}));\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nlet _uploadClient: S3Client | undefined;\n\n/**\n * S3 client for streaming compaction uploads. `s3Client()` is tuned fail-fast for the\n * readiness HEAD, but each multipart part upload can take as long as the link needs, so\n * the per-request timeout is disabled here. Temporal owns the retry budget, so the SDK\n * keeps only a small attempt count for transient connection blips.\n */\nexport function uploadS3Client(): S3Client {\n  if (!_uploadClient) {\n    _uploadClient = new S3Client({\n      endpoint: config.LOG_STORAGE_S3_ENDPOINT,\n      region: config.LOG_STORAGE_S3_REGION,\n      forcePathStyle: config.LOG_STORAGE_S3_FORCE_PATH_STYLE,\n      ...(explicitS3Credentials ? {credentials: explicitS3Credentials} : {}),\n      maxAttempts: 3,\n      requestHandler: {connectionTimeout: 5_000, requestTimeout: 0},\n    });\n  }\n  return _uploadClient;\n}\n\nexport function closeUploadS3Client(): void {\n  _uploadClient?.destroy();\n  _uploadClient = undefined;\n}\n\n/**\n * Per-attempt object key for a compacted stream: the stream's stable prefix\n * (`logObjectKey` -> `{prefix}/{workspaceId}/{jobId}/{stepId}/{attempt}`, shared with retention\n * and workspace prefix deletes) plus a unique `uploadToken` leaf. Each compaction attempt\n * uploads to its own key and the winner records it atomically, so a slow or zombie attempt\n * can never overwrite a published object. Losing and crashed attempts leave a leaf under the\n * same prefix, reclaimed by the bucket's incomplete-multipart-abort rule and prefix-scoped\n * retention.\n */\nexport function compactedObjectKey(identity: LogObjectKeyParams, uploadToken: string): string {\n  return `${logObjectKey(config.LOG_STORAGE_S3_PREFIX, identity)}/${uploadToken}`;\n}\n\n// lib-storage defaults (5MB parts, queue of 4); we pin a small queue so peak buffer memory\n// stays bounded under concurrent compactions (queueSize x partSize per active upload).\nconst UPLOAD_PART_SIZE = 5 * 1024 * 1024;\nconst UPLOAD_QUEUE_SIZE = 2;\n\nexport interface PutCompactedObjectParams {\n  key: string;\n  body: Readable;\n  metadata: Record<string, string>;\n  signal?: AbortSignal;\n  onProgress?: () => void;\n}\n\n/**\n * Streams `body` (gzip-compressed NDJSON) to the compacted object via a multipart upload.\n * `Content-Encoding: gzip` so a browser-direct presigned GET auto-decompresses. Aborts the\n * multipart upload when `signal` fires, so a cancelled activity leaves no dangling parts.\n *\n * A hard crash (no graceful abort) can still strand incomplete multipart parts, which on\n * pay-per-storage providers (S3, R2) bill until aborted. The worker never sets bucket\n * policy, so the deployed bucket must carry an `AbortIncompleteMultipartUpload` lifecycle\n * rule. `dev/garage/bootstrap.sh` provisions it for local Garage; self-hosters set the same\n * rule on their bucket.\n */\nexport async function putCompactedObject(params: PutCompactedObjectParams): Promise<void> {\n  const upload = new Upload({\n    client: uploadS3Client(),\n    partSize: UPLOAD_PART_SIZE,\n    queueSize: UPLOAD_QUEUE_SIZE,\n    params: {\n      Bucket: config.LOG_STORAGE_S3_BUCKET,\n      Key: params.key,\n      Body: params.body,\n      ContentType: 'application/x-ndjson',\n      ContentEncoding: 'gzip',\n      Metadata: params.metadata,\n    },\n  });\n\n  if (params.onProgress) upload.on('httpUploadProgress', params.onProgress);\n  if (params.signal) {\n    // Subscribe before the aborted-check so a signal firing between them cannot be missed,\n    // then handle the already-aborted case eagerly (an already-aborted signal never fires 'abort').\n    params.signal.addEventListener(\n      'abort',\n      () => {\n        void upload.abort().catch((error) => {\n          logger().error(\n            {err: error, objectKey: params.key},\n            'Failed to abort multipart log upload',\n          );\n          reportError(error, {\n            boundary: 'logs.cleanup',\n            operation: 'abort-multipart-upload',\n            extra: {objectKey: params.key},\n          });\n        });\n      },\n      {once: true},\n    );\n    if (params.signal.aborted) {\n      await upload.abort().catch((error) => {\n        logger().error({err: error, objectKey: params.key}, 'Failed to abort multipart log upload');\n        reportError(error, {\n          boundary: 'logs.cleanup',\n          operation: 'abort-multipart-upload',\n          extra: {objectKey: params.key},\n        });\n      });\n      throw new Error('Compaction upload aborted before it started');\n    }\n  }\n\n  await upload.done();\n}\n\n/**\n * Deletes a compacted object. S3 `DeleteObject` is idempotent (a missing key is a success),\n * so retention and the publish orphan guard can call it freely without a pre-check.\n */\nexport async function deleteObject(key: string): Promise<void> {\n  await s3Client().send(new DeleteObjectCommand({Bucket: config.LOG_STORAGE_S3_BUCKET, Key: key}));\n}\n\n/**\n * Presigns a GET for a compacted log object so the browser can fetch it directly,\n * bypassing API egress. `getSignedUrl` only computes a signature locally (no network\n * call), so the fail-fast `s3Client()` is fine here. `expiresAt` mirrors the URL's\n * `LOG_READ_URL_TTL_SECONDS` lifetime so the caller can hand the client an absolute\n * deadline without re-deriving it.\n */\nexport async function presignedGetUrl(objectKey: string): Promise<{url: string; expiresAt: Date}> {\n  const ttlSeconds = config.LOG_READ_URL_TTL_SECONDS;\n  // Stamp the deadline from before signing: getSignedUrl bakes its expiry from the clock at\n  // sign time, so reading Date.now() after the await could report a deadline slightly past the\n  // real one. Computing it first keeps expiresAt a conservative (never-late) bound.\n  const expiresAt = new Date(Date.now() + ttlSeconds * 1000);\n  const url = await getSignedUrl(\n    s3Client(),\n    new GetObjectCommand({Bucket: config.LOG_STORAGE_S3_BUCKET, Key: objectKey}),\n    {expiresIn: ttlSeconds},\n  );\n  return {url, expiresAt};\n}\n\n/**\n * Deletes all objects under a per-attempt prefix. Retention deletes by prefix, not only the\n * recorded `object_key`, to reclaim orphan leaves from failed compaction attempts.\n *\n * Pass a trailing slash so attempt `1` never matches attempt `10`.\n */\nexport async function deleteObjectsByPrefix(prefix: string): Promise<void> {\n  if (prefix === '') {\n    throw new Error(\n      'deleteObjectsByPrefix refuses an empty prefix (it would target the whole bucket)',\n    );\n  }\n  const client = s3Client();\n  let continuationToken: string | undefined;\n  do {\n    const listed = await client.send(\n      new ListObjectsV2Command({\n        Bucket: config.LOG_STORAGE_S3_BUCKET,\n        Prefix: prefix,\n        ContinuationToken: continuationToken,\n      }),\n    );\n    const objects = (listed.Contents ?? []).flatMap((object) =>\n      object.Key ? [{Key: object.Key}] : [],\n    );\n    if (objects.length > 0) {\n      const deleted = await client.send(\n        new DeleteObjectsCommand({\n          Bucket: config.LOG_STORAGE_S3_BUCKET,\n          Delete: {Objects: objects, Quiet: true},\n        }),\n      );\n      if (deleted.Errors && deleted.Errors.length > 0) {\n        const [first] = deleted.Errors;\n        throw new Error(\n          `Failed to delete ${deleted.Errors.length} object(s) under ${prefix}: ${first?.Key} ${first?.Message}`,\n        );\n      }\n    }\n    continuationToken = listed.IsTruncated ? listed.NextContinuationToken : undefined;\n  } while (continuationToken);\n}\n"],"names":["DeleteObjectCommand","DeleteObjectsCommand","GetObjectCommand","HeadBucketCommand","ListObjectsV2Command","S3Client","Upload","getSignedUrl","reportError","logger","config","logObjectKey","explicitS3Credentials","LOG_STORAGE_S3_ACCESS_KEY_ID","LOG_STORAGE_S3_SECRET_ACCESS_KEY","accessKeyId","secretAccessKey","undefined","_client","s3Client","endpoint","LOG_STORAGE_S3_ENDPOINT","region","LOG_STORAGE_S3_REGION","forcePathStyle","LOG_STORAGE_S3_FORCE_PATH_STYLE","credentials","maxAttempts","requestHandler","connectionTimeout","requestTimeout","closeS3Client","destroy","checkBucketReachable","send","Bucket","LOG_STORAGE_S3_BUCKET","_uploadClient","uploadS3Client","closeUploadS3Client","compactedObjectKey","identity","uploadToken","LOG_STORAGE_S3_PREFIX","UPLOAD_PART_SIZE","UPLOAD_QUEUE_SIZE","putCompactedObject","params","upload","client","partSize","queueSize","Key","key","Body","body","ContentType","ContentEncoding","Metadata","metadata","onProgress","on","signal","addEventListener","abort","catch","error","err","objectKey","boundary","operation","extra","once","aborted","Error","done","deleteObject","presignedGetUrl","ttlSeconds","LOG_READ_URL_TTL_SECONDS","expiresAt","Date","now","url","expiresIn","deleteObjectsByPrefix","prefix","continuationToken","listed","Prefix","ContinuationToken","objects","Contents","flatMap","object","length","deleted","Delete","Objects","Quiet","Errors","first","Message","IsTruncated","NextContinuationToken"],"mappings":"AACA,SACEA,mBAAmB,EACnBC,oBAAoB,EACpBC,gBAAgB,EAChBC,iBAAiB,EACjBC,oBAAoB,EACpBC,QAAQ,QACH,qBAAqB;AAC5B,SAAQC,MAAM,QAAO,uBAAuB;AAC5C,SAAQC,YAAY,QAAO,gCAAgC;AAC3D,SAAQC,WAAW,QAAO,iCAAiC;AAC3D,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAiCC,YAAY,QAAO,+BAA+B;AAEnF,MAAMC,wBACJF,OAAOG,4BAA4B,IAAIH,OAAOI,gCAAgC,GAC1E;IACEC,aAAaL,OAAOG,4BAA4B;IAChDG,iBAAiBN,OAAOI,gCAAgC;AAC1D,IACAG;AAEN,IAAIC;AAEJ,kFAAkF,GAClF,OAAO,SAASC;IACd,IAAI,CAACD,SAAS;QACZA,UAAU,IAAIb,SAAS;YACrBe,UAAUV,OAAOW,uBAAuB;YACxCC,QAAQZ,OAAOa,qBAAqB;YACpCC,gBAAgBd,OAAOe,+BAA+B;YACtD,GAAIb,wBAAwB;gBAACc,aAAad;YAAqB,IAAI,CAAC,CAAC;YACrE,yEAAyE;YACzE,0DAA0D;YAC1De,aAAa;YACbC,gBAAgB;gBAACC,mBAAmB;gBAAOC,gBAAgB;YAAK;QAClE;IACF;IACA,OAAOZ;AACT;AAEA,OAAO,SAASa;IACdb,SAASc;IACTd,UAAUD;AACZ;AAEA,2HAA2H,GAC3H,OAAO,eAAegB;IACpB,IAAI;QACF,MAAMd,WAAWe,IAAI,CAAC,IAAI/B,kBAAkB;YAACgC,QAAQzB,OAAO0B,qBAAqB;QAAA;QACjF,OAAO;IACT,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,IAAIC;AAEJ;;;;;CAKC,GACD,OAAO,SAASC;IACd,IAAI,CAACD,eAAe;QAClBA,gBAAgB,IAAIhC,SAAS;YAC3Be,UAAUV,OAAOW,uBAAuB;YACxCC,QAAQZ,OAAOa,qBAAqB;YACpCC,gBAAgBd,OAAOe,+BAA+B;YACtD,GAAIb,wBAAwB;gBAACc,aAAad;YAAqB,IAAI,CAAC,CAAC;YACrEe,aAAa;YACbC,gBAAgB;gBAACC,mBAAmB;gBAAOC,gBAAgB;YAAC;QAC9D;IACF;IACA,OAAOO;AACT;AAEA,OAAO,SAASE;IACdF,eAAeL;IACfK,gBAAgBpB;AAClB;AAEA;;;;;;;;CAQC,GACD,OAAO,SAASuB,mBAAmBC,QAA4B,EAAEC,WAAmB;IAClF,OAAO,GAAG/B,aAAaD,OAAOiC,qBAAqB,EAAEF,UAAU,CAAC,EAAEC,aAAa;AACjF;AAEA,2FAA2F;AAC3F,uFAAuF;AACvF,MAAME,mBAAmB,IAAI,OAAO;AACpC,MAAMC,oBAAoB;AAU1B;;;;;;;;;;CAUC,GACD,OAAO,eAAeC,mBAAmBC,MAAgC;IACvE,MAAMC,SAAS,IAAI1C,OAAO;QACxB2C,QAAQX;QACRY,UAAUN;QACVO,WAAWN;QACXE,QAAQ;YACNZ,QAAQzB,OAAO0B,qBAAqB;YACpCgB,KAAKL,OAAOM,GAAG;YACfC,MAAMP,OAAOQ,IAAI;YACjBC,aAAa;YACbC,iBAAiB;YACjBC,UAAUX,OAAOY,QAAQ;QAC3B;IACF;IAEA,IAAIZ,OAAOa,UAAU,EAAEZ,OAAOa,EAAE,CAAC,sBAAsBd,OAAOa,UAAU;IACxE,IAAIb,OAAOe,MAAM,EAAE;QACjB,uFAAuF;QACvF,gGAAgG;QAChGf,OAAOe,MAAM,CAACC,gBAAgB,CAC5B,SACA;YACE,KAAKf,OAAOgB,KAAK,GAAGC,KAAK,CAAC,CAACC;gBACzBzD,SAASyD,KAAK,CACZ;oBAACC,KAAKD;oBAAOE,WAAWrB,OAAOM,GAAG;gBAAA,GAClC;gBAEF7C,YAAY0D,OAAO;oBACjBG,UAAU;oBACVC,WAAW;oBACXC,OAAO;wBAACH,WAAWrB,OAAOM,GAAG;oBAAA;gBAC/B;YACF;QACF,GACA;YAACmB,MAAM;QAAI;QAEb,IAAIzB,OAAOe,MAAM,CAACW,OAAO,EAAE;YACzB,MAAMzB,OAAOgB,KAAK,GAAGC,KAAK,CAAC,CAACC;gBAC1BzD,SAASyD,KAAK,CAAC;oBAACC,KAAKD;oBAAOE,WAAWrB,OAAOM,GAAG;gBAAA,GAAG;gBACpD7C,YAAY0D,OAAO;oBACjBG,UAAU;oBACVC,WAAW;oBACXC,OAAO;wBAACH,WAAWrB,OAAOM,GAAG;oBAAA;gBAC/B;YACF;YACA,MAAM,IAAIqB,MAAM;QAClB;IACF;IAEA,MAAM1B,OAAO2B,IAAI;AACnB;AAEA;;;CAGC,GACD,OAAO,eAAeC,aAAavB,GAAW;IAC5C,MAAMlC,WAAWe,IAAI,CAAC,IAAIlC,oBAAoB;QAACmC,QAAQzB,OAAO0B,qBAAqB;QAAEgB,KAAKC;IAAG;AAC/F;AAEA;;;;;;CAMC,GACD,OAAO,eAAewB,gBAAgBT,SAAiB;IACrD,MAAMU,aAAapE,OAAOqE,wBAAwB;IAClD,0FAA0F;IAC1F,6FAA6F;IAC7F,kFAAkF;IAClF,MAAMC,YAAY,IAAIC,KAAKA,KAAKC,GAAG,KAAKJ,aAAa;IACrD,MAAMK,MAAM,MAAM5E,aAChBY,YACA,IAAIjB,iBAAiB;QAACiC,QAAQzB,OAAO0B,qBAAqB;QAAEgB,KAAKgB;IAAS,IAC1E;QAACgB,WAAWN;IAAU;IAExB,OAAO;QAACK;QAAKH;IAAS;AACxB;AAEA;;;;;CAKC,GACD,OAAO,eAAeK,sBAAsBC,MAAc;IACxD,IAAIA,WAAW,IAAI;QACjB,MAAM,IAAIZ,MACR;IAEJ;IACA,MAAMzB,SAAS9B;IACf,IAAIoE;IACJ,GAAG;QACD,MAAMC,SAAS,MAAMvC,OAAOf,IAAI,CAC9B,IAAI9B,qBAAqB;YACvB+B,QAAQzB,OAAO0B,qBAAqB;YACpCqD,QAAQH;YACRI,mBAAmBH;QACrB;QAEF,MAAMI,UAAU,AAACH,CAAAA,OAAOI,QAAQ,IAAI,EAAE,AAAD,EAAGC,OAAO,CAAC,CAACC,SAC/CA,OAAO1C,GAAG,GAAG;gBAAC;oBAACA,KAAK0C,OAAO1C,GAAG;gBAAA;aAAE,GAAG,EAAE;QAEvC,IAAIuC,QAAQI,MAAM,GAAG,GAAG;YACtB,MAAMC,UAAU,MAAM/C,OAAOf,IAAI,CAC/B,IAAIjC,qBAAqB;gBACvBkC,QAAQzB,OAAO0B,qBAAqB;gBACpC6D,QAAQ;oBAACC,SAASP;oBAASQ,OAAO;gBAAI;YACxC;YAEF,IAAIH,QAAQI,MAAM,IAAIJ,QAAQI,MAAM,CAACL,MAAM,GAAG,GAAG;gBAC/C,MAAM,CAACM,MAAM,GAAGL,QAAQI,MAAM;gBAC9B,MAAM,IAAI1B,MACR,CAAC,iBAAiB,EAAEsB,QAAQI,MAAM,CAACL,MAAM,CAAC,iBAAiB,EAAET,OAAO,EAAE,EAAEe,OAAOjD,IAAI,CAAC,EAAEiD,OAAOC,SAAS;YAE1G;QACF;QACAf,oBAAoBC,OAAOe,WAAW,GAAGf,OAAOgB,qBAAqB,GAAGvF;IAC1E,QAASsE,kBAAmB;AAC9B"}

package/dist/core/append-logs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"append-logs.d.ts","sourceRoot":"","sources":["../../src/core/append-logs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AAQnC,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,yCAAyC,CAAC;AAwBnF,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACjB;AAoND;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,gBAAgB,EACxB,SAAS,CAAC,EAAE,qBAAqB,GAChC,OAAO,CAAC,gBAAgB,CAAC,CAmG3B"}
+{"version":3,"file":"append-logs.d.ts","sourceRoot":"","sources":["../../src/core/append-logs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AAOnC,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,yCAAyC,CAAC;AAyBnF,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACjB;AAoND;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,gBAAgB,EACxB,SAAS,CAAC,EAAE,qBAAqB,GAChC,OAAO,CAAC,gBAAgB,CAAC,CAmG3B"}

package/dist/core/append-logs.js

@@ -1,7 +1,7 @@
 import { Buffer } from 'node:buffer';
-import { DEFAULT_HARNESS } from '@shipfox/api-agent-dto';
 import { parseLogRecordLine, parseRawLogRecordLine } from '@shipfox/api-logs-dto';
 import { logger } from '@shipfox/node-opentelemetry';
+import { DEFAULT_HARNESS } from '@shipfox/workflow-document';
 import { config } from '#config.js';
 import { accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped } from '#db/accounting.js';
 import { insertChunk } from '#db/chunks.js';

package/dist/core/append-logs.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/core/append-logs.ts"],"sourcesContent":["import {Buffer} from 'node:buffer';\nimport {DEFAULT_HARNESS, type Harness} from '@shipfox/api-agent-dto';\nimport {\n  type LogRecord,\n  parseLogRecordLine,\n  parseRawLogRecordLine,\n  type RawLogRecord,\n} from '@shipfox/api-logs-dto';\nimport type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {config} from '#config.js';\nimport {accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped} from '#db/accounting.js';\nimport {insertChunk} from '#db/chunks.js';\nimport {db, type Transaction} from '#db/db.js';\nimport {\n  casExtendCommittedLength,\n  getAttemptStream,\n  getOrCreateAttemptStreamWithStatus,\n  setDeclaredTotalBytes,\n} from '#db/streams.js';\nimport {\n  type LogRecordMetricKind,\n  recordAppendedCount,\n  streamClosedCount,\n  streamOpenedCount,\n} from '#metrics/instance.js';\nimport {allowedBudget} from './budget.js';\nimport {closeStream, controlTombstone} from './close-stream.js';\nimport {MalformedLogChunkError, OffsetGapError} from './errors.js';\nimport {parseSessionRecord} from './session/parse-session.js';\nimport type {AgentSessionRecord} from './session/session-record.js';\n\nexport interface AppendLogsParams {\n  jobId: string;\n  workspaceId: string;\n  projectId: string;\n  workflowRunAttemptId: string;\n  stepId: string;\n  attempt: number;\n  offset: number;\n  body: Buffer;\n}\n\nexport interface AppendLogsResult {\n  committedLength: number;\n  capped: boolean;\n}\n\ninterface ParsedBody {\n  declaredTotalBytes?: number;\n  records: RawLogRecord[];\n  hasAgentSessionRecord: boolean;\n}\n\n/**\n * Pure pre-transaction parse. Requires whole, newline-terminated lines so\n * `committed_length` always lands on a line boundary (one body = one chunk = whole\n * lines; no line ever spans two chunks). Runs before any lock is taken, so a\n * malformed or large body never holds a row. The offset CAS uses the raw append byte length; the\n * budget charges the normalized body built from these parsed records.\n *\n * Each line is validated against the raw record union (a forged server-only\n * `capped`/`runner_lost` fails here); the declared total is pulled from an `end`\n * record. A line that is a valid server-only record under the read union surfaces\n * its type via `forgedType` for the narrowed audit warn.\n */\nfunction parseAppendBody(body: Buffer): ParsedBody {\n  if (body.length === 0) return {records: [], hasAgentSessionRecord: false};\n\n  const text = body.toString('utf8');\n  if (!text.endsWith('\\n')) {\n    throw new MalformedLogChunkError('append body must end with a newline (whole records only)');\n  }\n  const lines = text.split('\\n');\n  lines.pop();\n\n  let declaredTotalBytes: number | undefined;\n  const records: RawLogRecord[] = [];\n  let hasAgentSessionRecord = false;\n  for (const line of lines) {\n    let record: ReturnType<typeof parseRawLogRecordLine>;\n    try {\n      record = parseRawLogRecordLine(line);\n    } catch {\n      throw new MalformedLogChunkError(\n        'append body contains an invalid NDJSON record',\n        detectForgedType(line),\n      );\n    }\n    records.push(record);\n    if (record.type === 'end') {\n      declaredTotalBytes = record.total_bytes;\n    }\n    // An agent_session line is one whole entry in one record; bound its size here (the DTO\n    // schema leaves `data` uncapped because it cannot read this runtime config). An over-cap\n    // line is rejected so a single entry can never blow the request body or the spool window.\n    if (\n      record.type === 'agent_session' &&\n      Buffer.byteLength(line, 'utf8') > config.LOG_MAX_SESSION_LINE_BYTES\n    ) {\n      throw new MalformedLogChunkError(\n        `agent_session line exceeds ${config.LOG_MAX_SESSION_LINE_BYTES} bytes`,\n      );\n    }\n    if (record.type === 'agent_session') {\n      hasAgentSessionRecord = true;\n    }\n  }\n\n  return declaredTotalBytes === undefined\n    ? {records, hasAgentSessionRecord}\n    : {declaredTotalBytes, records, hasAgentSessionRecord};\n}\n\n/**\n * A line that fails the raw write union but is a valid record under the read union\n * can only be a server-only `capped`/`runner_lost` tombstone — i.e. a forgery\n * attempt. Returns its type for the audit warn, or undefined for plain garbage.\n */\nfunction detectForgedType(line: string): string | undefined {\n  try {\n    return parseLogRecordLine(line).type;\n  } catch {\n    return undefined;\n  }\n}\n\nasync function getSessionHarness(\n  workflows: WorkflowsModuleClient | undefined,\n  stepId: string,\n): Promise<Harness> {\n  return workflows ? (await workflows.getStepLogContext({stepId})).harness : DEFAULT_HARNESS;\n}\n\n/**\n * Empty-body heartbeat: report the current committed length without materializing\n * a stream, so a runner cannot mint unbounded rows with empty appends.\n */\nasync function readHeartbeat(tx: Transaction, params: AppendLogsParams): Promise<AppendLogsResult> {\n  const existing = await getAttemptStream(tx, {\n    jobId: params.jobId,\n    stepId: params.stepId,\n    attempt: params.attempt,\n  });\n  return {\n    committedLength: existing?.committedLength ?? 0,\n    capped: await isJobCapped(tx, params.jobId),\n  };\n}\n\ninterface StoreChunkParams {\n  params: AppendLogsParams;\n  streamId: string;\n  body: Buffer;\n  committedLength: number;\n  declaredTotalBytes: number | undefined;\n}\n\ninterface StoreChunkResult extends AppendLogsResult {\n  /**\n   * Whether the runner chunk was persisted. False only when the job was already\n   * capped and the body (including any `end` record) was dropped, so the stream is\n   * not whole and must not be declared-closed.\n   */\n  stored: boolean;\n  recordCounts: Partial<Record<LogRecord['type'], number>>;\n}\n\n/**\n * Accrues the stored bytes, persists the chunk, and trips the per-job cap when\n * this append crosses the budget. Runs only after the offset-CAS extended\n * `committed_length`, so the committed length already reflects the accepted bytes.\n */\nasync function storeChunk(\n  tx: Transaction,\n  {params, streamId, body, committedLength, declaredTotalBytes}: StoreChunkParams,\n): Promise<StoreChunkResult> {\n  await ensureJobAccounting(tx, {jobId: params.jobId, workspaceId: params.workspaceId});\n  const storedByteLen = body.length;\n  const accrued = await accrueStoredBytes(tx, {jobId: params.jobId, delta: storedByteLen});\n\n  // Already capped: accept-and-drop. committed_length has advanced so the runner\n  // drains its spool cleanly instead of retry-looping; nothing is stored.\n  if (!accrued) return {committedLength, capped: true, stored: false, recordCounts: {}};\n\n  await insertChunk(tx, {\n    streamId,\n    streamOffset: params.offset,\n    byteLen: storedByteLen,\n    data: body,\n    origin: 'runner',\n  });\n  if (declaredTotalBytes !== undefined) {\n    await setDeclaredTotalBytes(tx, {streamId, declaredTotalBytes});\n  }\n\n  const allowed = allowedBudget({\n    baseBytes: config.LOG_BUDGET_BASE_BYTES,\n    ratePerMinuteBytes: config.LOG_BUDGET_RATE_BYTES_PER_MINUTE,\n    elapsedMs: Date.now() - accrued.startedAt.getTime(),\n  });\n  if (accrued.used <= allowed) {\n    return {committedLength, capped: false, stored: true, recordCounts: {}};\n  }\n\n  // Over budget. No hard ceiling: this crossing append is stored in full (overshoot\n  // bounded by one body). Claim the cap once and inject an in-band `capped` tombstone\n  // for the winner.\n  const won = await claimCap(tx, params.jobId);\n  if (won) {\n    const tombstone = controlTombstone('capped');\n    await insertChunk(tx, {\n      streamId,\n      streamOffset: committedLength,\n      byteLen: tombstone.length,\n      data: tombstone,\n      origin: 'control',\n    });\n  }\n  return {\n    committedLength,\n    capped: true,\n    stored: true,\n    recordCounts: won ? {capped: 1} : {},\n  };\n}\n\nfunction buildStoredBody(\n  records: readonly RawLogRecord[],\n  harness: Harness,\n): {body: Buffer; recordCounts: Partial<Record<LogRecord['type'], number>>} {\n  const storedRecords: LogRecord[] = [];\n  for (const record of records) {\n    if (record.type !== 'agent_session') {\n      storedRecords.push(record);\n      continue;\n    }\n\n    for (const row of parseSessionRecord(agentSessionRecord(record), harness)) {\n      storedRecords.push({v: 1, ts: row.timestamp, type: 'agent_session', row});\n    }\n  }\n\n  const body = Buffer.from(storedRecords.map((record) => `${JSON.stringify(record)}\\n`).join(''));\n  const recordCounts: Partial<Record<LogRecord['type'], number>> = {};\n  for (const record of storedRecords) {\n    recordCounts[record.type] = (recordCounts[record.type] ?? 0) + 1;\n  }\n\n  return {body, recordCounts};\n}\n\nfunction agentSessionRecord(\n  record: Extract<RawLogRecord, {type: 'agent_session'}>,\n): AgentSessionRecord {\n  return {data: record.data, ts: record.ts};\n}\n\n/**\n * Concurrency is serialized through Postgres row locks taken implicitly by the\n * conditional UPDATEs, not an explicit `SELECT ... FOR UPDATE`. Appends for one\n * job contend on its single accounting row, so the path is multi-instance safe\n * but not lock-free.\n */\nexport async function appendLogs(\n  params: AppendLogsParams,\n  workflows?: WorkflowsModuleClient,\n): Promise<AppendLogsResult> {\n  let parsed: ParsedBody;\n  try {\n    parsed = parseAppendBody(params.body);\n  } catch (error) {\n    // Narrowed audit: only the detectable forgery case, never the payload or a token.\n    if (error instanceof MalformedLogChunkError && error.forgedType !== undefined) {\n      logger().warn(\n        {\n          jobId: params.jobId,\n          stepId: params.stepId,\n          offendingType: error.forgedType,\n        },\n        'Rejected forged server-only log record on append',\n      );\n    }\n    throw error;\n  }\n  const {declaredTotalBytes} = parsed;\n  const sessionHarness = parsed.hasAgentSessionRecord\n    ? await getSessionHarness(workflows, params.stepId)\n    : DEFAULT_HARNESS;\n  const stored = buildStoredBody(parsed.records, sessionHarness);\n  const commitByteLen = params.body.length;\n  const metrics = {\n    recordCounts: {} as Partial<Record<LogRecordMetricKind, number>>,\n    streamClosedReason: undefined as 'declared' | undefined,\n    streamOpened: false,\n  };\n\n  const result = await db().transaction(async (tx) => {\n    if (commitByteLen === 0) return readHeartbeat(tx, params);\n\n    const {created, stream} = await getOrCreateAttemptStreamWithStatus(tx, {\n      jobId: params.jobId,\n      stepId: params.stepId,\n      attempt: params.attempt,\n      workspaceId: params.workspaceId,\n      projectId: params.projectId,\n      workflowRunAttemptId: params.workflowRunAttemptId,\n    });\n    metrics.streamOpened = created;\n\n    // Closed stream (the runner's end already landed, or the job-terminated sweep ran):\n    // accept-and-drop so a late chunk can never race compaction. committed_length is\n    // frozen at close, so this reports the final offset and the runner stops cleanly.\n    if (stream.state === 'closed') {\n      return {committedLength: stream.committedLength, capped: await isJobCapped(tx, params.jobId)};\n    }\n\n    const cas = await casExtendCommittedLength(tx, {\n      streamId: stream.id,\n      offset: params.offset,\n      byteLen: commitByteLen,\n    });\n    if (cas.outcome === 'gap') throw new OffsetGapError(cas.committedLength);\n    if (cas.outcome === 'retry') {\n      return {committedLength: cas.committedLength, capped: await isJobCapped(tx, params.jobId)};\n    }\n\n    const {\n      recordCounts,\n      stored: chunkStored,\n      ...result\n    } = await storeChunk(tx, {\n      params,\n      streamId: stream.id,\n      body: stored.body,\n      committedLength: cas.committedLength,\n      declaredTotalBytes,\n    });\n    if (chunkStored) {\n      addRecordCounts(metrics.recordCounts, stored.recordCounts);\n    }\n    addRecordCounts(metrics.recordCounts, recordCounts);\n\n    // The runner's end record was committed in this append (offset-CAS guarantees\n    // everything before it is already committed), so the stream is whole. Declared-close\n    // it in-band so compaction starts at once instead of waiting for the timeout sweep.\n    // Only when the chunk was actually stored: an end body dropped because the job was\n    // already capped persists nothing, so the stream is not whole and stays open for the\n    // timeout sweep to close it as truncated.\n    if (declaredTotalBytes !== undefined && chunkStored) {\n      const closed = await closeStream(tx, {streamId: stream.id, reason: 'declared'});\n      if (closed) metrics.streamClosedReason = 'declared';\n    }\n\n    return result;\n  });\n\n  if (metrics.streamOpened) streamOpenedCount.add(1);\n  for (const [kind, count] of Object.entries(metrics.recordCounts)) {\n    if (count > 0) recordAppendedCount.add(count, {kind: kind as LogRecordMetricKind});\n  }\n  if (metrics.streamClosedReason) {\n    streamClosedCount.add(1, {reason: metrics.streamClosedReason});\n  }\n\n  return result;\n}\n\nfunction addRecordCounts(\n  target: Partial<Record<LogRecordMetricKind, number>>,\n  source: Partial<Record<LogRecordMetricKind, number>>,\n): void {\n  for (const [kind, count] of Object.entries(source)) {\n    target[kind as LogRecordMetricKind] = (target[kind as LogRecordMetricKind] ?? 0) + (count ?? 0);\n  }\n}\n"],"names":["Buffer","DEFAULT_HARNESS","parseLogRecordLine","parseRawLogRecordLine","logger","config","accrueStoredBytes","claimCap","ensureJobAccounting","isJobCapped","insertChunk","db","casExtendCommittedLength","getAttemptStream","getOrCreateAttemptStreamWithStatus","setDeclaredTotalBytes","recordAppendedCount","streamClosedCount","streamOpenedCount","allowedBudget","closeStream","controlTombstone","MalformedLogChunkError","OffsetGapError","parseSessionRecord","parseAppendBody","body","length","records","hasAgentSessionRecord","text","toString","endsWith","lines","split","pop","declaredTotalBytes","line","record","detectForgedType","push","type","total_bytes","byteLength","LOG_MAX_SESSION_LINE_BYTES","undefined","getSessionHarness","workflows","stepId","getStepLogContext","harness","readHeartbeat","tx","params","existing","jobId","attempt","committedLength","capped","storeChunk","streamId","workspaceId","storedByteLen","accrued","delta","stored","recordCounts","streamOffset","offset","byteLen","data","origin","allowed","baseBytes","LOG_BUDGET_BASE_BYTES","ratePerMinuteBytes","LOG_BUDGET_RATE_BYTES_PER_MINUTE","elapsedMs","Date","now","startedAt","getTime","used","won","tombstone","buildStoredBody","storedRecords","row","agentSessionRecord","v","ts","timestamp","from","map","JSON","stringify","join","appendLogs","parsed","error","forgedType","warn","offendingType","sessionHarness","commitByteLen","metrics","streamClosedReason","streamOpened","result","transaction","created","stream","projectId","workflowRunAttemptId","state","cas","id","outcome","chunkStored","addRecordCounts","closed","reason","add","kind","count","Object","entries","target","source"],"mappings":"AAAA,SAAQA,MAAM,QAAO,cAAc;AACnC,SAAQC,eAAe,QAAqB,yBAAyB;AACrE,SAEEC,kBAAkB,EAClBC,qBAAqB,QAEhB,wBAAwB;AAE/B,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAQC,iBAAiB,EAAEC,QAAQ,EAAEC,mBAAmB,EAAEC,WAAW,QAAO,oBAAoB;AAChG,SAAQC,WAAW,QAAO,gBAAgB;AAC1C,SAAQC,EAAE,QAAyB,YAAY;AAC/C,SACEC,wBAAwB,EACxBC,gBAAgB,EAChBC,kCAAkC,EAClCC,qBAAqB,QAChB,iBAAiB;AACxB,SAEEC,mBAAmB,EACnBC,iBAAiB,EACjBC,iBAAiB,QACZ,uBAAuB;AAC9B,SAAQC,aAAa,QAAO,cAAc;AAC1C,SAAQC,WAAW,EAAEC,gBAAgB,QAAO,oBAAoB;AAChE,SAAQC,sBAAsB,EAAEC,cAAc,QAAO,cAAc;AACnE,SAAQC,kBAAkB,QAAO,6BAA6B;AAyB9D;;;;;;;;;;;CAWC,GACD,SAASC,gBAAgBC,IAAY;IACnC,IAAIA,KAAKC,MAAM,KAAK,GAAG,OAAO;QAACC,SAAS,EAAE;QAAEC,uBAAuB;IAAK;IAExE,MAAMC,OAAOJ,KAAKK,QAAQ,CAAC;IAC3B,IAAI,CAACD,KAAKE,QAAQ,CAAC,OAAO;QACxB,MAAM,IAAIV,uBAAuB;IACnC;IACA,MAAMW,QAAQH,KAAKI,KAAK,CAAC;IACzBD,MAAME,GAAG;IAET,IAAIC;IACJ,MAAMR,UAA0B,EAAE;IAClC,IAAIC,wBAAwB;IAC5B,KAAK,MAAMQ,QAAQJ,MAAO;QACxB,IAAIK;QACJ,IAAI;YACFA,SAASnC,sBAAsBkC;QACjC,EAAE,OAAM;YACN,MAAM,IAAIf,uBACR,iDACAiB,iBAAiBF;QAErB;QACAT,QAAQY,IAAI,CAACF;QACb,IAAIA,OAAOG,IAAI,KAAK,OAAO;YACzBL,qBAAqBE,OAAOI,WAAW;QACzC;QACA,uFAAuF;QACvF,yFAAyF;QACzF,0FAA0F;QAC1F,IACEJ,OAAOG,IAAI,KAAK,mBAChBzC,OAAO2C,UAAU,CAACN,MAAM,UAAUhC,OAAOuC,0BAA0B,EACnE;YACA,MAAM,IAAItB,uBACR,CAAC,2BAA2B,EAAEjB,OAAOuC,0BAA0B,CAAC,MAAM,CAAC;QAE3E;QACA,IAAIN,OAAOG,IAAI,KAAK,iBAAiB;YACnCZ,wBAAwB;QAC1B;IACF;IAEA,OAAOO,uBAAuBS,YAC1B;QAACjB;QAASC;IAAqB,IAC/B;QAACO;QAAoBR;QAASC;IAAqB;AACzD;AAEA;;;;CAIC,GACD,SAASU,iBAAiBF,IAAY;IACpC,IAAI;QACF,OAAOnC,mBAAmBmC,MAAMI,IAAI;IACtC,EAAE,OAAM;QACN,OAAOI;IACT;AACF;AAEA,eAAeC,kBACbC,SAA4C,EAC5CC,MAAc;IAEd,OAAOD,YAAY,AAAC,CAAA,MAAMA,UAAUE,iBAAiB,CAAC;QAACD;IAAM,EAAC,EAAGE,OAAO,GAAGjD;AAC7E;AAEA;;;CAGC,GACD,eAAekD,cAAcC,EAAe,EAAEC,MAAwB;IACpE,MAAMC,WAAW,MAAMzC,iBAAiBuC,IAAI;QAC1CG,OAAOF,OAAOE,KAAK;QACnBP,QAAQK,OAAOL,MAAM;QACrBQ,SAASH,OAAOG,OAAO;IACzB;IACA,OAAO;QACLC,iBAAiBH,UAAUG,mBAAmB;QAC9CC,QAAQ,MAAMjD,YAAY2C,IAAIC,OAAOE,KAAK;IAC5C;AACF;AAoBA;;;;CAIC,GACD,eAAeI,WACbP,EAAe,EACf,EAACC,MAAM,EAAEO,QAAQ,EAAElC,IAAI,EAAE+B,eAAe,EAAErB,kBAAkB,EAAmB;IAE/E,MAAM5B,oBAAoB4C,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAEM,aAAaR,OAAOQ,WAAW;IAAA;IACnF,MAAMC,gBAAgBpC,KAAKC,MAAM;IACjC,MAAMoC,UAAU,MAAMzD,kBAAkB8C,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAES,OAAOF;IAAa;IAEtF,+EAA+E;IAC/E,wEAAwE;IACxE,IAAI,CAACC,SAAS,OAAO;QAACN;QAAiBC,QAAQ;QAAMO,QAAQ;QAAOC,cAAc,CAAC;IAAC;IAEpF,MAAMxD,YAAY0C,IAAI;QACpBQ;QACAO,cAAcd,OAAOe,MAAM;QAC3BC,SAASP;QACTQ,MAAM5C;QACN6C,QAAQ;IACV;IACA,IAAInC,uBAAuBS,WAAW;QACpC,MAAM9B,sBAAsBqC,IAAI;YAACQ;YAAUxB;QAAkB;IAC/D;IAEA,MAAMoC,UAAUrD,cAAc;QAC5BsD,WAAWpE,OAAOqE,qBAAqB;QACvCC,oBAAoBtE,OAAOuE,gCAAgC;QAC3DC,WAAWC,KAAKC,GAAG,KAAKhB,QAAQiB,SAAS,CAACC,OAAO;IACnD;IACA,IAAIlB,QAAQmB,IAAI,IAAIV,SAAS;QAC3B,OAAO;YAACf;YAAiBC,QAAQ;YAAOO,QAAQ;YAAMC,cAAc,CAAC;QAAC;IACxE;IAEA,kFAAkF;IAClF,oFAAoF;IACpF,kBAAkB;IAClB,MAAMiB,MAAM,MAAM5E,SAAS6C,IAAIC,OAAOE,KAAK;IAC3C,IAAI4B,KAAK;QACP,MAAMC,YAAY/D,iBAAiB;QACnC,MAAMX,YAAY0C,IAAI;YACpBQ;YACAO,cAAcV;YACdY,SAASe,UAAUzD,MAAM;YACzB2C,MAAMc;YACNb,QAAQ;QACV;IACF;IACA,OAAO;QACLd;QACAC,QAAQ;QACRO,QAAQ;QACRC,cAAciB,MAAM;YAACzB,QAAQ;QAAC,IAAI,CAAC;IACrC;AACF;AAEA,SAAS2B,gBACPzD,OAAgC,EAChCsB,OAAgB;IAEhB,MAAMoC,gBAA6B,EAAE;IACrC,KAAK,MAAMhD,UAAUV,QAAS;QAC5B,IAAIU,OAAOG,IAAI,KAAK,iBAAiB;YACnC6C,cAAc9C,IAAI,CAACF;YACnB;QACF;QAEA,KAAK,MAAMiD,OAAO/D,mBAAmBgE,mBAAmBlD,SAASY,SAAU;YACzEoC,cAAc9C,IAAI,CAAC;gBAACiD,GAAG;gBAAGC,IAAIH,IAAII,SAAS;gBAAElD,MAAM;gBAAiB8C;YAAG;QACzE;IACF;IAEA,MAAM7D,OAAO1B,OAAO4F,IAAI,CAACN,cAAcO,GAAG,CAAC,CAACvD,SAAW,GAAGwD,KAAKC,SAAS,CAACzD,QAAQ,EAAE,CAAC,EAAE0D,IAAI,CAAC;IAC3F,MAAM9B,eAA2D,CAAC;IAClE,KAAK,MAAM5B,UAAUgD,cAAe;QAClCpB,YAAY,CAAC5B,OAAOG,IAAI,CAAC,GAAG,AAACyB,CAAAA,YAAY,CAAC5B,OAAOG,IAAI,CAAC,IAAI,CAAA,IAAK;IACjE;IAEA,OAAO;QAACf;QAAMwC;IAAY;AAC5B;AAEA,SAASsB,mBACPlD,MAAsD;IAEtD,OAAO;QAACgC,MAAMhC,OAAOgC,IAAI;QAAEoB,IAAIpD,OAAOoD,EAAE;IAAA;AAC1C;AAEA;;;;;CAKC,GACD,OAAO,eAAeO,WACpB5C,MAAwB,EACxBN,SAAiC;IAEjC,IAAImD;IACJ,IAAI;QACFA,SAASzE,gBAAgB4B,OAAO3B,IAAI;IACtC,EAAE,OAAOyE,OAAO;QACd,kFAAkF;QAClF,IAAIA,iBAAiB7E,0BAA0B6E,MAAMC,UAAU,KAAKvD,WAAW;YAC7EzC,SAASiG,IAAI,CACX;gBACE9C,OAAOF,OAAOE,KAAK;gBACnBP,QAAQK,OAAOL,MAAM;gBACrBsD,eAAeH,MAAMC,UAAU;YACjC,GACA;QAEJ;QACA,MAAMD;IACR;IACA,MAAM,EAAC/D,kBAAkB,EAAC,GAAG8D;IAC7B,MAAMK,iBAAiBL,OAAOrE,qBAAqB,GAC/C,MAAMiB,kBAAkBC,WAAWM,OAAOL,MAAM,IAChD/C;IACJ,MAAMgE,SAASoB,gBAAgBa,OAAOtE,OAAO,EAAE2E;IAC/C,MAAMC,gBAAgBnD,OAAO3B,IAAI,CAACC,MAAM;IACxC,MAAM8E,UAAU;QACdvC,cAAc,CAAC;QACfwC,oBAAoB7D;QACpB8D,cAAc;IAChB;IAEA,MAAMC,SAAS,MAAMjG,KAAKkG,WAAW,CAAC,OAAOzD;QAC3C,IAAIoD,kBAAkB,GAAG,OAAOrD,cAAcC,IAAIC;QAElD,MAAM,EAACyD,OAAO,EAAEC,MAAM,EAAC,GAAG,MAAMjG,mCAAmCsC,IAAI;YACrEG,OAAOF,OAAOE,KAAK;YACnBP,QAAQK,OAAOL,MAAM;YACrBQ,SAASH,OAAOG,OAAO;YACvBK,aAAaR,OAAOQ,WAAW;YAC/BmD,WAAW3D,OAAO2D,SAAS;YAC3BC,sBAAsB5D,OAAO4D,oBAAoB;QACnD;QACAR,QAAQE,YAAY,GAAGG;QAEvB,oFAAoF;QACpF,iFAAiF;QACjF,kFAAkF;QAClF,IAAIC,OAAOG,KAAK,KAAK,UAAU;YAC7B,OAAO;gBAACzD,iBAAiBsD,OAAOtD,eAAe;gBAAEC,QAAQ,MAAMjD,YAAY2C,IAAIC,OAAOE,KAAK;YAAC;QAC9F;QAEA,MAAM4D,MAAM,MAAMvG,yBAAyBwC,IAAI;YAC7CQ,UAAUmD,OAAOK,EAAE;YACnBhD,QAAQf,OAAOe,MAAM;YACrBC,SAASmC;QACX;QACA,IAAIW,IAAIE,OAAO,KAAK,OAAO,MAAM,IAAI9F,eAAe4F,IAAI1D,eAAe;QACvE,IAAI0D,IAAIE,OAAO,KAAK,SAAS;YAC3B,OAAO;gBAAC5D,iBAAiB0D,IAAI1D,eAAe;gBAAEC,QAAQ,MAAMjD,YAAY2C,IAAIC,OAAOE,KAAK;YAAC;QAC3F;QAEA,MAAM,EACJW,YAAY,EACZD,QAAQqD,WAAW,EACnB,GAAGV,QACJ,GAAG,MAAMjD,WAAWP,IAAI;YACvBC;YACAO,UAAUmD,OAAOK,EAAE;YACnB1F,MAAMuC,OAAOvC,IAAI;YACjB+B,iBAAiB0D,IAAI1D,eAAe;YACpCrB;QACF;QACA,IAAIkF,aAAa;YACfC,gBAAgBd,QAAQvC,YAAY,EAAED,OAAOC,YAAY;QAC3D;QACAqD,gBAAgBd,QAAQvC,YAAY,EAAEA;QAEtC,8EAA8E;QAC9E,qFAAqF;QACrF,oFAAoF;QACpF,mFAAmF;QACnF,qFAAqF;QACrF,0CAA0C;QAC1C,IAAI9B,uBAAuBS,aAAayE,aAAa;YACnD,MAAME,SAAS,MAAMpG,YAAYgC,IAAI;gBAACQ,UAAUmD,OAAOK,EAAE;gBAAEK,QAAQ;YAAU;YAC7E,IAAID,QAAQf,QAAQC,kBAAkB,GAAG;QAC3C;QAEA,OAAOE;IACT;IAEA,IAAIH,QAAQE,YAAY,EAAEzF,kBAAkBwG,GAAG,CAAC;IAChD,KAAK,MAAM,CAACC,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACrB,QAAQvC,YAAY,EAAG;QAChE,IAAI0D,QAAQ,GAAG5G,oBAAoB0G,GAAG,CAACE,OAAO;YAACD,MAAMA;QAA2B;IAClF;IACA,IAAIlB,QAAQC,kBAAkB,EAAE;QAC9BzF,kBAAkByG,GAAG,CAAC,GAAG;YAACD,QAAQhB,QAAQC,kBAAkB;QAAA;IAC9D;IAEA,OAAOE;AACT;AAEA,SAASW,gBACPQ,MAAoD,EACpDC,MAAoD;IAEpD,KAAK,MAAM,CAACL,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACE,QAAS;QAClDD,MAAM,CAACJ,KAA4B,GAAG,AAACI,CAAAA,MAAM,CAACJ,KAA4B,IAAI,CAAA,IAAMC,CAAAA,SAAS,CAAA;IAC/F;AACF"}
+{"version":3,"sources":["../../src/core/append-logs.ts"],"sourcesContent":["import {Buffer} from 'node:buffer';\nimport {\n  type LogRecord,\n  parseLogRecordLine,\n  parseRawLogRecordLine,\n  type RawLogRecord,\n} from '@shipfox/api-logs-dto';\nimport type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {DEFAULT_HARNESS, type Harness} from '@shipfox/workflow-document';\nimport {config} from '#config.js';\nimport {accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped} from '#db/accounting.js';\nimport {insertChunk} from '#db/chunks.js';\nimport {db, type Transaction} from '#db/db.js';\nimport {\n  casExtendCommittedLength,\n  getAttemptStream,\n  getOrCreateAttemptStreamWithStatus,\n  setDeclaredTotalBytes,\n} from '#db/streams.js';\nimport {\n  type LogRecordMetricKind,\n  recordAppendedCount,\n  streamClosedCount,\n  streamOpenedCount,\n} from '#metrics/instance.js';\nimport {allowedBudget} from './budget.js';\nimport {closeStream, controlTombstone} from './close-stream.js';\nimport {MalformedLogChunkError, OffsetGapError} from './errors.js';\nimport {parseSessionRecord} from './session/parse-session.js';\nimport type {AgentSessionRecord} from './session/session-record.js';\n\nexport interface AppendLogsParams {\n  jobId: string;\n  workspaceId: string;\n  projectId: string;\n  workflowRunAttemptId: string;\n  stepId: string;\n  attempt: number;\n  offset: number;\n  body: Buffer;\n}\n\nexport interface AppendLogsResult {\n  committedLength: number;\n  capped: boolean;\n}\n\ninterface ParsedBody {\n  declaredTotalBytes?: number;\n  records: RawLogRecord[];\n  hasAgentSessionRecord: boolean;\n}\n\n/**\n * Pure pre-transaction parse. Requires whole, newline-terminated lines so\n * `committed_length` always lands on a line boundary (one body = one chunk = whole\n * lines; no line ever spans two chunks). Runs before any lock is taken, so a\n * malformed or large body never holds a row. The offset CAS uses the raw append byte length; the\n * budget charges the normalized body built from these parsed records.\n *\n * Each line is validated against the raw record union (a forged server-only\n * `capped`/`runner_lost` fails here); the declared total is pulled from an `end`\n * record. A line that is a valid server-only record under the read union surfaces\n * its type via `forgedType` for the narrowed audit warn.\n */\nfunction parseAppendBody(body: Buffer): ParsedBody {\n  if (body.length === 0) return {records: [], hasAgentSessionRecord: false};\n\n  const text = body.toString('utf8');\n  if (!text.endsWith('\\n')) {\n    throw new MalformedLogChunkError('append body must end with a newline (whole records only)');\n  }\n  const lines = text.split('\\n');\n  lines.pop();\n\n  let declaredTotalBytes: number | undefined;\n  const records: RawLogRecord[] = [];\n  let hasAgentSessionRecord = false;\n  for (const line of lines) {\n    let record: ReturnType<typeof parseRawLogRecordLine>;\n    try {\n      record = parseRawLogRecordLine(line);\n    } catch {\n      throw new MalformedLogChunkError(\n        'append body contains an invalid NDJSON record',\n        detectForgedType(line),\n      );\n    }\n    records.push(record);\n    if (record.type === 'end') {\n      declaredTotalBytes = record.total_bytes;\n    }\n    // An agent_session line is one whole entry in one record; bound its size here (the DTO\n    // schema leaves `data` uncapped because it cannot read this runtime config). An over-cap\n    // line is rejected so a single entry can never blow the request body or the spool window.\n    if (\n      record.type === 'agent_session' &&\n      Buffer.byteLength(line, 'utf8') > config.LOG_MAX_SESSION_LINE_BYTES\n    ) {\n      throw new MalformedLogChunkError(\n        `agent_session line exceeds ${config.LOG_MAX_SESSION_LINE_BYTES} bytes`,\n      );\n    }\n    if (record.type === 'agent_session') {\n      hasAgentSessionRecord = true;\n    }\n  }\n\n  return declaredTotalBytes === undefined\n    ? {records, hasAgentSessionRecord}\n    : {declaredTotalBytes, records, hasAgentSessionRecord};\n}\n\n/**\n * A line that fails the raw write union but is a valid record under the read union\n * can only be a server-only `capped`/`runner_lost` tombstone — i.e. a forgery\n * attempt. Returns its type for the audit warn, or undefined for plain garbage.\n */\nfunction detectForgedType(line: string): string | undefined {\n  try {\n    return parseLogRecordLine(line).type;\n  } catch {\n    return undefined;\n  }\n}\n\nasync function getSessionHarness(\n  workflows: WorkflowsModuleClient | undefined,\n  stepId: string,\n): Promise<Harness> {\n  return workflows ? (await workflows.getStepLogContext({stepId})).harness : DEFAULT_HARNESS;\n}\n\n/**\n * Empty-body heartbeat: report the current committed length without materializing\n * a stream, so a runner cannot mint unbounded rows with empty appends.\n */\nasync function readHeartbeat(tx: Transaction, params: AppendLogsParams): Promise<AppendLogsResult> {\n  const existing = await getAttemptStream(tx, {\n    jobId: params.jobId,\n    stepId: params.stepId,\n    attempt: params.attempt,\n  });\n  return {\n    committedLength: existing?.committedLength ?? 0,\n    capped: await isJobCapped(tx, params.jobId),\n  };\n}\n\ninterface StoreChunkParams {\n  params: AppendLogsParams;\n  streamId: string;\n  body: Buffer;\n  committedLength: number;\n  declaredTotalBytes: number | undefined;\n}\n\ninterface StoreChunkResult extends AppendLogsResult {\n  /**\n   * Whether the runner chunk was persisted. False only when the job was already\n   * capped and the body (including any `end` record) was dropped, so the stream is\n   * not whole and must not be declared-closed.\n   */\n  stored: boolean;\n  recordCounts: Partial<Record<LogRecord['type'], number>>;\n}\n\n/**\n * Accrues the stored bytes, persists the chunk, and trips the per-job cap when\n * this append crosses the budget. Runs only after the offset-CAS extended\n * `committed_length`, so the committed length already reflects the accepted bytes.\n */\nasync function storeChunk(\n  tx: Transaction,\n  {params, streamId, body, committedLength, declaredTotalBytes}: StoreChunkParams,\n): Promise<StoreChunkResult> {\n  await ensureJobAccounting(tx, {jobId: params.jobId, workspaceId: params.workspaceId});\n  const storedByteLen = body.length;\n  const accrued = await accrueStoredBytes(tx, {jobId: params.jobId, delta: storedByteLen});\n\n  // Already capped: accept-and-drop. committed_length has advanced so the runner\n  // drains its spool cleanly instead of retry-looping; nothing is stored.\n  if (!accrued) return {committedLength, capped: true, stored: false, recordCounts: {}};\n\n  await insertChunk(tx, {\n    streamId,\n    streamOffset: params.offset,\n    byteLen: storedByteLen,\n    data: body,\n    origin: 'runner',\n  });\n  if (declaredTotalBytes !== undefined) {\n    await setDeclaredTotalBytes(tx, {streamId, declaredTotalBytes});\n  }\n\n  const allowed = allowedBudget({\n    baseBytes: config.LOG_BUDGET_BASE_BYTES,\n    ratePerMinuteBytes: config.LOG_BUDGET_RATE_BYTES_PER_MINUTE,\n    elapsedMs: Date.now() - accrued.startedAt.getTime(),\n  });\n  if (accrued.used <= allowed) {\n    return {committedLength, capped: false, stored: true, recordCounts: {}};\n  }\n\n  // Over budget. No hard ceiling: this crossing append is stored in full (overshoot\n  // bounded by one body). Claim the cap once and inject an in-band `capped` tombstone\n  // for the winner.\n  const won = await claimCap(tx, params.jobId);\n  if (won) {\n    const tombstone = controlTombstone('capped');\n    await insertChunk(tx, {\n      streamId,\n      streamOffset: committedLength,\n      byteLen: tombstone.length,\n      data: tombstone,\n      origin: 'control',\n    });\n  }\n  return {\n    committedLength,\n    capped: true,\n    stored: true,\n    recordCounts: won ? {capped: 1} : {},\n  };\n}\n\nfunction buildStoredBody(\n  records: readonly RawLogRecord[],\n  harness: Harness,\n): {body: Buffer; recordCounts: Partial<Record<LogRecord['type'], number>>} {\n  const storedRecords: LogRecord[] = [];\n  for (const record of records) {\n    if (record.type !== 'agent_session') {\n      storedRecords.push(record);\n      continue;\n    }\n\n    for (const row of parseSessionRecord(agentSessionRecord(record), harness)) {\n      storedRecords.push({v: 1, ts: row.timestamp, type: 'agent_session', row});\n    }\n  }\n\n  const body = Buffer.from(storedRecords.map((record) => `${JSON.stringify(record)}\\n`).join(''));\n  const recordCounts: Partial<Record<LogRecord['type'], number>> = {};\n  for (const record of storedRecords) {\n    recordCounts[record.type] = (recordCounts[record.type] ?? 0) + 1;\n  }\n\n  return {body, recordCounts};\n}\n\nfunction agentSessionRecord(\n  record: Extract<RawLogRecord, {type: 'agent_session'}>,\n): AgentSessionRecord {\n  return {data: record.data, ts: record.ts};\n}\n\n/**\n * Concurrency is serialized through Postgres row locks taken implicitly by the\n * conditional UPDATEs, not an explicit `SELECT ... FOR UPDATE`. Appends for one\n * job contend on its single accounting row, so the path is multi-instance safe\n * but not lock-free.\n */\nexport async function appendLogs(\n  params: AppendLogsParams,\n  workflows?: WorkflowsModuleClient,\n): Promise<AppendLogsResult> {\n  let parsed: ParsedBody;\n  try {\n    parsed = parseAppendBody(params.body);\n  } catch (error) {\n    // Narrowed audit: only the detectable forgery case, never the payload or a token.\n    if (error instanceof MalformedLogChunkError && error.forgedType !== undefined) {\n      logger().warn(\n        {\n          jobId: params.jobId,\n          stepId: params.stepId,\n          offendingType: error.forgedType,\n        },\n        'Rejected forged server-only log record on append',\n      );\n    }\n    throw error;\n  }\n  const {declaredTotalBytes} = parsed;\n  const sessionHarness = parsed.hasAgentSessionRecord\n    ? await getSessionHarness(workflows, params.stepId)\n    : DEFAULT_HARNESS;\n  const stored = buildStoredBody(parsed.records, sessionHarness);\n  const commitByteLen = params.body.length;\n  const metrics = {\n    recordCounts: {} as Partial<Record<LogRecordMetricKind, number>>,\n    streamClosedReason: undefined as 'declared' | undefined,\n    streamOpened: false,\n  };\n\n  const result = await db().transaction(async (tx) => {\n    if (commitByteLen === 0) return readHeartbeat(tx, params);\n\n    const {created, stream} = await getOrCreateAttemptStreamWithStatus(tx, {\n      jobId: params.jobId,\n      stepId: params.stepId,\n      attempt: params.attempt,\n      workspaceId: params.workspaceId,\n      projectId: params.projectId,\n      workflowRunAttemptId: params.workflowRunAttemptId,\n    });\n    metrics.streamOpened = created;\n\n    // Closed stream (the runner's end already landed, or the job-terminated sweep ran):\n    // accept-and-drop so a late chunk can never race compaction. committed_length is\n    // frozen at close, so this reports the final offset and the runner stops cleanly.\n    if (stream.state === 'closed') {\n      return {committedLength: stream.committedLength, capped: await isJobCapped(tx, params.jobId)};\n    }\n\n    const cas = await casExtendCommittedLength(tx, {\n      streamId: stream.id,\n      offset: params.offset,\n      byteLen: commitByteLen,\n    });\n    if (cas.outcome === 'gap') throw new OffsetGapError(cas.committedLength);\n    if (cas.outcome === 'retry') {\n      return {committedLength: cas.committedLength, capped: await isJobCapped(tx, params.jobId)};\n    }\n\n    const {\n      recordCounts,\n      stored: chunkStored,\n      ...result\n    } = await storeChunk(tx, {\n      params,\n      streamId: stream.id,\n      body: stored.body,\n      committedLength: cas.committedLength,\n      declaredTotalBytes,\n    });\n    if (chunkStored) {\n      addRecordCounts(metrics.recordCounts, stored.recordCounts);\n    }\n    addRecordCounts(metrics.recordCounts, recordCounts);\n\n    // The runner's end record was committed in this append (offset-CAS guarantees\n    // everything before it is already committed), so the stream is whole. Declared-close\n    // it in-band so compaction starts at once instead of waiting for the timeout sweep.\n    // Only when the chunk was actually stored: an end body dropped because the job was\n    // already capped persists nothing, so the stream is not whole and stays open for the\n    // timeout sweep to close it as truncated.\n    if (declaredTotalBytes !== undefined && chunkStored) {\n      const closed = await closeStream(tx, {streamId: stream.id, reason: 'declared'});\n      if (closed) metrics.streamClosedReason = 'declared';\n    }\n\n    return result;\n  });\n\n  if (metrics.streamOpened) streamOpenedCount.add(1);\n  for (const [kind, count] of Object.entries(metrics.recordCounts)) {\n    if (count > 0) recordAppendedCount.add(count, {kind: kind as LogRecordMetricKind});\n  }\n  if (metrics.streamClosedReason) {\n    streamClosedCount.add(1, {reason: metrics.streamClosedReason});\n  }\n\n  return result;\n}\n\nfunction addRecordCounts(\n  target: Partial<Record<LogRecordMetricKind, number>>,\n  source: Partial<Record<LogRecordMetricKind, number>>,\n): void {\n  for (const [kind, count] of Object.entries(source)) {\n    target[kind as LogRecordMetricKind] = (target[kind as LogRecordMetricKind] ?? 0) + (count ?? 0);\n  }\n}\n"],"names":["Buffer","parseLogRecordLine","parseRawLogRecordLine","logger","DEFAULT_HARNESS","config","accrueStoredBytes","claimCap","ensureJobAccounting","isJobCapped","insertChunk","db","casExtendCommittedLength","getAttemptStream","getOrCreateAttemptStreamWithStatus","setDeclaredTotalBytes","recordAppendedCount","streamClosedCount","streamOpenedCount","allowedBudget","closeStream","controlTombstone","MalformedLogChunkError","OffsetGapError","parseSessionRecord","parseAppendBody","body","length","records","hasAgentSessionRecord","text","toString","endsWith","lines","split","pop","declaredTotalBytes","line","record","detectForgedType","push","type","total_bytes","byteLength","LOG_MAX_SESSION_LINE_BYTES","undefined","getSessionHarness","workflows","stepId","getStepLogContext","harness","readHeartbeat","tx","params","existing","jobId","attempt","committedLength","capped","storeChunk","streamId","workspaceId","storedByteLen","accrued","delta","stored","recordCounts","streamOffset","offset","byteLen","data","origin","allowed","baseBytes","LOG_BUDGET_BASE_BYTES","ratePerMinuteBytes","LOG_BUDGET_RATE_BYTES_PER_MINUTE","elapsedMs","Date","now","startedAt","getTime","used","won","tombstone","buildStoredBody","storedRecords","row","agentSessionRecord","v","ts","timestamp","from","map","JSON","stringify","join","appendLogs","parsed","error","forgedType","warn","offendingType","sessionHarness","commitByteLen","metrics","streamClosedReason","streamOpened","result","transaction","created","stream","projectId","workflowRunAttemptId","state","cas","id","outcome","chunkStored","addRecordCounts","closed","reason","add","kind","count","Object","entries","target","source"],"mappings":"AAAA,SAAQA,MAAM,QAAO,cAAc;AACnC,SAEEC,kBAAkB,EAClBC,qBAAqB,QAEhB,wBAAwB;AAE/B,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,eAAe,QAAqB,6BAA6B;AACzE,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAQC,iBAAiB,EAAEC,QAAQ,EAAEC,mBAAmB,EAAEC,WAAW,QAAO,oBAAoB;AAChG,SAAQC,WAAW,QAAO,gBAAgB;AAC1C,SAAQC,EAAE,QAAyB,YAAY;AAC/C,SACEC,wBAAwB,EACxBC,gBAAgB,EAChBC,kCAAkC,EAClCC,qBAAqB,QAChB,iBAAiB;AACxB,SAEEC,mBAAmB,EACnBC,iBAAiB,EACjBC,iBAAiB,QACZ,uBAAuB;AAC9B,SAAQC,aAAa,QAAO,cAAc;AAC1C,SAAQC,WAAW,EAAEC,gBAAgB,QAAO,oBAAoB;AAChE,SAAQC,sBAAsB,EAAEC,cAAc,QAAO,cAAc;AACnE,SAAQC,kBAAkB,QAAO,6BAA6B;AAyB9D;;;;;;;;;;;CAWC,GACD,SAASC,gBAAgBC,IAAY;IACnC,IAAIA,KAAKC,MAAM,KAAK,GAAG,OAAO;QAACC,SAAS,EAAE;QAAEC,uBAAuB;IAAK;IAExE,MAAMC,OAAOJ,KAAKK,QAAQ,CAAC;IAC3B,IAAI,CAACD,KAAKE,QAAQ,CAAC,OAAO;QACxB,MAAM,IAAIV,uBAAuB;IACnC;IACA,MAAMW,QAAQH,KAAKI,KAAK,CAAC;IACzBD,MAAME,GAAG;IAET,IAAIC;IACJ,MAAMR,UAA0B,EAAE;IAClC,IAAIC,wBAAwB;IAC5B,KAAK,MAAMQ,QAAQJ,MAAO;QACxB,IAAIK;QACJ,IAAI;YACFA,SAASpC,sBAAsBmC;QACjC,EAAE,OAAM;YACN,MAAM,IAAIf,uBACR,iDACAiB,iBAAiBF;QAErB;QACAT,QAAQY,IAAI,CAACF;QACb,IAAIA,OAAOG,IAAI,KAAK,OAAO;YACzBL,qBAAqBE,OAAOI,WAAW;QACzC;QACA,uFAAuF;QACvF,yFAAyF;QACzF,0FAA0F;QAC1F,IACEJ,OAAOG,IAAI,KAAK,mBAChBzC,OAAO2C,UAAU,CAACN,MAAM,UAAUhC,OAAOuC,0BAA0B,EACnE;YACA,MAAM,IAAItB,uBACR,CAAC,2BAA2B,EAAEjB,OAAOuC,0BAA0B,CAAC,MAAM,CAAC;QAE3E;QACA,IAAIN,OAAOG,IAAI,KAAK,iBAAiB;YACnCZ,wBAAwB;QAC1B;IACF;IAEA,OAAOO,uBAAuBS,YAC1B;QAACjB;QAASC;IAAqB,IAC/B;QAACO;QAAoBR;QAASC;IAAqB;AACzD;AAEA;;;;CAIC,GACD,SAASU,iBAAiBF,IAAY;IACpC,IAAI;QACF,OAAOpC,mBAAmBoC,MAAMI,IAAI;IACtC,EAAE,OAAM;QACN,OAAOI;IACT;AACF;AAEA,eAAeC,kBACbC,SAA4C,EAC5CC,MAAc;IAEd,OAAOD,YAAY,AAAC,CAAA,MAAMA,UAAUE,iBAAiB,CAAC;QAACD;IAAM,EAAC,EAAGE,OAAO,GAAG9C;AAC7E;AAEA;;;CAGC,GACD,eAAe+C,cAAcC,EAAe,EAAEC,MAAwB;IACpE,MAAMC,WAAW,MAAMzC,iBAAiBuC,IAAI;QAC1CG,OAAOF,OAAOE,KAAK;QACnBP,QAAQK,OAAOL,MAAM;QACrBQ,SAASH,OAAOG,OAAO;IACzB;IACA,OAAO;QACLC,iBAAiBH,UAAUG,mBAAmB;QAC9CC,QAAQ,MAAMjD,YAAY2C,IAAIC,OAAOE,KAAK;IAC5C;AACF;AAoBA;;;;CAIC,GACD,eAAeI,WACbP,EAAe,EACf,EAACC,MAAM,EAAEO,QAAQ,EAAElC,IAAI,EAAE+B,eAAe,EAAErB,kBAAkB,EAAmB;IAE/E,MAAM5B,oBAAoB4C,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAEM,aAAaR,OAAOQ,WAAW;IAAA;IACnF,MAAMC,gBAAgBpC,KAAKC,MAAM;IACjC,MAAMoC,UAAU,MAAMzD,kBAAkB8C,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAES,OAAOF;IAAa;IAEtF,+EAA+E;IAC/E,wEAAwE;IACxE,IAAI,CAACC,SAAS,OAAO;QAACN;QAAiBC,QAAQ;QAAMO,QAAQ;QAAOC,cAAc,CAAC;IAAC;IAEpF,MAAMxD,YAAY0C,IAAI;QACpBQ;QACAO,cAAcd,OAAOe,MAAM;QAC3BC,SAASP;QACTQ,MAAM5C;QACN6C,QAAQ;IACV;IACA,IAAInC,uBAAuBS,WAAW;QACpC,MAAM9B,sBAAsBqC,IAAI;YAACQ;YAAUxB;QAAkB;IAC/D;IAEA,MAAMoC,UAAUrD,cAAc;QAC5BsD,WAAWpE,OAAOqE,qBAAqB;QACvCC,oBAAoBtE,OAAOuE,gCAAgC;QAC3DC,WAAWC,KAAKC,GAAG,KAAKhB,QAAQiB,SAAS,CAACC,OAAO;IACnD;IACA,IAAIlB,QAAQmB,IAAI,IAAIV,SAAS;QAC3B,OAAO;YAACf;YAAiBC,QAAQ;YAAOO,QAAQ;YAAMC,cAAc,CAAC;QAAC;IACxE;IAEA,kFAAkF;IAClF,oFAAoF;IACpF,kBAAkB;IAClB,MAAMiB,MAAM,MAAM5E,SAAS6C,IAAIC,OAAOE,KAAK;IAC3C,IAAI4B,KAAK;QACP,MAAMC,YAAY/D,iBAAiB;QACnC,MAAMX,YAAY0C,IAAI;YACpBQ;YACAO,cAAcV;YACdY,SAASe,UAAUzD,MAAM;YACzB2C,MAAMc;YACNb,QAAQ;QACV;IACF;IACA,OAAO;QACLd;QACAC,QAAQ;QACRO,QAAQ;QACRC,cAAciB,MAAM;YAACzB,QAAQ;QAAC,IAAI,CAAC;IACrC;AACF;AAEA,SAAS2B,gBACPzD,OAAgC,EAChCsB,OAAgB;IAEhB,MAAMoC,gBAA6B,EAAE;IACrC,KAAK,MAAMhD,UAAUV,QAAS;QAC5B,IAAIU,OAAOG,IAAI,KAAK,iBAAiB;YACnC6C,cAAc9C,IAAI,CAACF;YACnB;QACF;QAEA,KAAK,MAAMiD,OAAO/D,mBAAmBgE,mBAAmBlD,SAASY,SAAU;YACzEoC,cAAc9C,IAAI,CAAC;gBAACiD,GAAG;gBAAGC,IAAIH,IAAII,SAAS;gBAAElD,MAAM;gBAAiB8C;YAAG;QACzE;IACF;IAEA,MAAM7D,OAAO1B,OAAO4F,IAAI,CAACN,cAAcO,GAAG,CAAC,CAACvD,SAAW,GAAGwD,KAAKC,SAAS,CAACzD,QAAQ,EAAE,CAAC,EAAE0D,IAAI,CAAC;IAC3F,MAAM9B,eAA2D,CAAC;IAClE,KAAK,MAAM5B,UAAUgD,cAAe;QAClCpB,YAAY,CAAC5B,OAAOG,IAAI,CAAC,GAAG,AAACyB,CAAAA,YAAY,CAAC5B,OAAOG,IAAI,CAAC,IAAI,CAAA,IAAK;IACjE;IAEA,OAAO;QAACf;QAAMwC;IAAY;AAC5B;AAEA,SAASsB,mBACPlD,MAAsD;IAEtD,OAAO;QAACgC,MAAMhC,OAAOgC,IAAI;QAAEoB,IAAIpD,OAAOoD,EAAE;IAAA;AAC1C;AAEA;;;;;CAKC,GACD,OAAO,eAAeO,WACpB5C,MAAwB,EACxBN,SAAiC;IAEjC,IAAImD;IACJ,IAAI;QACFA,SAASzE,gBAAgB4B,OAAO3B,IAAI;IACtC,EAAE,OAAOyE,OAAO;QACd,kFAAkF;QAClF,IAAIA,iBAAiB7E,0BAA0B6E,MAAMC,UAAU,KAAKvD,WAAW;YAC7E1C,SAASkG,IAAI,CACX;gBACE9C,OAAOF,OAAOE,KAAK;gBACnBP,QAAQK,OAAOL,MAAM;gBACrBsD,eAAeH,MAAMC,UAAU;YACjC,GACA;QAEJ;QACA,MAAMD;IACR;IACA,MAAM,EAAC/D,kBAAkB,EAAC,GAAG8D;IAC7B,MAAMK,iBAAiBL,OAAOrE,qBAAqB,GAC/C,MAAMiB,kBAAkBC,WAAWM,OAAOL,MAAM,IAChD5C;IACJ,MAAM6D,SAASoB,gBAAgBa,OAAOtE,OAAO,EAAE2E;IAC/C,MAAMC,gBAAgBnD,OAAO3B,IAAI,CAACC,MAAM;IACxC,MAAM8E,UAAU;QACdvC,cAAc,CAAC;QACfwC,oBAAoB7D;QACpB8D,cAAc;IAChB;IAEA,MAAMC,SAAS,MAAMjG,KAAKkG,WAAW,CAAC,OAAOzD;QAC3C,IAAIoD,kBAAkB,GAAG,OAAOrD,cAAcC,IAAIC;QAElD,MAAM,EAACyD,OAAO,EAAEC,MAAM,EAAC,GAAG,MAAMjG,mCAAmCsC,IAAI;YACrEG,OAAOF,OAAOE,KAAK;YACnBP,QAAQK,OAAOL,MAAM;YACrBQ,SAASH,OAAOG,OAAO;YACvBK,aAAaR,OAAOQ,WAAW;YAC/BmD,WAAW3D,OAAO2D,SAAS;YAC3BC,sBAAsB5D,OAAO4D,oBAAoB;QACnD;QACAR,QAAQE,YAAY,GAAGG;QAEvB,oFAAoF;QACpF,iFAAiF;QACjF,kFAAkF;QAClF,IAAIC,OAAOG,KAAK,KAAK,UAAU;YAC7B,OAAO;gBAACzD,iBAAiBsD,OAAOtD,eAAe;gBAAEC,QAAQ,MAAMjD,YAAY2C,IAAIC,OAAOE,KAAK;YAAC;QAC9F;QAEA,MAAM4D,MAAM,MAAMvG,yBAAyBwC,IAAI;YAC7CQ,UAAUmD,OAAOK,EAAE;YACnBhD,QAAQf,OAAOe,MAAM;YACrBC,SAASmC;QACX;QACA,IAAIW,IAAIE,OAAO,KAAK,OAAO,MAAM,IAAI9F,eAAe4F,IAAI1D,eAAe;QACvE,IAAI0D,IAAIE,OAAO,KAAK,SAAS;YAC3B,OAAO;gBAAC5D,iBAAiB0D,IAAI1D,eAAe;gBAAEC,QAAQ,MAAMjD,YAAY2C,IAAIC,OAAOE,KAAK;YAAC;QAC3F;QAEA,MAAM,EACJW,YAAY,EACZD,QAAQqD,WAAW,EACnB,GAAGV,QACJ,GAAG,MAAMjD,WAAWP,IAAI;YACvBC;YACAO,UAAUmD,OAAOK,EAAE;YACnB1F,MAAMuC,OAAOvC,IAAI;YACjB+B,iBAAiB0D,IAAI1D,eAAe;YACpCrB;QACF;QACA,IAAIkF,aAAa;YACfC,gBAAgBd,QAAQvC,YAAY,EAAED,OAAOC,YAAY;QAC3D;QACAqD,gBAAgBd,QAAQvC,YAAY,EAAEA;QAEtC,8EAA8E;QAC9E,qFAAqF;QACrF,oFAAoF;QACpF,mFAAmF;QACnF,qFAAqF;QACrF,0CAA0C;QAC1C,IAAI9B,uBAAuBS,aAAayE,aAAa;YACnD,MAAME,SAAS,MAAMpG,YAAYgC,IAAI;gBAACQ,UAAUmD,OAAOK,EAAE;gBAAEK,QAAQ;YAAU;YAC7E,IAAID,QAAQf,QAAQC,kBAAkB,GAAG;QAC3C;QAEA,OAAOE;IACT;IAEA,IAAIH,QAAQE,YAAY,EAAEzF,kBAAkBwG,GAAG,CAAC;IAChD,KAAK,MAAM,CAACC,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACrB,QAAQvC,YAAY,EAAG;QAChE,IAAI0D,QAAQ,GAAG5G,oBAAoB0G,GAAG,CAACE,OAAO;YAACD,MAAMA;QAA2B;IAClF;IACA,IAAIlB,QAAQC,kBAAkB,EAAE;QAC9BzF,kBAAkByG,GAAG,CAAC,GAAG;YAACD,QAAQhB,QAAQC,kBAAkB;QAAA;IAC9D;IAEA,OAAOE;AACT;AAEA,SAASW,gBACPQ,MAAoD,EACpDC,MAAoD;IAEpD,KAAK,MAAM,CAACL,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACE,QAAS;QAClDD,MAAM,CAACJ,KAA4B,GAAG,AAACI,CAAAA,MAAM,CAACJ,KAA4B,IAAI,CAAA,IAAMC,CAAAA,SAAS,CAAA;IAC/F;AACF"}

package/dist/core/reap-stale-open-streams.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reap-stale-open-streams.d.ts","sourceRoot":"","sources":["../../src/core/reap-stale-open-streams.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,0BAA0B;IACzC,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,0BAA0B;IACzC;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,0BAA0B,GACjC,OAAO,CAAC,0BAA0B,CAAC,CAwBrC"}
+{"version":3,"file":"reap-stale-open-streams.d.ts","sourceRoot":"","sources":["../../src/core/reap-stale-open-streams.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,0BAA0B;IACzC,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,0BAA0B;IACzC;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,0BAA0B,GACjC,OAAO,CAAC,0BAA0B,CAAC,CAyBrC"}

package/dist/core/reap-stale-open-streams.js

@@ -1,3 +1,4 @@
+import { reportError } from '@shipfox/node-error-monitoring';
 import { logger } from '@shipfox/node-opentelemetry';
 import { db } from '#db/db.js';
 import { listStaleOpenStreams } from '#db/streams.js';
@@ -42,6 +43,12 @@ import { closeStream } from './close-stream.js';
                 err: error,
                 streamId: stream.id
             }, 'Failed to reap stale open log stream');
+            reportError(error, {
+                boundary: 'logs.maintenance',
+                extra: {
+                    streamId: stream.id
+                }
+            });
         }
     }
     return result;

package/dist/core/reap-stale-open-streams.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/core/reap-stale-open-streams.ts"],"sourcesContent":["import {logger} from '@shipfox/node-opentelemetry';\nimport {db} from '#db/db.js';\nimport {listStaleOpenStreams} from '#db/streams.js';\nimport {recordAppendedCount, streamClosedCount} from '#metrics/instance.js';\nimport {closeStream} from './close-stream.js';\n\nexport interface ReapStaleOpenStreamsResult {\n  /** Streams force-closed by this run. */\n  reaped: number;\n  /** Streams whose guarded close threw; logged, skipped, retried next run. */\n  failed: number;\n}\n\nexport interface ReapStaleOpenStreamsParams {\n  /**\n   * A stream open longer than this since `created_at` can no longer be appended to: its job\n   * holds one lease minted at-or-before the stream's creation, so once the lease TTL has\n   * elapsed no append can land and the close is safe. Set above `AUTH_JOB_LEASE_TOKEN_EXPIRES_IN`.\n   */\n  olderThanSeconds: number;\n  batchLimit: number;\n}\n\n/**\n * Force-closes open streams the one-shot job-terminal sweep missed: a stream whose first append\n * landed after that sweep's snapshot would otherwise stay `open` forever, invisible to\n * compaction and retention (both partial on `state = 'closed'`). Reusing the guarded\n * `closeStream` re-drives each into the normal closed -> compact -> retention lifecycle.\n *\n * Each row closes in its own transaction, so a tick/retry overlap is idempotent (the guarded\n * `markStreamClosed` lets only one close win) and one failed row is logged and skipped instead\n * of aborting the batch.\n */\nexport async function reapStaleOpenStreams(\n  params: ReapStaleOpenStreamsParams,\n): Promise<ReapStaleOpenStreamsResult> {\n  const stale = await listStaleOpenStreams({\n    olderThanSeconds: params.olderThanSeconds,\n    limit: params.batchLimit,\n  });\n\n  const result: ReapStaleOpenStreamsResult = {reaped: 0, failed: 0};\n  for (const stream of stale) {\n    try {\n      const closed = await db().transaction((tx) =>\n        closeStream(tx, {streamId: stream.id, reason: 'timeout'}),\n      );\n      if (closed) {\n        result.reaped += 1;\n        recordAppendedCount.add(1, {kind: 'runner_lost'});\n        streamClosedCount.add(1, {reason: 'timeout'});\n      }\n    } catch (error) {\n      result.failed += 1;\n      logger().error({err: error, streamId: stream.id}, 'Failed to reap stale open log stream');\n    }\n  }\n\n  return result;\n}\n"],"names":["logger","db","listStaleOpenStreams","recordAppendedCount","streamClosedCount","closeStream","reapStaleOpenStreams","params","stale","olderThanSeconds","limit","batchLimit","result","reaped","failed","stream","closed","transaction","tx","streamId","id","reason","add","kind","error","err"],"mappings":"AAAA,SAAQA,MAAM,QAAO,8BAA8B;AACnD,SAAQC,EAAE,QAAO,YAAY;AAC7B,SAAQC,oBAAoB,QAAO,iBAAiB;AACpD,SAAQC,mBAAmB,EAAEC,iBAAiB,QAAO,uBAAuB;AAC5E,SAAQC,WAAW,QAAO,oBAAoB;AAmB9C;;;;;;;;;CASC,GACD,OAAO,eAAeC,qBACpBC,MAAkC;IAElC,MAAMC,QAAQ,MAAMN,qBAAqB;QACvCO,kBAAkBF,OAAOE,gBAAgB;QACzCC,OAAOH,OAAOI,UAAU;IAC1B;IAEA,MAAMC,SAAqC;QAACC,QAAQ;QAAGC,QAAQ;IAAC;IAChE,KAAK,MAAMC,UAAUP,MAAO;QAC1B,IAAI;YACF,MAAMQ,SAAS,MAAMf,KAAKgB,WAAW,CAAC,CAACC,KACrCb,YAAYa,IAAI;oBAACC,UAAUJ,OAAOK,EAAE;oBAAEC,QAAQ;gBAAS;YAEzD,IAAIL,QAAQ;gBACVJ,OAAOC,MAAM,IAAI;gBACjBV,oBAAoBmB,GAAG,CAAC,GAAG;oBAACC,MAAM;gBAAa;gBAC/CnB,kBAAkBkB,GAAG,CAAC,GAAG;oBAACD,QAAQ;gBAAS;YAC7C;QACF,EAAE,OAAOG,OAAO;YACdZ,OAAOE,MAAM,IAAI;YACjBd,SAASwB,KAAK,CAAC;gBAACC,KAAKD;gBAAOL,UAAUJ,OAAOK,EAAE;YAAA,GAAG;QACpD;IACF;IAEA,OAAOR;AACT"}
+{"version":3,"sources":["../../src/core/reap-stale-open-streams.ts"],"sourcesContent":["import {reportError} from '@shipfox/node-error-monitoring';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {db} from '#db/db.js';\nimport {listStaleOpenStreams} from '#db/streams.js';\nimport {recordAppendedCount, streamClosedCount} from '#metrics/instance.js';\nimport {closeStream} from './close-stream.js';\n\nexport interface ReapStaleOpenStreamsResult {\n  /** Streams force-closed by this run. */\n  reaped: number;\n  /** Streams whose guarded close threw; logged, skipped, retried next run. */\n  failed: number;\n}\n\nexport interface ReapStaleOpenStreamsParams {\n  /**\n   * A stream open longer than this since `created_at` can no longer be appended to: its job\n   * holds one lease minted at-or-before the stream's creation, so once the lease TTL has\n   * elapsed no append can land and the close is safe. Set above `AUTH_JOB_LEASE_TOKEN_EXPIRES_IN`.\n   */\n  olderThanSeconds: number;\n  batchLimit: number;\n}\n\n/**\n * Force-closes open streams the one-shot job-terminal sweep missed: a stream whose first append\n * landed after that sweep's snapshot would otherwise stay `open` forever, invisible to\n * compaction and retention (both partial on `state = 'closed'`). Reusing the guarded\n * `closeStream` re-drives each into the normal closed -> compact -> retention lifecycle.\n *\n * Each row closes in its own transaction, so a tick/retry overlap is idempotent (the guarded\n * `markStreamClosed` lets only one close win) and one failed row is logged and skipped instead\n * of aborting the batch.\n */\nexport async function reapStaleOpenStreams(\n  params: ReapStaleOpenStreamsParams,\n): Promise<ReapStaleOpenStreamsResult> {\n  const stale = await listStaleOpenStreams({\n    olderThanSeconds: params.olderThanSeconds,\n    limit: params.batchLimit,\n  });\n\n  const result: ReapStaleOpenStreamsResult = {reaped: 0, failed: 0};\n  for (const stream of stale) {\n    try {\n      const closed = await db().transaction((tx) =>\n        closeStream(tx, {streamId: stream.id, reason: 'timeout'}),\n      );\n      if (closed) {\n        result.reaped += 1;\n        recordAppendedCount.add(1, {kind: 'runner_lost'});\n        streamClosedCount.add(1, {reason: 'timeout'});\n      }\n    } catch (error) {\n      result.failed += 1;\n      logger().error({err: error, streamId: stream.id}, 'Failed to reap stale open log stream');\n      reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});\n    }\n  }\n\n  return result;\n}\n"],"names":["reportError","logger","db","listStaleOpenStreams","recordAppendedCount","streamClosedCount","closeStream","reapStaleOpenStreams","params","stale","olderThanSeconds","limit","batchLimit","result","reaped","failed","stream","closed","transaction","tx","streamId","id","reason","add","kind","error","err","boundary","extra"],"mappings":"AAAA,SAAQA,WAAW,QAAO,iCAAiC;AAC3D,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,EAAE,QAAO,YAAY;AAC7B,SAAQC,oBAAoB,QAAO,iBAAiB;AACpD,SAAQC,mBAAmB,EAAEC,iBAAiB,QAAO,uBAAuB;AAC5E,SAAQC,WAAW,QAAO,oBAAoB;AAmB9C;;;;;;;;;CASC,GACD,OAAO,eAAeC,qBACpBC,MAAkC;IAElC,MAAMC,QAAQ,MAAMN,qBAAqB;QACvCO,kBAAkBF,OAAOE,gBAAgB;QACzCC,OAAOH,OAAOI,UAAU;IAC1B;IAEA,MAAMC,SAAqC;QAACC,QAAQ;QAAGC,QAAQ;IAAC;IAChE,KAAK,MAAMC,UAAUP,MAAO;QAC1B,IAAI;YACF,MAAMQ,SAAS,MAAMf,KAAKgB,WAAW,CAAC,CAACC,KACrCb,YAAYa,IAAI;oBAACC,UAAUJ,OAAOK,EAAE;oBAAEC,QAAQ;gBAAS;YAEzD,IAAIL,QAAQ;gBACVJ,OAAOC,MAAM,IAAI;gBACjBV,oBAAoBmB,GAAG,CAAC,GAAG;oBAACC,MAAM;gBAAa;gBAC/CnB,kBAAkBkB,GAAG,CAAC,GAAG;oBAACD,QAAQ;gBAAS;YAC7C;QACF,EAAE,OAAOG,OAAO;YACdZ,OAAOE,MAAM,IAAI;YACjBd,SAASwB,KAAK,CAAC;gBAACC,KAAKD;gBAAOL,UAAUJ,OAAOK,EAAE;YAAA,GAAG;YAClDrB,YAAYyB,OAAO;gBAACE,UAAU;gBAAoBC,OAAO;oBAACR,UAAUJ,OAAOK,EAAE;gBAAA;YAAC;QAChF;IACF;IAEA,OAAOR;AACT"}

package/dist/core/retention.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"retention.d.ts","sourceRoot":"","sources":["../../src/core/retention.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,oBAAoB;IACnC,kDAAkD;IAClD,OAAO,EAAE,MAAM,CAAC;IAChB,mGAAmG;IACnG,KAAK,EAAE,MAAM,CAAC;IACd,uGAAuG;IACvG,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,0FAA0F;IAC1F,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,mFAAmF;IACnF,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,wFAAwF;IACxF,UAAU,CAAC,EAAE,MAAM,IAAI,CAAC;CACzB;AA0BD;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,oBAAoB,CAAC,CA+G/B"}
+{"version":3,"file":"retention.d.ts","sourceRoot":"","sources":["../../src/core/retention.ts"],"names":[],"mappings":"AAeA,MAAM,WAAW,oBAAoB;IACnC,kDAAkD;IAClD,OAAO,EAAE,MAAM,CAAC;IAChB,mGAAmG;IACnG,KAAK,EAAE,MAAM,CAAC;IACd,uGAAuG;IACvG,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,0FAA0F;IAC1F,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,mFAAmF;IACnF,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,wFAAwF;IACxF,UAAU,CAAC,EAAE,MAAM,IAAI,CAAC;CACzB;AA0BD;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,oBAAoB,CAAC,CAmH/B"}

package/dist/core/retention.js

@@ -1,3 +1,4 @@
+import { reportError } from '@shipfox/node-error-monitoring';
 import { logger } from '@shipfox/node-opentelemetry';
 import { deleteObjectsByPrefix } from '#api/object-storage.js';
 import { config } from '#config.js';
@@ -81,6 +82,12 @@ function deleteExpiredStreamRow(params) {
                     err: error,
                     streamId: stream.id
                 }, 'Failed to delete expired log stream objects');
+                reportError(error, {
+                    boundary: 'logs.maintenance',
+                    extra: {
+                        streamId: stream.id
+                    }
+                });
                 continue;
             }
             let outcome;
@@ -96,6 +103,12 @@ function deleteExpiredStreamRow(params) {
                     err: error,
                     streamId: stream.id
                 }, 'Failed to delete expired log stream row');
+                reportError(error, {
+                    boundary: 'logs.maintenance',
+                    extra: {
+                        streamId: stream.id
+                    }
+                });
                 continue;
             }
             if (!outcome.deleted) {
@@ -109,6 +122,12 @@ function deleteExpiredStreamRow(params) {
                         err: error,
                         streamId: stream.id
                     }, 'Failed to reload raced expired log stream');
+                    reportError(error, {
+                        boundary: 'logs.maintenance',
+                        extra: {
+                            streamId: stream.id
+                        }
+                    });
                     continue;
                 }
                 if (current) {
@@ -125,6 +144,12 @@ function deleteExpiredStreamRow(params) {
                             err: error,
                             streamId: stream.id
                         }, 'Failed to delete raced expired log stream');
+                        reportError(error, {
+                            boundary: 'logs.maintenance',
+                            extra: {
+                                streamId: stream.id
+                            }
+                        });
                         continue;
                     }
                 }

package/dist/core/retention.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/core/retention.ts"],"sourcesContent":["import {logger} from '@shipfox/node-opentelemetry';\nimport {deleteObjectsByPrefix} from '#api/object-storage.js';\nimport {config} from '#config.js';\nimport type {AttemptStream} from '#core/entities/attempt-stream.js';\nimport {logObjectKey} from '#core/entities/log-object.js';\nimport {deleteJobAccounting} from '#db/accounting.js';\nimport {db} from '#db/db.js';\nimport {\n  accountingHasNoStreams,\n  deleteExpiredStream,\n  getAttemptStreamById,\n  listExpiredClosedStreams,\n} from '#db/streams.js';\n\nexport interface RetentionSweepResult {\n  /** Streams whose objects and row were deleted. */\n  deleted: number;\n  /** Streams skipped because compaction changed `object_key` after we read it (retried next run). */\n  raced: number;\n  /** Streams whose object cleanup or guarded row delete threw; logged, skipped, and retried next run. */\n  failed: number;\n  accountingPruned: number;\n  iterations: number;\n  /** True when the sweep stopped on its wall-clock budget with backlog likely remaining. */\n  timedOut: boolean;\n}\n\nexport interface RunRetentionSweepParams {\n  retentionDays: number;\n  batchLimit: number;\n  timeBudgetMs: number;\n  maxIterations: number;\n  /** Wall clock; injectable so tests can drive the time budget deterministically. */\n  now?: () => number;\n  /** Liveness signal (e.g. the activity heartbeat); invoked once per processed stream. */\n  onProgress?: () => void;\n}\n\nfunction deleteExpiredStreamObjects(stream: AttemptStream): Promise<void> {\n  return deleteObjectsByPrefix(`${logObjectKey(config.LOG_STORAGE_S3_PREFIX, stream)}/`);\n}\n\nfunction deleteExpiredStreamRow(params: {\n  stream: AttemptStream;\n  retentionDays: number;\n}): Promise<{deleted: boolean; prunedAccounting: boolean}> {\n  return db().transaction(async (tx) => {\n    const {deleted, jobId} = await deleteExpiredStream(tx, {\n      streamId: params.stream.id,\n      observedObjectKey: params.stream.objectKey,\n    });\n    if (deleted && jobId && (await accountingHasNoStreams(tx, jobId))) {\n      const pruned = await deleteJobAccounting(tx, {\n        jobId,\n        retentionDays: params.retentionDays,\n      });\n      return {deleted, prunedAccounting: pruned.deleted};\n    }\n    return {deleted, prunedAccounting: false};\n  });\n}\n\n/**\n * Deletes expired closed streams and prunes accounting for emptied jobs.\n *\n * The loop self-bounds because a Temporal `startToCloseTimeout` marks the activity failed but\n * does not stop already-running JS. Objects are deleted before rows so a cleanup failure leaves\n * the row discoverable for the next sweep; the row delete stays guarded on the observed\n * `object_key`, so a racing compaction publish is re-read before the row is removed.\n */\nexport async function runRetentionSweep(\n  params: RunRetentionSweepParams,\n): Promise<RetentionSweepResult> {\n  const now = params.now ?? Date.now;\n  const deadline = now() + params.timeBudgetMs;\n  const result: RetentionSweepResult = {\n    deleted: 0,\n    raced: 0,\n    failed: 0,\n    accountingPruned: 0,\n    iterations: 0,\n    timedOut: false,\n  };\n\n  // Failed or raced rows retry next run; skipping them here keeps the rest of the backlog moving.\n  const skip = new Set<string>();\n  while (result.iterations < params.maxIterations) {\n    if (now() >= deadline) {\n      result.timedOut = true;\n      break;\n    }\n\n    const batch = await listExpiredClosedStreams({\n      retentionDays: params.retentionDays,\n      limit: params.batchLimit,\n      excludeIds: skip.size > 0 ? [...skip] : undefined,\n    });\n    if (batch.length === 0) break;\n\n    let timedOutMidBatch = false;\n    for (const stream of batch) {\n      // Per-stream checks keep long batches within the heartbeat and wall-clock budgets.\n      params.onProgress?.();\n      if (now() >= deadline) {\n        result.timedOut = true;\n        timedOutMidBatch = true;\n        break;\n      }\n\n      try {\n        await deleteExpiredStreamObjects(stream);\n      } catch (error) {\n        result.failed += 1;\n        skip.add(stream.id);\n        logger().error(\n          {err: error, streamId: stream.id},\n          'Failed to delete expired log stream objects',\n        );\n        continue;\n      }\n\n      let outcome: {deleted: boolean; prunedAccounting: boolean};\n      try {\n        outcome = await deleteExpiredStreamRow({stream, retentionDays: params.retentionDays});\n      } catch (error) {\n        result.failed += 1;\n        skip.add(stream.id);\n        logger().error(\n          {err: error, streamId: stream.id},\n          'Failed to delete expired log stream row',\n        );\n        continue;\n      }\n\n      if (!outcome.deleted) {\n        let current: AttemptStream | null;\n        try {\n          current = await getAttemptStreamById(stream.id);\n        } catch (error) {\n          result.failed += 1;\n          skip.add(stream.id);\n          logger().error(\n            {err: error, streamId: stream.id},\n            'Failed to reload raced expired log stream',\n          );\n          continue;\n        }\n        if (current) {\n          try {\n            await deleteExpiredStreamObjects(current);\n            outcome = await deleteExpiredStreamRow({\n              stream: current,\n              retentionDays: params.retentionDays,\n            });\n          } catch (error) {\n            result.failed += 1;\n            skip.add(stream.id);\n            logger().error(\n              {err: error, streamId: stream.id},\n              'Failed to delete raced expired log stream',\n            );\n            continue;\n          }\n        }\n        if (!current || !outcome.deleted) {\n          // `object_key` changed again or the row disappeared; the next sweep will re-read it.\n          result.raced += 1;\n          skip.add(stream.id);\n          continue;\n        }\n      }\n\n      result.deleted += 1;\n      if (outcome.prunedAccounting) result.accountingPruned += 1;\n    }\n\n    if (timedOutMidBatch) break;\n\n    result.iterations += 1;\n    if (batch.length < params.batchLimit) break;\n  }\n\n  return result;\n}\n"],"names":["logger","deleteObjectsByPrefix","config","logObjectKey","deleteJobAccounting","db","accountingHasNoStreams","deleteExpiredStream","getAttemptStreamById","listExpiredClosedStreams","deleteExpiredStreamObjects","stream","LOG_STORAGE_S3_PREFIX","deleteExpiredStreamRow","params","transaction","tx","deleted","jobId","streamId","id","observedObjectKey","objectKey","pruned","retentionDays","prunedAccounting","runRetentionSweep","now","Date","deadline","timeBudgetMs","result","raced","failed","accountingPruned","iterations","timedOut","skip","Set","maxIterations","batch","limit","batchLimit","excludeIds","size","undefined","length","timedOutMidBatch","onProgress","error","add","err","outcome","current"],"mappings":"AAAA,SAAQA,MAAM,QAAO,8BAA8B;AACnD,SAAQC,qBAAqB,QAAO,yBAAyB;AAC7D,SAAQC,MAAM,QAAO,aAAa;AAElC,SAAQC,YAAY,QAAO,+BAA+B;AAC1D,SAAQC,mBAAmB,QAAO,oBAAoB;AACtD,SAAQC,EAAE,QAAO,YAAY;AAC7B,SACEC,sBAAsB,EACtBC,mBAAmB,EACnBC,oBAAoB,EACpBC,wBAAwB,QACnB,iBAAiB;AA0BxB,SAASC,2BAA2BC,MAAqB;IACvD,OAAOV,sBAAsB,GAAGE,aAAaD,OAAOU,qBAAqB,EAAED,QAAQ,CAAC,CAAC;AACvF;AAEA,SAASE,uBAAuBC,MAG/B;IACC,OAAOT,KAAKU,WAAW,CAAC,OAAOC;QAC7B,MAAM,EAACC,OAAO,EAAEC,KAAK,EAAC,GAAG,MAAMX,oBAAoBS,IAAI;YACrDG,UAAUL,OAAOH,MAAM,CAACS,EAAE;YAC1BC,mBAAmBP,OAAOH,MAAM,CAACW,SAAS;QAC5C;QACA,IAAIL,WAAWC,SAAU,MAAMZ,uBAAuBU,IAAIE,QAAS;YACjE,MAAMK,SAAS,MAAMnB,oBAAoBY,IAAI;gBAC3CE;gBACAM,eAAeV,OAAOU,aAAa;YACrC;YACA,OAAO;gBAACP;gBAASQ,kBAAkBF,OAAON,OAAO;YAAA;QACnD;QACA,OAAO;YAACA;YAASQ,kBAAkB;QAAK;IAC1C;AACF;AAEA;;;;;;;CAOC,GACD,OAAO,eAAeC,kBACpBZ,MAA+B;IAE/B,MAAMa,MAAMb,OAAOa,GAAG,IAAIC,KAAKD,GAAG;IAClC,MAAME,WAAWF,QAAQb,OAAOgB,YAAY;IAC5C,MAAMC,SAA+B;QACnCd,SAAS;QACTe,OAAO;QACPC,QAAQ;QACRC,kBAAkB;QAClBC,YAAY;QACZC,UAAU;IACZ;IAEA,gGAAgG;IAChG,MAAMC,OAAO,IAAIC;IACjB,MAAOP,OAAOI,UAAU,GAAGrB,OAAOyB,aAAa,CAAE;QAC/C,IAAIZ,SAASE,UAAU;YACrBE,OAAOK,QAAQ,GAAG;YAClB;QACF;QAEA,MAAMI,QAAQ,MAAM/B,yBAAyB;YAC3Ce,eAAeV,OAAOU,aAAa;YACnCiB,OAAO3B,OAAO4B,UAAU;YACxBC,YAAYN,KAAKO,IAAI,GAAG,IAAI;mBAAIP;aAAK,GAAGQ;QAC1C;QACA,IAAIL,MAAMM,MAAM,KAAK,GAAG;QAExB,IAAIC,mBAAmB;QACvB,KAAK,MAAMpC,UAAU6B,MAAO;YAC1B,mFAAmF;YACnF1B,OAAOkC,UAAU;YACjB,IAAIrB,SAASE,UAAU;gBACrBE,OAAOK,QAAQ,GAAG;gBAClBW,mBAAmB;gBACnB;YACF;YAEA,IAAI;gBACF,MAAMrC,2BAA2BC;YACnC,EAAE,OAAOsC,OAAO;gBACdlB,OAAOE,MAAM,IAAI;gBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;gBAClBpB,SAASiD,KAAK,CACZ;oBAACE,KAAKF;oBAAO9B,UAAUR,OAAOS,EAAE;gBAAA,GAChC;gBAEF;YACF;YAEA,IAAIgC;YACJ,IAAI;gBACFA,UAAU,MAAMvC,uBAAuB;oBAACF;oBAAQa,eAAeV,OAAOU,aAAa;gBAAA;YACrF,EAAE,OAAOyB,OAAO;gBACdlB,OAAOE,MAAM,IAAI;gBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;gBAClBpB,SAASiD,KAAK,CACZ;oBAACE,KAAKF;oBAAO9B,UAAUR,OAAOS,EAAE;gBAAA,GAChC;gBAEF;YACF;YAEA,IAAI,CAACgC,QAAQnC,OAAO,EAAE;gBACpB,IAAIoC;gBACJ,IAAI;oBACFA,UAAU,MAAM7C,qBAAqBG,OAAOS,EAAE;gBAChD,EAAE,OAAO6B,OAAO;oBACdlB,OAAOE,MAAM,IAAI;oBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;oBAClBpB,SAASiD,KAAK,CACZ;wBAACE,KAAKF;wBAAO9B,UAAUR,OAAOS,EAAE;oBAAA,GAChC;oBAEF;gBACF;gBACA,IAAIiC,SAAS;oBACX,IAAI;wBACF,MAAM3C,2BAA2B2C;wBACjCD,UAAU,MAAMvC,uBAAuB;4BACrCF,QAAQ0C;4BACR7B,eAAeV,OAAOU,aAAa;wBACrC;oBACF,EAAE,OAAOyB,OAAO;wBACdlB,OAAOE,MAAM,IAAI;wBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;wBAClBpB,SAASiD,KAAK,CACZ;4BAACE,KAAKF;4BAAO9B,UAAUR,OAAOS,EAAE;wBAAA,GAChC;wBAEF;oBACF;gBACF;gBACA,IAAI,CAACiC,WAAW,CAACD,QAAQnC,OAAO,EAAE;oBAChC,qFAAqF;oBACrFc,OAAOC,KAAK,IAAI;oBAChBK,KAAKa,GAAG,CAACvC,OAAOS,EAAE;oBAClB;gBACF;YACF;YAEAW,OAAOd,OAAO,IAAI;YAClB,IAAImC,QAAQ3B,gBAAgB,EAAEM,OAAOG,gBAAgB,IAAI;QAC3D;QAEA,IAAIa,kBAAkB;QAEtBhB,OAAOI,UAAU,IAAI;QACrB,IAAIK,MAAMM,MAAM,GAAGhC,OAAO4B,UAAU,EAAE;IACxC;IAEA,OAAOX;AACT"}
+{"version":3,"sources":["../../src/core/retention.ts"],"sourcesContent":["import {reportError} from '@shipfox/node-error-monitoring';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {deleteObjectsByPrefix} from '#api/object-storage.js';\nimport {config} from '#config.js';\nimport type {AttemptStream} from '#core/entities/attempt-stream.js';\nimport {logObjectKey} from '#core/entities/log-object.js';\nimport {deleteJobAccounting} from '#db/accounting.js';\nimport {db} from '#db/db.js';\nimport {\n  accountingHasNoStreams,\n  deleteExpiredStream,\n  getAttemptStreamById,\n  listExpiredClosedStreams,\n} from '#db/streams.js';\n\nexport interface RetentionSweepResult {\n  /** Streams whose objects and row were deleted. */\n  deleted: number;\n  /** Streams skipped because compaction changed `object_key` after we read it (retried next run). */\n  raced: number;\n  /** Streams whose object cleanup or guarded row delete threw; logged, skipped, and retried next run. */\n  failed: number;\n  accountingPruned: number;\n  iterations: number;\n  /** True when the sweep stopped on its wall-clock budget with backlog likely remaining. */\n  timedOut: boolean;\n}\n\nexport interface RunRetentionSweepParams {\n  retentionDays: number;\n  batchLimit: number;\n  timeBudgetMs: number;\n  maxIterations: number;\n  /** Wall clock; injectable so tests can drive the time budget deterministically. */\n  now?: () => number;\n  /** Liveness signal (e.g. the activity heartbeat); invoked once per processed stream. */\n  onProgress?: () => void;\n}\n\nfunction deleteExpiredStreamObjects(stream: AttemptStream): Promise<void> {\n  return deleteObjectsByPrefix(`${logObjectKey(config.LOG_STORAGE_S3_PREFIX, stream)}/`);\n}\n\nfunction deleteExpiredStreamRow(params: {\n  stream: AttemptStream;\n  retentionDays: number;\n}): Promise<{deleted: boolean; prunedAccounting: boolean}> {\n  return db().transaction(async (tx) => {\n    const {deleted, jobId} = await deleteExpiredStream(tx, {\n      streamId: params.stream.id,\n      observedObjectKey: params.stream.objectKey,\n    });\n    if (deleted && jobId && (await accountingHasNoStreams(tx, jobId))) {\n      const pruned = await deleteJobAccounting(tx, {\n        jobId,\n        retentionDays: params.retentionDays,\n      });\n      return {deleted, prunedAccounting: pruned.deleted};\n    }\n    return {deleted, prunedAccounting: false};\n  });\n}\n\n/**\n * Deletes expired closed streams and prunes accounting for emptied jobs.\n *\n * The loop self-bounds because a Temporal `startToCloseTimeout` marks the activity failed but\n * does not stop already-running JS. Objects are deleted before rows so a cleanup failure leaves\n * the row discoverable for the next sweep; the row delete stays guarded on the observed\n * `object_key`, so a racing compaction publish is re-read before the row is removed.\n */\nexport async function runRetentionSweep(\n  params: RunRetentionSweepParams,\n): Promise<RetentionSweepResult> {\n  const now = params.now ?? Date.now;\n  const deadline = now() + params.timeBudgetMs;\n  const result: RetentionSweepResult = {\n    deleted: 0,\n    raced: 0,\n    failed: 0,\n    accountingPruned: 0,\n    iterations: 0,\n    timedOut: false,\n  };\n\n  // Failed or raced rows retry next run; skipping them here keeps the rest of the backlog moving.\n  const skip = new Set<string>();\n  while (result.iterations < params.maxIterations) {\n    if (now() >= deadline) {\n      result.timedOut = true;\n      break;\n    }\n\n    const batch = await listExpiredClosedStreams({\n      retentionDays: params.retentionDays,\n      limit: params.batchLimit,\n      excludeIds: skip.size > 0 ? [...skip] : undefined,\n    });\n    if (batch.length === 0) break;\n\n    let timedOutMidBatch = false;\n    for (const stream of batch) {\n      // Per-stream checks keep long batches within the heartbeat and wall-clock budgets.\n      params.onProgress?.();\n      if (now() >= deadline) {\n        result.timedOut = true;\n        timedOutMidBatch = true;\n        break;\n      }\n\n      try {\n        await deleteExpiredStreamObjects(stream);\n      } catch (error) {\n        result.failed += 1;\n        skip.add(stream.id);\n        logger().error(\n          {err: error, streamId: stream.id},\n          'Failed to delete expired log stream objects',\n        );\n        reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});\n        continue;\n      }\n\n      let outcome: {deleted: boolean; prunedAccounting: boolean};\n      try {\n        outcome = await deleteExpiredStreamRow({stream, retentionDays: params.retentionDays});\n      } catch (error) {\n        result.failed += 1;\n        skip.add(stream.id);\n        logger().error(\n          {err: error, streamId: stream.id},\n          'Failed to delete expired log stream row',\n        );\n        reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});\n        continue;\n      }\n\n      if (!outcome.deleted) {\n        let current: AttemptStream | null;\n        try {\n          current = await getAttemptStreamById(stream.id);\n        } catch (error) {\n          result.failed += 1;\n          skip.add(stream.id);\n          logger().error(\n            {err: error, streamId: stream.id},\n            'Failed to reload raced expired log stream',\n          );\n          reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});\n          continue;\n        }\n        if (current) {\n          try {\n            await deleteExpiredStreamObjects(current);\n            outcome = await deleteExpiredStreamRow({\n              stream: current,\n              retentionDays: params.retentionDays,\n            });\n          } catch (error) {\n            result.failed += 1;\n            skip.add(stream.id);\n            logger().error(\n              {err: error, streamId: stream.id},\n              'Failed to delete raced expired log stream',\n            );\n            reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});\n            continue;\n          }\n        }\n        if (!current || !outcome.deleted) {\n          // `object_key` changed again or the row disappeared; the next sweep will re-read it.\n          result.raced += 1;\n          skip.add(stream.id);\n          continue;\n        }\n      }\n\n      result.deleted += 1;\n      if (outcome.prunedAccounting) result.accountingPruned += 1;\n    }\n\n    if (timedOutMidBatch) break;\n\n    result.iterations += 1;\n    if (batch.length < params.batchLimit) break;\n  }\n\n  return result;\n}\n"],"names":["reportError","logger","deleteObjectsByPrefix","config","logObjectKey","deleteJobAccounting","db","accountingHasNoStreams","deleteExpiredStream","getAttemptStreamById","listExpiredClosedStreams","deleteExpiredStreamObjects","stream","LOG_STORAGE_S3_PREFIX","deleteExpiredStreamRow","params","transaction","tx","deleted","jobId","streamId","id","observedObjectKey","objectKey","pruned","retentionDays","prunedAccounting","runRetentionSweep","now","Date","deadline","timeBudgetMs","result","raced","failed","accountingPruned","iterations","timedOut","skip","Set","maxIterations","batch","limit","batchLimit","excludeIds","size","undefined","length","timedOutMidBatch","onProgress","error","add","err","boundary","extra","outcome","current"],"mappings":"AAAA,SAAQA,WAAW,QAAO,iCAAiC;AAC3D,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,qBAAqB,QAAO,yBAAyB;AAC7D,SAAQC,MAAM,QAAO,aAAa;AAElC,SAAQC,YAAY,QAAO,+BAA+B;AAC1D,SAAQC,mBAAmB,QAAO,oBAAoB;AACtD,SAAQC,EAAE,QAAO,YAAY;AAC7B,SACEC,sBAAsB,EACtBC,mBAAmB,EACnBC,oBAAoB,EACpBC,wBAAwB,QACnB,iBAAiB;AA0BxB,SAASC,2BAA2BC,MAAqB;IACvD,OAAOV,sBAAsB,GAAGE,aAAaD,OAAOU,qBAAqB,EAAED,QAAQ,CAAC,CAAC;AACvF;AAEA,SAASE,uBAAuBC,MAG/B;IACC,OAAOT,KAAKU,WAAW,CAAC,OAAOC;QAC7B,MAAM,EAACC,OAAO,EAAEC,KAAK,EAAC,GAAG,MAAMX,oBAAoBS,IAAI;YACrDG,UAAUL,OAAOH,MAAM,CAACS,EAAE;YAC1BC,mBAAmBP,OAAOH,MAAM,CAACW,SAAS;QAC5C;QACA,IAAIL,WAAWC,SAAU,MAAMZ,uBAAuBU,IAAIE,QAAS;YACjE,MAAMK,SAAS,MAAMnB,oBAAoBY,IAAI;gBAC3CE;gBACAM,eAAeV,OAAOU,aAAa;YACrC;YACA,OAAO;gBAACP;gBAASQ,kBAAkBF,OAAON,OAAO;YAAA;QACnD;QACA,OAAO;YAACA;YAASQ,kBAAkB;QAAK;IAC1C;AACF;AAEA;;;;;;;CAOC,GACD,OAAO,eAAeC,kBACpBZ,MAA+B;IAE/B,MAAMa,MAAMb,OAAOa,GAAG,IAAIC,KAAKD,GAAG;IAClC,MAAME,WAAWF,QAAQb,OAAOgB,YAAY;IAC5C,MAAMC,SAA+B;QACnCd,SAAS;QACTe,OAAO;QACPC,QAAQ;QACRC,kBAAkB;QAClBC,YAAY;QACZC,UAAU;IACZ;IAEA,gGAAgG;IAChG,MAAMC,OAAO,IAAIC;IACjB,MAAOP,OAAOI,UAAU,GAAGrB,OAAOyB,aAAa,CAAE;QAC/C,IAAIZ,SAASE,UAAU;YACrBE,OAAOK,QAAQ,GAAG;YAClB;QACF;QAEA,MAAMI,QAAQ,MAAM/B,yBAAyB;YAC3Ce,eAAeV,OAAOU,aAAa;YACnCiB,OAAO3B,OAAO4B,UAAU;YACxBC,YAAYN,KAAKO,IAAI,GAAG,IAAI;mBAAIP;aAAK,GAAGQ;QAC1C;QACA,IAAIL,MAAMM,MAAM,KAAK,GAAG;QAExB,IAAIC,mBAAmB;QACvB,KAAK,MAAMpC,UAAU6B,MAAO;YAC1B,mFAAmF;YACnF1B,OAAOkC,UAAU;YACjB,IAAIrB,SAASE,UAAU;gBACrBE,OAAOK,QAAQ,GAAG;gBAClBW,mBAAmB;gBACnB;YACF;YAEA,IAAI;gBACF,MAAMrC,2BAA2BC;YACnC,EAAE,OAAOsC,OAAO;gBACdlB,OAAOE,MAAM,IAAI;gBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;gBAClBpB,SAASiD,KAAK,CACZ;oBAACE,KAAKF;oBAAO9B,UAAUR,OAAOS,EAAE;gBAAA,GAChC;gBAEFrB,YAAYkD,OAAO;oBAACG,UAAU;oBAAoBC,OAAO;wBAAClC,UAAUR,OAAOS,EAAE;oBAAA;gBAAC;gBAC9E;YACF;YAEA,IAAIkC;YACJ,IAAI;gBACFA,UAAU,MAAMzC,uBAAuB;oBAACF;oBAAQa,eAAeV,OAAOU,aAAa;gBAAA;YACrF,EAAE,OAAOyB,OAAO;gBACdlB,OAAOE,MAAM,IAAI;gBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;gBAClBpB,SAASiD,KAAK,CACZ;oBAACE,KAAKF;oBAAO9B,UAAUR,OAAOS,EAAE;gBAAA,GAChC;gBAEFrB,YAAYkD,OAAO;oBAACG,UAAU;oBAAoBC,OAAO;wBAAClC,UAAUR,OAAOS,EAAE;oBAAA;gBAAC;gBAC9E;YACF;YAEA,IAAI,CAACkC,QAAQrC,OAAO,EAAE;gBACpB,IAAIsC;gBACJ,IAAI;oBACFA,UAAU,MAAM/C,qBAAqBG,OAAOS,EAAE;gBAChD,EAAE,OAAO6B,OAAO;oBACdlB,OAAOE,MAAM,IAAI;oBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;oBAClBpB,SAASiD,KAAK,CACZ;wBAACE,KAAKF;wBAAO9B,UAAUR,OAAOS,EAAE;oBAAA,GAChC;oBAEFrB,YAAYkD,OAAO;wBAACG,UAAU;wBAAoBC,OAAO;4BAAClC,UAAUR,OAAOS,EAAE;wBAAA;oBAAC;oBAC9E;gBACF;gBACA,IAAImC,SAAS;oBACX,IAAI;wBACF,MAAM7C,2BAA2B6C;wBACjCD,UAAU,MAAMzC,uBAAuB;4BACrCF,QAAQ4C;4BACR/B,eAAeV,OAAOU,aAAa;wBACrC;oBACF,EAAE,OAAOyB,OAAO;wBACdlB,OAAOE,MAAM,IAAI;wBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;wBAClBpB,SAASiD,KAAK,CACZ;4BAACE,KAAKF;4BAAO9B,UAAUR,OAAOS,EAAE;wBAAA,GAChC;wBAEFrB,YAAYkD,OAAO;4BAACG,UAAU;4BAAoBC,OAAO;gCAAClC,UAAUR,OAAOS,EAAE;4BAAA;wBAAC;wBAC9E;oBACF;gBACF;gBACA,IAAI,CAACmC,WAAW,CAACD,QAAQrC,OAAO,EAAE;oBAChC,qFAAqF;oBACrFc,OAAOC,KAAK,IAAI;oBAChBK,KAAKa,GAAG,CAACvC,OAAOS,EAAE;oBAClB;gBACF;YACF;YAEAW,OAAOd,OAAO,IAAI;YAClB,IAAIqC,QAAQ7B,gBAAgB,EAAEM,OAAOG,gBAAgB,IAAI;QAC3D;QAEA,IAAIa,kBAAkB;QAEtBhB,OAAOI,UAAU,IAAI;QACrB,IAAIK,MAAMM,MAAM,GAAGhC,OAAO4B,UAAU,EAAE;IACxC;IAEA,OAAOX;AACT"}

package/dist/core/session/parse-session.d.ts

@@ -1,5 +1,5 @@
-import type { Harness } from '@shipfox/api-agent-dto';
 import type { SessionViewRow } from '@shipfox/api-logs-dto';
+import type { Harness } from '@shipfox/workflow-document';
 import type { AgentSessionRecord } from './session-record.js';
 export declare function parseSessionRecord(record: AgentSessionRecord, harness: Harness): readonly SessionViewRow[];
 //# sourceMappingURL=parse-session.d.ts.map

package/dist/core/session/parse-session.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-session.d.ts","sourceRoot":"","sources":["../../../src/core/session/parse-session.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,wBAAwB,CAAC;AACpD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uBAAuB,CAAC;AAG1D,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,qBAAqB,CAAC;AAE5D,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,kBAAkB,EAC1B,OAAO,EAAE,OAAO,GACf,SAAS,cAAc,EAAE,CAa3B"}
+{"version":3,"file":"parse-session.d.ts","sourceRoot":"","sources":["../../../src/core/session/parse-session.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uBAAuB,CAAC;AAC1D,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,4BAA4B,CAAC;AAGxD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,qBAAqB,CAAC;AAE5D,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,kBAAkB,EAC1B,OAAO,EAAE,OAAO,GACf,SAAS,cAAc,EAAE,CAa3B"}

package/dist/core/session/parse-session.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/core/session/parse-session.ts"],"sourcesContent":["import type {Harness} from '@shipfox/api-agent-dto';\nimport type {SessionViewRow} from '@shipfox/api-logs-dto';\nimport {parseClaudeSessionRecord} from './claude-parser.js';\nimport {parsePiSessionRecord} from './pi-parser.js';\nimport type {AgentSessionRecord} from './session-record.js';\n\nexport function parseSessionRecord(\n  record: AgentSessionRecord,\n  harness: Harness,\n): readonly SessionViewRow[] {\n  try {\n    return harness === 'claude' ? parseClaudeSessionRecord(record) : parsePiSessionRecord(record);\n  } catch {\n    return [\n      {\n        kind: 'raw',\n        timestamp: record.ts,\n        label: 'Unsupported session entry',\n        raw: record.data,\n      },\n    ];\n  }\n}\n"],"names":["parseClaudeSessionRecord","parsePiSessionRecord","parseSessionRecord","record","harness","kind","timestamp","ts","label","raw","data"],"mappings":"AAEA,SAAQA,wBAAwB,QAAO,qBAAqB;AAC5D,SAAQC,oBAAoB,QAAO,iBAAiB;AAGpD,OAAO,SAASC,mBACdC,MAA0B,EAC1BC,OAAgB;IAEhB,IAAI;QACF,OAAOA,YAAY,WAAWJ,yBAAyBG,UAAUF,qBAAqBE;IACxF,EAAE,OAAM;QACN,OAAO;YACL;gBACEE,MAAM;gBACNC,WAAWH,OAAOI,EAAE;gBACpBC,OAAO;gBACPC,KAAKN,OAAOO,IAAI;YAClB;SACD;IACH;AACF"}
+{"version":3,"sources":["../../../src/core/session/parse-session.ts"],"sourcesContent":["import type {SessionViewRow} from '@shipfox/api-logs-dto';\nimport type {Harness} from '@shipfox/workflow-document';\nimport {parseClaudeSessionRecord} from './claude-parser.js';\nimport {parsePiSessionRecord} from './pi-parser.js';\nimport type {AgentSessionRecord} from './session-record.js';\n\nexport function parseSessionRecord(\n  record: AgentSessionRecord,\n  harness: Harness,\n): readonly SessionViewRow[] {\n  try {\n    return harness === 'claude' ? parseClaudeSessionRecord(record) : parsePiSessionRecord(record);\n  } catch {\n    return [\n      {\n        kind: 'raw',\n        timestamp: record.ts,\n        label: 'Unsupported session entry',\n        raw: record.data,\n      },\n    ];\n  }\n}\n"],"names":["parseClaudeSessionRecord","parsePiSessionRecord","parseSessionRecord","record","harness","kind","timestamp","ts","label","raw","data"],"mappings":"AAEA,SAAQA,wBAAwB,QAAO,qBAAqB;AAC5D,SAAQC,oBAAoB,QAAO,iBAAiB;AAGpD,OAAO,SAASC,mBACdC,MAA0B,EAC1BC,OAAgB;IAEhB,IAAI;QACF,OAAOA,YAAY,WAAWJ,yBAAyBG,UAAUF,qBAAqBE;IACxF,EAAE,OAAM;QACN,OAAO;YACL;gBACEE,MAAM;gBACNC,WAAWH,OAAOI,EAAE;gBACpBC,OAAO;gBACPC,KAAKN,OAAOO,IAAI;YAClB;SACD;IACH;AACF"}

package/dist/temporal/activities/compact-stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compact-stream.d.ts","sourceRoot":"","sources":["../../../src/temporal/activities/compact-stream.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAK,KAAK,WAAW,EAAC,MAAM,WAAW,CAAC;AAI/C,MAAM,MAAM,mBAAmB,GAC3B;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GACjB;IAAC,OAAO,EAAE,mBAAmB,CAAA;CAAC,GAC9B;IAAC,OAAO,EAAE,YAAY,CAAA;CAAC,GACvB;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAC,GAC5B;IAAC,OAAO,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAA;CAAC,CAAC;AAE7F,UAAU,yBAAyB;IACjC,mBAAmB,EAAE,OAAO,mBAAmB,CAAC;IAChD,2BAA2B,EAAE,CAC3B,EAAE,EAAE,WAAW,EACf,MAAM,EAAE;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAC,KAC1C,OAAO,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAC,CAAC,CAAC;CAClC;AAsFD,wBAAgB,2BAA2B,CACzC,YAAY,GAAE,yBAA+C,GAC5D,CAAC,MAAM,EAAE;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,KAAK,OAAO,CAAC,mBAAmB,CAAC,CAW9D;AAED,eAAO,MAAM,qBAAqB,WAbtB;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,KAAK,OAAO,CAAC,mBAAmB,CAaI,CAAC"}
+{"version":3,"file":"compact-stream.d.ts","sourceRoot":"","sources":["../../../src/temporal/activities/compact-stream.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAK,KAAK,WAAW,EAAC,MAAM,WAAW,CAAC;AAI/C,MAAM,MAAM,mBAAmB,GAC3B;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GACjB;IAAC,OAAO,EAAE,mBAAmB,CAAA;CAAC,GAC9B;IAAC,OAAO,EAAE,YAAY,CAAA;CAAC,GACvB;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAC,GAC5B;IAAC,OAAO,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAA;CAAC,CAAC;AAE7F,UAAU,yBAAyB;IACjC,mBAAmB,EAAE,OAAO,mBAAmB,CAAC;IAChD,2BAA2B,EAAE,CAC3B,EAAE,EAAE,WAAW,EACf,MAAM,EAAE;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAC,KAC1C,OAAO,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAC,CAAC,CAAC;CAClC;AAgGD,wBAAgB,2BAA2B,CACzC,YAAY,GAAE,yBAA+C,GAC5D,CAAC,MAAM,EAAE;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,KAAK,OAAO,CAAC,mBAAmB,CAAC,CAW9D;AAED,eAAO,MAAM,qBAAqB,WAbtB;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,KAAK,OAAO,CAAC,mBAAmB,CAaI,CAAC"}

package/dist/temporal/activities/compact-stream.js

@@ -1,3 +1,5 @@
+import { reportError } from '@shipfox/node-error-monitoring';
+import { logger } from '@shipfox/node-opentelemetry';
 import { Context } from '@temporalio/activity';
 import { compactedObjectKey, deleteObject, putCompactedObject } from '#api/object-storage.js';
 import { compactedGzipStream } from '#core/compaction.js';
@@ -55,7 +57,21 @@ const defaultDependencies = {
         }
     });
     if (stats.chunkCount !== expected.count || stats.lastSeq !== expected.maxSeq || stats.uncompressedBytes !== expected.uncompressedBytes) {
-        await deleteObject(uploadKey).catch(()=>undefined);
+        await deleteObject(uploadKey).catch((error)=>{
+            logger().error({
+                err: error,
+                streamId: stream.id,
+                objectKey: uploadKey
+            }, 'Failed to delete invalid compacted log object');
+            reportError(error, {
+                boundary: 'logs.cleanup',
+                operation: 'delete-invalid-compaction-object',
+                extra: {
+                    streamId: stream.id,
+                    objectKey: uploadKey
+                }
+            });
+        });
         throw new Error(`Compaction integrity check failed for stream ${stream.id}: streamed ${stats.chunkCount} chunks / ${stats.uncompressedBytes} bytes up to seq ${stats.lastSeq}, table holds ${expected.count} / ${expected.uncompressedBytes} bytes up to seq ${expected.maxSeq}`);
     }
     const { updated } = await db().transaction((tx)=>dependencies.setObjectKeyAndDeleteChunks(tx, {

package/dist/temporal/activities/compact-stream.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/temporal/activities/compact-stream.ts"],"sourcesContent":["import {Context} from '@temporalio/activity';\nimport {compactedObjectKey, deleteObject, putCompactedObject} from '#api/object-storage.js';\nimport {compactedGzipStream} from '#core/compaction.js';\nimport {chunkStats} from '#db/chunks.js';\nimport {db, type Transaction} from '#db/db.js';\nimport {getAttemptStreamById, setObjectKeyAndDeleteChunks} from '#db/streams.js';\nimport {type CompactionMetricOutcome, compactionCount} from '#metrics/instance.js';\n\nexport type CompactStreamResult =\n  | {outcome: 'gone'}\n  | {outcome: 'already-compacted'}\n  | {outcome: 'superseded'}\n  | {outcome: 'retention-raced'}\n  | {outcome: 'compacted'; objectKey: string; chunkCount: number; uncompressedBytes: number};\n\ninterface CompactStreamDependencies {\n  compactedGzipStream: typeof compactedGzipStream;\n  setObjectKeyAndDeleteChunks: (\n    tx: Transaction,\n    params: {streamId: string; objectKey: string},\n  ) => Promise<{updated: boolean}>;\n}\n\nconst defaultDependencies: CompactStreamDependencies = {\n  compactedGzipStream,\n  setObjectKeyAndDeleteChunks,\n};\n\n/**\n * Compacts one closed stream into a single gzip NDJSON object, then deletes its chunk rows.\n *\n *   load stream ──┬─ gone ─────────────► no-op (retention raced)\n *                 ├─ object_key set ───► no-op (idempotent re-run)\n *                 └─ else: upload to a per-attempt key ─► gzip ─► multipart (abort-aware, heartbeat)\n *                          │\n *                 verify streamed count/maxSeq/bytes == table  (mismatch ─► delete upload, throw, retry)\n *                          │\n *                 tx: set object_key (state='closed' AND object_key IS NULL) + delete chunks\n *                          └─ 0 rows ─► delete this attempt's upload, then re-read the row:\n *                                         gone ─► retention raced · keyed ─► superseded by another attempt\n *\n * Each attempt uploads to its own `compactedObjectKey(stream, uuid)`, so a slow or zombie\n * attempt can never overwrite a published object. The single-winner publish (object_key set\n * + chunk delete, atomic) drops chunks only once a complete object is durable; the integrity\n * check (count, maxSeq, and byte total) guards a read bug from publishing a truncated object\n * before the only copy of the source is gone (S3 part checksums cover byte transfer).\n */\nasync function compactStream(\n  params: {streamId: string},\n  dependencies: CompactStreamDependencies,\n): Promise<CompactStreamResult> {\n  const stream = await getAttemptStreamById(params.streamId);\n  if (!stream) return {outcome: 'gone'};\n  if (stream.objectKey) return {outcome: 'already-compacted'};\n\n  const ctx = Context.current();\n  const uploadKey = compactedObjectKey(stream, crypto.randomUUID());\n  const expected = await chunkStats(stream.id);\n  const {body, stats} = dependencies.compactedGzipStream({\n    streamId: stream.id,\n    onPage: () => ctx.heartbeat(),\n  });\n\n  await putCompactedObject({\n    key: uploadKey,\n    body,\n    signal: ctx.cancellationSignal,\n    onProgress: () => ctx.heartbeat(),\n    metadata: {\n      stream_id: stream.id,\n      chunk_count: String(expected.count),\n      uncompressed_bytes: String(expected.uncompressedBytes),\n      last_seq: String(expected.maxSeq),\n    },\n  });\n\n  if (\n    stats.chunkCount !== expected.count ||\n    stats.lastSeq !== expected.maxSeq ||\n    stats.uncompressedBytes !== expected.uncompressedBytes\n  ) {\n    await deleteObject(uploadKey).catch(() => undefined);\n    throw new Error(\n      `Compaction integrity check failed for stream ${stream.id}: streamed ${stats.chunkCount} chunks / ${stats.uncompressedBytes} bytes up to seq ${stats.lastSeq}, table holds ${expected.count} / ${expected.uncompressedBytes} bytes up to seq ${expected.maxSeq}`,\n    );\n  }\n\n  const {updated} = await db().transaction((tx) =>\n    dependencies.setObjectKeyAndDeleteChunks(tx, {\n      streamId: stream.id,\n      objectKey: uploadKey,\n    }),\n  );\n  if (!updated) {\n    await deleteObject(uploadKey);\n    const current = await getAttemptStreamById(stream.id);\n    return {outcome: current ? 'superseded' : 'retention-raced'};\n  }\n\n  return {\n    outcome: 'compacted',\n    objectKey: uploadKey,\n    chunkCount: stats.chunkCount,\n    uncompressedBytes: stats.uncompressedBytes,\n  };\n}\n\nexport function createCompactStreamActivity(\n  dependencies: CompactStreamDependencies = defaultDependencies,\n): (params: {streamId: string}) => Promise<CompactStreamResult> {\n  return async (params) => {\n    let outcome: CompactionMetricOutcome = 'failed';\n    try {\n      const result = await compactStream(params, dependencies);\n      outcome = result.outcome;\n      return result;\n    } finally {\n      compactionCount.add(1, {outcome});\n    }\n  };\n}\n\nexport const compactStreamActivity = createCompactStreamActivity();\n"],"names":["Context","compactedObjectKey","deleteObject","putCompactedObject","compactedGzipStream","chunkStats","db","getAttemptStreamById","setObjectKeyAndDeleteChunks","compactionCount","defaultDependencies","compactStream","params","dependencies","stream","streamId","outcome","objectKey","ctx","current","uploadKey","crypto","randomUUID","expected","id","body","stats","onPage","heartbeat","key","signal","cancellationSignal","onProgress","metadata","stream_id","chunk_count","String","count","uncompressed_bytes","uncompressedBytes","last_seq","maxSeq","chunkCount","lastSeq","catch","undefined","Error","updated","transaction","tx","createCompactStreamActivity","result","add","compactStreamActivity"],"mappings":"AAAA,SAAQA,OAAO,QAAO,uBAAuB;AAC7C,SAAQC,kBAAkB,EAAEC,YAAY,EAAEC,kBAAkB,QAAO,yBAAyB;AAC5F,SAAQC,mBAAmB,QAAO,sBAAsB;AACxD,SAAQC,UAAU,QAAO,gBAAgB;AACzC,SAAQC,EAAE,QAAyB,YAAY;AAC/C,SAAQC,oBAAoB,EAAEC,2BAA2B,QAAO,iBAAiB;AACjF,SAAsCC,eAAe,QAAO,uBAAuB;AAiBnF,MAAMC,sBAAiD;IACrDN;IACAI;AACF;AAEA;;;;;;;;;;;;;;;;;;CAkBC,GACD,eAAeG,cACbC,MAA0B,EAC1BC,YAAuC;IAEvC,MAAMC,SAAS,MAAMP,qBAAqBK,OAAOG,QAAQ;IACzD,IAAI,CAACD,QAAQ,OAAO;QAACE,SAAS;IAAM;IACpC,IAAIF,OAAOG,SAAS,EAAE,OAAO;QAACD,SAAS;IAAmB;IAE1D,MAAME,MAAMlB,QAAQmB,OAAO;IAC3B,MAAMC,YAAYnB,mBAAmBa,QAAQO,OAAOC,UAAU;IAC9D,MAAMC,WAAW,MAAMlB,WAAWS,OAAOU,EAAE;IAC3C,MAAM,EAACC,IAAI,EAAEC,KAAK,EAAC,GAAGb,aAAaT,mBAAmB,CAAC;QACrDW,UAAUD,OAAOU,EAAE;QACnBG,QAAQ,IAAMT,IAAIU,SAAS;IAC7B;IAEA,MAAMzB,mBAAmB;QACvB0B,KAAKT;QACLK;QACAK,QAAQZ,IAAIa,kBAAkB;QAC9BC,YAAY,IAAMd,IAAIU,SAAS;QAC/BK,UAAU;YACRC,WAAWpB,OAAOU,EAAE;YACpBW,aAAaC,OAAOb,SAASc,KAAK;YAClCC,oBAAoBF,OAAOb,SAASgB,iBAAiB;YACrDC,UAAUJ,OAAOb,SAASkB,MAAM;QAClC;IACF;IAEA,IACEf,MAAMgB,UAAU,KAAKnB,SAASc,KAAK,IACnCX,MAAMiB,OAAO,KAAKpB,SAASkB,MAAM,IACjCf,MAAMa,iBAAiB,KAAKhB,SAASgB,iBAAiB,EACtD;QACA,MAAMrC,aAAakB,WAAWwB,KAAK,CAAC,IAAMC;QAC1C,MAAM,IAAIC,MACR,CAAC,6CAA6C,EAAEhC,OAAOU,EAAE,CAAC,WAAW,EAAEE,MAAMgB,UAAU,CAAC,UAAU,EAAEhB,MAAMa,iBAAiB,CAAC,iBAAiB,EAAEb,MAAMiB,OAAO,CAAC,cAAc,EAAEpB,SAASc,KAAK,CAAC,GAAG,EAAEd,SAASgB,iBAAiB,CAAC,iBAAiB,EAAEhB,SAASkB,MAAM,EAAE;IAEpQ;IAEA,MAAM,EAACM,OAAO,EAAC,GAAG,MAAMzC,KAAK0C,WAAW,CAAC,CAACC,KACxCpC,aAAaL,2BAA2B,CAACyC,IAAI;YAC3ClC,UAAUD,OAAOU,EAAE;YACnBP,WAAWG;QACb;IAEF,IAAI,CAAC2B,SAAS;QACZ,MAAM7C,aAAakB;QACnB,MAAMD,UAAU,MAAMZ,qBAAqBO,OAAOU,EAAE;QACpD,OAAO;YAACR,SAASG,UAAU,eAAe;QAAiB;IAC7D;IAEA,OAAO;QACLH,SAAS;QACTC,WAAWG;QACXsB,YAAYhB,MAAMgB,UAAU;QAC5BH,mBAAmBb,MAAMa,iBAAiB;IAC5C;AACF;AAEA,OAAO,SAASW,4BACdrC,eAA0CH,mBAAmB;IAE7D,OAAO,OAAOE;QACZ,IAAII,UAAmC;QACvC,IAAI;YACF,MAAMmC,SAAS,MAAMxC,cAAcC,QAAQC;YAC3CG,UAAUmC,OAAOnC,OAAO;YACxB,OAAOmC;QACT,SAAU;YACR1C,gBAAgB2C,GAAG,CAAC,GAAG;gBAACpC;YAAO;QACjC;IACF;AACF;AAEA,OAAO,MAAMqC,wBAAwBH,8BAA8B"}
+{"version":3,"sources":["../../../src/temporal/activities/compact-stream.ts"],"sourcesContent":["import {reportError} from '@shipfox/node-error-monitoring';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {Context} from '@temporalio/activity';\nimport {compactedObjectKey, deleteObject, putCompactedObject} from '#api/object-storage.js';\nimport {compactedGzipStream} from '#core/compaction.js';\nimport {chunkStats} from '#db/chunks.js';\nimport {db, type Transaction} from '#db/db.js';\nimport {getAttemptStreamById, setObjectKeyAndDeleteChunks} from '#db/streams.js';\nimport {type CompactionMetricOutcome, compactionCount} from '#metrics/instance.js';\n\nexport type CompactStreamResult =\n  | {outcome: 'gone'}\n  | {outcome: 'already-compacted'}\n  | {outcome: 'superseded'}\n  | {outcome: 'retention-raced'}\n  | {outcome: 'compacted'; objectKey: string; chunkCount: number; uncompressedBytes: number};\n\ninterface CompactStreamDependencies {\n  compactedGzipStream: typeof compactedGzipStream;\n  setObjectKeyAndDeleteChunks: (\n    tx: Transaction,\n    params: {streamId: string; objectKey: string},\n  ) => Promise<{updated: boolean}>;\n}\n\nconst defaultDependencies: CompactStreamDependencies = {\n  compactedGzipStream,\n  setObjectKeyAndDeleteChunks,\n};\n\n/**\n * Compacts one closed stream into a single gzip NDJSON object, then deletes its chunk rows.\n *\n *   load stream ──┬─ gone ─────────────► no-op (retention raced)\n *                 ├─ object_key set ───► no-op (idempotent re-run)\n *                 └─ else: upload to a per-attempt key ─► gzip ─► multipart (abort-aware, heartbeat)\n *                          │\n *                 verify streamed count/maxSeq/bytes == table  (mismatch ─► delete upload, throw, retry)\n *                          │\n *                 tx: set object_key (state='closed' AND object_key IS NULL) + delete chunks\n *                          └─ 0 rows ─► delete this attempt's upload, then re-read the row:\n *                                         gone ─► retention raced · keyed ─► superseded by another attempt\n *\n * Each attempt uploads to its own `compactedObjectKey(stream, uuid)`, so a slow or zombie\n * attempt can never overwrite a published object. The single-winner publish (object_key set\n * + chunk delete, atomic) drops chunks only once a complete object is durable; the integrity\n * check (count, maxSeq, and byte total) guards a read bug from publishing a truncated object\n * before the only copy of the source is gone (S3 part checksums cover byte transfer).\n */\nasync function compactStream(\n  params: {streamId: string},\n  dependencies: CompactStreamDependencies,\n): Promise<CompactStreamResult> {\n  const stream = await getAttemptStreamById(params.streamId);\n  if (!stream) return {outcome: 'gone'};\n  if (stream.objectKey) return {outcome: 'already-compacted'};\n\n  const ctx = Context.current();\n  const uploadKey = compactedObjectKey(stream, crypto.randomUUID());\n  const expected = await chunkStats(stream.id);\n  const {body, stats} = dependencies.compactedGzipStream({\n    streamId: stream.id,\n    onPage: () => ctx.heartbeat(),\n  });\n\n  await putCompactedObject({\n    key: uploadKey,\n    body,\n    signal: ctx.cancellationSignal,\n    onProgress: () => ctx.heartbeat(),\n    metadata: {\n      stream_id: stream.id,\n      chunk_count: String(expected.count),\n      uncompressed_bytes: String(expected.uncompressedBytes),\n      last_seq: String(expected.maxSeq),\n    },\n  });\n\n  if (\n    stats.chunkCount !== expected.count ||\n    stats.lastSeq !== expected.maxSeq ||\n    stats.uncompressedBytes !== expected.uncompressedBytes\n  ) {\n    await deleteObject(uploadKey).catch((error) => {\n      logger().error(\n        {err: error, streamId: stream.id, objectKey: uploadKey},\n        'Failed to delete invalid compacted log object',\n      );\n      reportError(error, {\n        boundary: 'logs.cleanup',\n        operation: 'delete-invalid-compaction-object',\n        extra: {streamId: stream.id, objectKey: uploadKey},\n      });\n    });\n    throw new Error(\n      `Compaction integrity check failed for stream ${stream.id}: streamed ${stats.chunkCount} chunks / ${stats.uncompressedBytes} bytes up to seq ${stats.lastSeq}, table holds ${expected.count} / ${expected.uncompressedBytes} bytes up to seq ${expected.maxSeq}`,\n    );\n  }\n\n  const {updated} = await db().transaction((tx) =>\n    dependencies.setObjectKeyAndDeleteChunks(tx, {\n      streamId: stream.id,\n      objectKey: uploadKey,\n    }),\n  );\n  if (!updated) {\n    await deleteObject(uploadKey);\n    const current = await getAttemptStreamById(stream.id);\n    return {outcome: current ? 'superseded' : 'retention-raced'};\n  }\n\n  return {\n    outcome: 'compacted',\n    objectKey: uploadKey,\n    chunkCount: stats.chunkCount,\n    uncompressedBytes: stats.uncompressedBytes,\n  };\n}\n\nexport function createCompactStreamActivity(\n  dependencies: CompactStreamDependencies = defaultDependencies,\n): (params: {streamId: string}) => Promise<CompactStreamResult> {\n  return async (params) => {\n    let outcome: CompactionMetricOutcome = 'failed';\n    try {\n      const result = await compactStream(params, dependencies);\n      outcome = result.outcome;\n      return result;\n    } finally {\n      compactionCount.add(1, {outcome});\n    }\n  };\n}\n\nexport const compactStreamActivity = createCompactStreamActivity();\n"],"names":["reportError","logger","Context","compactedObjectKey","deleteObject","putCompactedObject","compactedGzipStream","chunkStats","db","getAttemptStreamById","setObjectKeyAndDeleteChunks","compactionCount","defaultDependencies","compactStream","params","dependencies","stream","streamId","outcome","objectKey","ctx","current","uploadKey","crypto","randomUUID","expected","id","body","stats","onPage","heartbeat","key","signal","cancellationSignal","onProgress","metadata","stream_id","chunk_count","String","count","uncompressed_bytes","uncompressedBytes","last_seq","maxSeq","chunkCount","lastSeq","catch","error","err","boundary","operation","extra","Error","updated","transaction","tx","createCompactStreamActivity","result","add","compactStreamActivity"],"mappings":"AAAA,SAAQA,WAAW,QAAO,iCAAiC;AAC3D,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,OAAO,QAAO,uBAAuB;AAC7C,SAAQC,kBAAkB,EAAEC,YAAY,EAAEC,kBAAkB,QAAO,yBAAyB;AAC5F,SAAQC,mBAAmB,QAAO,sBAAsB;AACxD,SAAQC,UAAU,QAAO,gBAAgB;AACzC,SAAQC,EAAE,QAAyB,YAAY;AAC/C,SAAQC,oBAAoB,EAAEC,2BAA2B,QAAO,iBAAiB;AACjF,SAAsCC,eAAe,QAAO,uBAAuB;AAiBnF,MAAMC,sBAAiD;IACrDN;IACAI;AACF;AAEA;;;;;;;;;;;;;;;;;;CAkBC,GACD,eAAeG,cACbC,MAA0B,EAC1BC,YAAuC;IAEvC,MAAMC,SAAS,MAAMP,qBAAqBK,OAAOG,QAAQ;IACzD,IAAI,CAACD,QAAQ,OAAO;QAACE,SAAS;IAAM;IACpC,IAAIF,OAAOG,SAAS,EAAE,OAAO;QAACD,SAAS;IAAmB;IAE1D,MAAME,MAAMlB,QAAQmB,OAAO;IAC3B,MAAMC,YAAYnB,mBAAmBa,QAAQO,OAAOC,UAAU;IAC9D,MAAMC,WAAW,MAAMlB,WAAWS,OAAOU,EAAE;IAC3C,MAAM,EAACC,IAAI,EAAEC,KAAK,EAAC,GAAGb,aAAaT,mBAAmB,CAAC;QACrDW,UAAUD,OAAOU,EAAE;QACnBG,QAAQ,IAAMT,IAAIU,SAAS;IAC7B;IAEA,MAAMzB,mBAAmB;QACvB0B,KAAKT;QACLK;QACAK,QAAQZ,IAAIa,kBAAkB;QAC9BC,YAAY,IAAMd,IAAIU,SAAS;QAC/BK,UAAU;YACRC,WAAWpB,OAAOU,EAAE;YACpBW,aAAaC,OAAOb,SAASc,KAAK;YAClCC,oBAAoBF,OAAOb,SAASgB,iBAAiB;YACrDC,UAAUJ,OAAOb,SAASkB,MAAM;QAClC;IACF;IAEA,IACEf,MAAMgB,UAAU,KAAKnB,SAASc,KAAK,IACnCX,MAAMiB,OAAO,KAAKpB,SAASkB,MAAM,IACjCf,MAAMa,iBAAiB,KAAKhB,SAASgB,iBAAiB,EACtD;QACA,MAAMrC,aAAakB,WAAWwB,KAAK,CAAC,CAACC;YACnC9C,SAAS8C,KAAK,CACZ;gBAACC,KAAKD;gBAAO9B,UAAUD,OAAOU,EAAE;gBAAEP,WAAWG;YAAS,GACtD;YAEFtB,YAAY+C,OAAO;gBACjBE,UAAU;gBACVC,WAAW;gBACXC,OAAO;oBAAClC,UAAUD,OAAOU,EAAE;oBAAEP,WAAWG;gBAAS;YACnD;QACF;QACA,MAAM,IAAI8B,MACR,CAAC,6CAA6C,EAAEpC,OAAOU,EAAE,CAAC,WAAW,EAAEE,MAAMgB,UAAU,CAAC,UAAU,EAAEhB,MAAMa,iBAAiB,CAAC,iBAAiB,EAAEb,MAAMiB,OAAO,CAAC,cAAc,EAAEpB,SAASc,KAAK,CAAC,GAAG,EAAEd,SAASgB,iBAAiB,CAAC,iBAAiB,EAAEhB,SAASkB,MAAM,EAAE;IAEpQ;IAEA,MAAM,EAACU,OAAO,EAAC,GAAG,MAAM7C,KAAK8C,WAAW,CAAC,CAACC,KACxCxC,aAAaL,2BAA2B,CAAC6C,IAAI;YAC3CtC,UAAUD,OAAOU,EAAE;YACnBP,WAAWG;QACb;IAEF,IAAI,CAAC+B,SAAS;QACZ,MAAMjD,aAAakB;QACnB,MAAMD,UAAU,MAAMZ,qBAAqBO,OAAOU,EAAE;QACpD,OAAO;YAACR,SAASG,UAAU,eAAe;QAAiB;IAC7D;IAEA,OAAO;QACLH,SAAS;QACTC,WAAWG;QACXsB,YAAYhB,MAAMgB,UAAU;QAC5BH,mBAAmBb,MAAMa,iBAAiB;IAC5C;AACF;AAEA,OAAO,SAASe,4BACdzC,eAA0CH,mBAAmB;IAE7D,OAAO,OAAOE;QACZ,IAAII,UAAmC;QACvC,IAAI;YACF,MAAMuC,SAAS,MAAM5C,cAAcC,QAAQC;YAC3CG,UAAUuC,OAAOvC,OAAO;YACxB,OAAOuC;QACT,SAAU;YACR9C,gBAAgB+C,GAAG,CAAC,GAAG;gBAACxC;YAAO;QACjC;IACF;AACF;AAEA,OAAO,MAAMyC,wBAAwBH,8BAA8B"}