@shipfox/api-logs 6.0.0 → 7.1.0

package/.turbo/turbo-build.log

@@ -1,21 +1,21 @@
 $ shipfox-swc && shipfox-temporal-bundle dist/temporal/workflows/index.js
-Successfully compiled: 59 files with swc (510.82ms)
-2026-07-21T08:11:10.748Z [INFO] asset workflow-bundle-2850a9a125f53b77d607.js 1.46 MiB [emitted] [immutable] (name: main)
-runtime modules 670 bytes 3 modules
-modules by path ../../../node_modules/.pnpm/ 424 KiB
-  modules by path ../../../node_modules/.pnpm/@temporalio+common@1.18.1/node_modules/@temporal...(truncated) 155 KiB 34 modules
-  modules by path ../../../node_modules/.pnpm/@temporalio+workflow@1.18.1/node_modules/@tempor...(truncated) 201 KiB 26 modules
-  modules by path ../../../node_modules/.pnpm/nexus-rpc@0.0.2/node_modules/nexus-rpc/lib/ 22.3 KiB 13 modules
-  + 2 modules
-modules by path ./dist/temporal/workflows/ 4.17 KiB
-  ./dist/temporal/workflows/index-autogenerated-entrypoint.cjs 523 bytes [built] [code generated]
+Successfully compiled: 59 files with swc (568.34ms)
+2026-07-22T13:11:51.110Z [INFO] asset workflow-bundle-17839eee3b21fbae5b8c.js 3.18 MiB [emitted] [immutable] (name: main)
+orphan modules 33.5 KiB [orphan] 21 modules
+runtime modules 1.13 KiB 5 modules
+modules by path ../../../node_modules/.pnpm/ 970 KiB 196 modules
+modules by path ./dist/temporal/workflows/ 4.57 KiB
+  ./dist/temporal/workflows/index-autogenerated-entrypoint.cjs 929 bytes [built] [code generated]
   ./dist/temporal/workflows/index.js 340 bytes [built] [code generated]
   ./dist/temporal/workflows/close-abandoned-streams.js 753 bytes [built] [code generated]
   ./dist/temporal/workflows/compact-stream.js 941 bytes [built] [code generated]
-  + 3 modules
+  ./dist/temporal/workflows/compaction-reconcile-cron.js 559 bytes [built] [code generated]
+  ./dist/temporal/workflows/reap-stale-open-streams-cron.js 429 bytes [built] [code generated]
+  ./dist/temporal/workflows/retention-sweep-cron.js 727 bytes [built] [code generated]
 optional modules 30 bytes [optional]
   __temporal_custom_payload_converter (ignored) 15 bytes [optional] [built] [code generated]
   __temporal_custom_failure_converter (ignored) 15 bytes [optional] [built] [code generated]
-webpack 5.107.2 compiled successfully in 3800 ms
-2026-07-21T08:11:10.761Z [INFO] Workflow bundle created { size: '1.46MB' }
-/home/runner/work/shipfox/shipfox/libs/api/logs/dist/temporal/workflows/index.js: 1530555 bytes
+../../shared/node/temporal/dist/workflow-error-interceptor.js 1.21 KiB [built] [code generated]
+webpack 5.107.2 compiled successfully in 8256 ms
+2026-07-22T13:11:51.130Z [INFO] Workflow bundle created { size: '3.18MB' }
+/home/runner/work/shipfox/shipfox/libs/api/logs/dist/temporal/workflows/index.js: 3335222 bytes

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @shipfox/api-logs
 
+## 7.1.0
+
+### Patch Changes
+
+- ac42c96: Adds boundary-owned reporting for unexpected API runtime failures while preserving expected client and provider outcomes.
+- Updated dependencies [ac42c96]
+- Updated dependencies [6ce08c0]
+  - @shipfox/node-error-monitoring@0.2.0
+  - @shipfox/node-fastify@0.3.0
+  - @shipfox/node-module@0.5.0
+  - @shipfox/node-temporal@0.4.0
+  - @shipfox/node-opentelemetry@0.6.0
+  - @shipfox/api-auth-context@7.1.0
+
 ## 6.0.0
 
 ### Major Changes

package/dist/api/object-storage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"object-storage.d.ts","sourceRoot":"","sources":["../../src/api/object-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,EAML,QAAQ,EACT,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAC,KAAK,kBAAkB,EAAe,MAAM,8BAA8B,CAAC;AAYnF,oFAAoF;AACpF,wBAAgB,QAAQ,IAAI,QAAQ,CAcnC;AAED,wBAAgB,aAAa,IAAI,IAAI,CAGpC;AAED,6HAA6H;AAC7H,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,OAAO,CAAC,CAO7D;AAID;;;;;GAKG;AACH,wBAAgB,cAAc,IAAI,QAAQ,CAYzC;AAED,wBAAgB,mBAAmB,IAAI,IAAI,CAG1C;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAE5F;AAOD,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,QAAQ,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,IAAI,CAAC;CACzB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiCxF;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE7D;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAC,CAAC,CAYhG;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCzE"}
+{"version":3,"file":"object-storage.d.ts","sourceRoot":"","sources":["../../src/api/object-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,EAML,QAAQ,EACT,MAAM,oBAAoB,CAAC;AAM5B,OAAO,EAAC,KAAK,kBAAkB,EAAe,MAAM,8BAA8B,CAAC;AAYnF,oFAAoF;AACpF,wBAAgB,QAAQ,IAAI,QAAQ,CAcnC;AAED,wBAAgB,aAAa,IAAI,IAAI,CAGpC;AAED,6HAA6H;AAC7H,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,OAAO,CAAC,CAO7D;AAID;;;;;GAKG;AACH,wBAAgB,cAAc,IAAI,QAAQ,CAYzC;AAED,wBAAgB,mBAAmB,IAAI,IAAI,CAG1C;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAE5F;AAOD,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,QAAQ,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,IAAI,CAAC;CACzB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAkDxF;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE7D;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAC,CAAC,CAYhG;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCzE"}

package/dist/api/object-storage.js

@@ -1,6 +1,8 @@
 import { DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, HeadBucketCommand, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import { reportError } from '@shipfox/node-error-monitoring';
+import { logger } from '@shipfox/node-opentelemetry';
 import { config } from '#config.js';
 import { logObjectKey } from '#core/entities/log-object.js';
 const explicitS3Credentials = config.LOG_STORAGE_S3_ACCESS_KEY_ID && config.LOG_STORAGE_S3_SECRET_ACCESS_KEY ? {
@@ -114,12 +116,36 @@ const UPLOAD_QUEUE_SIZE = 2;
         // Subscribe before the aborted-check so a signal firing between them cannot be missed,
         // then handle the already-aborted case eagerly (an already-aborted signal never fires 'abort').
         params.signal.addEventListener('abort', ()=>{
-            void upload.abort().catch(()=>undefined);
+            void upload.abort().catch((error)=>{
+                logger().error({
+                    err: error,
+                    objectKey: params.key
+                }, 'Failed to abort multipart log upload');
+                reportError(error, {
+                    boundary: 'logs.cleanup',
+                    operation: 'abort-multipart-upload',
+                    extra: {
+                        objectKey: params.key
+                    }
+                });
+            });
         }, {
             once: true
         });
         if (params.signal.aborted) {
-            await upload.abort().catch(()=>undefined);
+            await upload.abort().catch((error)=>{
+                logger().error({
+                    err: error,
+                    objectKey: params.key
+                }, 'Failed to abort multipart log upload');
+                reportError(error, {
+                    boundary: 'logs.cleanup',
+                    operation: 'abort-multipart-upload',
+                    extra: {
+                        objectKey: params.key
+                    }
+                });
+            });
             throw new Error('Compaction upload aborted before it started');
         }
     }

package/dist/api/object-storage.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/api/object-storage.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  DeleteObjectCommand,\n  DeleteObjectsCommand,\n  GetObjectCommand,\n  HeadBucketCommand,\n  ListObjectsV2Command,\n  S3Client,\n} from '@aws-sdk/client-s3';\nimport {Upload} from '@aws-sdk/lib-storage';\nimport {getSignedUrl} from '@aws-sdk/s3-request-presigner';\nimport {config} from '#config.js';\nimport {type LogObjectKeyParams, logObjectKey} from '#core/entities/log-object.js';\n\nconst explicitS3Credentials =\n  config.LOG_STORAGE_S3_ACCESS_KEY_ID && config.LOG_STORAGE_S3_SECRET_ACCESS_KEY\n    ? {\n        accessKeyId: config.LOG_STORAGE_S3_ACCESS_KEY_ID,\n        secretAccessKey: config.LOG_STORAGE_S3_SECRET_ACCESS_KEY,\n      }\n    : undefined;\n\nlet _client: S3Client | undefined;\n\n/** Lazily-built S3 client targeting the configured object store (Garage in dev). */\nexport function s3Client(): S3Client {\n  if (!_client) {\n    _client = new S3Client({\n      endpoint: config.LOG_STORAGE_S3_ENDPOINT,\n      region: config.LOG_STORAGE_S3_REGION,\n      forcePathStyle: config.LOG_STORAGE_S3_FORCE_PATH_STYLE,\n      ...(explicitS3Credentials ? {credentials: explicitS3Credentials} : {}),\n      // Fail fast: a slow or black-holed endpoint must not hang callers behind\n      // SDK backoff. One retry, short connect/request timeouts.\n      maxAttempts: 2,\n      requestHandler: {connectionTimeout: 1_000, requestTimeout: 3_000},\n    });\n  }\n  return _client;\n}\n\nexport function closeS3Client(): void {\n  _client?.destroy();\n  _client = undefined;\n}\n\n/** Readiness probe: true when the configured bucket answers a HEAD. Used by the app readiness check, not at process boot. */\nexport async function checkBucketReachable(): Promise<boolean> {\n  try {\n    await s3Client().send(new HeadBucketCommand({Bucket: config.LOG_STORAGE_S3_BUCKET}));\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nlet _uploadClient: S3Client | undefined;\n\n/**\n * S3 client for streaming compaction uploads. `s3Client()` is tuned fail-fast for the\n * readiness HEAD, but each multipart part upload can take as long as the link needs, so\n * the per-request timeout is disabled here. Temporal owns the retry budget, so the SDK\n * keeps only a small attempt count for transient connection blips.\n */\nexport function uploadS3Client(): S3Client {\n  if (!_uploadClient) {\n    _uploadClient = new S3Client({\n      endpoint: config.LOG_STORAGE_S3_ENDPOINT,\n      region: config.LOG_STORAGE_S3_REGION,\n      forcePathStyle: config.LOG_STORAGE_S3_FORCE_PATH_STYLE,\n      ...(explicitS3Credentials ? {credentials: explicitS3Credentials} : {}),\n      maxAttempts: 3,\n      requestHandler: {connectionTimeout: 5_000, requestTimeout: 0},\n    });\n  }\n  return _uploadClient;\n}\n\nexport function closeUploadS3Client(): void {\n  _uploadClient?.destroy();\n  _uploadClient = undefined;\n}\n\n/**\n * Per-attempt object key for a compacted stream: the stream's stable prefix\n * (`logObjectKey` -> `{prefix}/{workspaceId}/{jobId}/{stepId}/{attempt}`, shared with retention\n * and workspace prefix deletes) plus a unique `uploadToken` leaf. Each compaction attempt\n * uploads to its own key and the winner records it atomically, so a slow or zombie attempt\n * can never overwrite a published object. Losing and crashed attempts leave a leaf under the\n * same prefix, reclaimed by the bucket's incomplete-multipart-abort rule and prefix-scoped\n * retention.\n */\nexport function compactedObjectKey(identity: LogObjectKeyParams, uploadToken: string): string {\n  return `${logObjectKey(config.LOG_STORAGE_S3_PREFIX, identity)}/${uploadToken}`;\n}\n\n// lib-storage defaults (5MB parts, queue of 4); we pin a small queue so peak buffer memory\n// stays bounded under concurrent compactions (queueSize x partSize per active upload).\nconst UPLOAD_PART_SIZE = 5 * 1024 * 1024;\nconst UPLOAD_QUEUE_SIZE = 2;\n\nexport interface PutCompactedObjectParams {\n  key: string;\n  body: Readable;\n  metadata: Record<string, string>;\n  signal?: AbortSignal;\n  onProgress?: () => void;\n}\n\n/**\n * Streams `body` (gzip-compressed NDJSON) to the compacted object via a multipart upload.\n * `Content-Encoding: gzip` so a browser-direct presigned GET auto-decompresses. Aborts the\n * multipart upload when `signal` fires, so a cancelled activity leaves no dangling parts.\n *\n * A hard crash (no graceful abort) can still strand incomplete multipart parts, which on\n * pay-per-storage providers (S3, R2) bill until aborted. The worker never sets bucket\n * policy, so the deployed bucket must carry an `AbortIncompleteMultipartUpload` lifecycle\n * rule. `dev/garage/bootstrap.sh` provisions it for local Garage; self-hosters set the same\n * rule on their bucket.\n */\nexport async function putCompactedObject(params: PutCompactedObjectParams): Promise<void> {\n  const upload = new Upload({\n    client: uploadS3Client(),\n    partSize: UPLOAD_PART_SIZE,\n    queueSize: UPLOAD_QUEUE_SIZE,\n    params: {\n      Bucket: config.LOG_STORAGE_S3_BUCKET,\n      Key: params.key,\n      Body: params.body,\n      ContentType: 'application/x-ndjson',\n      ContentEncoding: 'gzip',\n      Metadata: params.metadata,\n    },\n  });\n\n  if (params.onProgress) upload.on('httpUploadProgress', params.onProgress);\n  if (params.signal) {\n    // Subscribe before the aborted-check so a signal firing between them cannot be missed,\n    // then handle the already-aborted case eagerly (an already-aborted signal never fires 'abort').\n    params.signal.addEventListener(\n      'abort',\n      () => {\n        void upload.abort().catch(() => undefined);\n      },\n      {once: true},\n    );\n    if (params.signal.aborted) {\n      await upload.abort().catch(() => undefined);\n      throw new Error('Compaction upload aborted before it started');\n    }\n  }\n\n  await upload.done();\n}\n\n/**\n * Deletes a compacted object. S3 `DeleteObject` is idempotent (a missing key is a success),\n * so retention and the publish orphan guard can call it freely without a pre-check.\n */\nexport async function deleteObject(key: string): Promise<void> {\n  await s3Client().send(new DeleteObjectCommand({Bucket: config.LOG_STORAGE_S3_BUCKET, Key: key}));\n}\n\n/**\n * Presigns a GET for a compacted log object so the browser can fetch it directly,\n * bypassing API egress. `getSignedUrl` only computes a signature locally (no network\n * call), so the fail-fast `s3Client()` is fine here. `expiresAt` mirrors the URL's\n * `LOG_READ_URL_TTL_SECONDS` lifetime so the caller can hand the client an absolute\n * deadline without re-deriving it.\n */\nexport async function presignedGetUrl(objectKey: string): Promise<{url: string; expiresAt: Date}> {\n  const ttlSeconds = config.LOG_READ_URL_TTL_SECONDS;\n  // Stamp the deadline from before signing: getSignedUrl bakes its expiry from the clock at\n  // sign time, so reading Date.now() after the await could report a deadline slightly past the\n  // real one. Computing it first keeps expiresAt a conservative (never-late) bound.\n  const expiresAt = new Date(Date.now() + ttlSeconds * 1000);\n  const url = await getSignedUrl(\n    s3Client(),\n    new GetObjectCommand({Bucket: config.LOG_STORAGE_S3_BUCKET, Key: objectKey}),\n    {expiresIn: ttlSeconds},\n  );\n  return {url, expiresAt};\n}\n\n/**\n * Deletes all objects under a per-attempt prefix. Retention deletes by prefix, not only the\n * recorded `object_key`, to reclaim orphan leaves from failed compaction attempts.\n *\n * Pass a trailing slash so attempt `1` never matches attempt `10`.\n */\nexport async function deleteObjectsByPrefix(prefix: string): Promise<void> {\n  if (prefix === '') {\n    throw new Error(\n      'deleteObjectsByPrefix refuses an empty prefix (it would target the whole bucket)',\n    );\n  }\n  const client = s3Client();\n  let continuationToken: string | undefined;\n  do {\n    const listed = await client.send(\n      new ListObjectsV2Command({\n        Bucket: config.LOG_STORAGE_S3_BUCKET,\n        Prefix: prefix,\n        ContinuationToken: continuationToken,\n      }),\n    );\n    const objects = (listed.Contents ?? []).flatMap((object) =>\n      object.Key ? [{Key: object.Key}] : [],\n    );\n    if (objects.length > 0) {\n      const deleted = await client.send(\n        new DeleteObjectsCommand({\n          Bucket: config.LOG_STORAGE_S3_BUCKET,\n          Delete: {Objects: objects, Quiet: true},\n        }),\n      );\n      if (deleted.Errors && deleted.Errors.length > 0) {\n        const [first] = deleted.Errors;\n        throw new Error(\n          `Failed to delete ${deleted.Errors.length} object(s) under ${prefix}: ${first?.Key} ${first?.Message}`,\n        );\n      }\n    }\n    continuationToken = listed.IsTruncated ? listed.NextContinuationToken : undefined;\n  } while (continuationToken);\n}\n"],"names":["DeleteObjectCommand","DeleteObjectsCommand","GetObjectCommand","HeadBucketCommand","ListObjectsV2Command","S3Client","Upload","getSignedUrl","config","logObjectKey","explicitS3Credentials","LOG_STORAGE_S3_ACCESS_KEY_ID","LOG_STORAGE_S3_SECRET_ACCESS_KEY","accessKeyId","secretAccessKey","undefined","_client","s3Client","endpoint","LOG_STORAGE_S3_ENDPOINT","region","LOG_STORAGE_S3_REGION","forcePathStyle","LOG_STORAGE_S3_FORCE_PATH_STYLE","credentials","maxAttempts","requestHandler","connectionTimeout","requestTimeout","closeS3Client","destroy","checkBucketReachable","send","Bucket","LOG_STORAGE_S3_BUCKET","_uploadClient","uploadS3Client","closeUploadS3Client","compactedObjectKey","identity","uploadToken","LOG_STORAGE_S3_PREFIX","UPLOAD_PART_SIZE","UPLOAD_QUEUE_SIZE","putCompactedObject","params","upload","client","partSize","queueSize","Key","key","Body","body","ContentType","ContentEncoding","Metadata","metadata","onProgress","on","signal","addEventListener","abort","catch","once","aborted","Error","done","deleteObject","presignedGetUrl","objectKey","ttlSeconds","LOG_READ_URL_TTL_SECONDS","expiresAt","Date","now","url","expiresIn","deleteObjectsByPrefix","prefix","continuationToken","listed","Prefix","ContinuationToken","objects","Contents","flatMap","object","length","deleted","Delete","Objects","Quiet","Errors","first","Message","IsTruncated","NextContinuationToken"],"mappings":"AACA,SACEA,mBAAmB,EACnBC,oBAAoB,EACpBC,gBAAgB,EAChBC,iBAAiB,EACjBC,oBAAoB,EACpBC,QAAQ,QACH,qBAAqB;AAC5B,SAAQC,MAAM,QAAO,uBAAuB;AAC5C,SAAQC,YAAY,QAAO,gCAAgC;AAC3D,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAiCC,YAAY,QAAO,+BAA+B;AAEnF,MAAMC,wBACJF,OAAOG,4BAA4B,IAAIH,OAAOI,gCAAgC,GAC1E;IACEC,aAAaL,OAAOG,4BAA4B;IAChDG,iBAAiBN,OAAOI,gCAAgC;AAC1D,IACAG;AAEN,IAAIC;AAEJ,kFAAkF,GAClF,OAAO,SAASC;IACd,IAAI,CAACD,SAAS;QACZA,UAAU,IAAIX,SAAS;YACrBa,UAAUV,OAAOW,uBAAuB;YACxCC,QAAQZ,OAAOa,qBAAqB;YACpCC,gBAAgBd,OAAOe,+BAA+B;YACtD,GAAIb,wBAAwB;gBAACc,aAAad;YAAqB,IAAI,CAAC,CAAC;YACrE,yEAAyE;YACzE,0DAA0D;YAC1De,aAAa;YACbC,gBAAgB;gBAACC,mBAAmB;gBAAOC,gBAAgB;YAAK;QAClE;IACF;IACA,OAAOZ;AACT;AAEA,OAAO,SAASa;IACdb,SAASc;IACTd,UAAUD;AACZ;AAEA,2HAA2H,GAC3H,OAAO,eAAegB;IACpB,IAAI;QACF,MAAMd,WAAWe,IAAI,CAAC,IAAI7B,kBAAkB;YAAC8B,QAAQzB,OAAO0B,qBAAqB;QAAA;QACjF,OAAO;IACT,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,IAAIC;AAEJ;;;;;CAKC,GACD,OAAO,SAASC;IACd,IAAI,CAACD,eAAe;QAClBA,gBAAgB,IAAI9B,SAAS;YAC3Ba,UAAUV,OAAOW,uBAAuB;YACxCC,QAAQZ,OAAOa,qBAAqB;YACpCC,gBAAgBd,OAAOe,+BAA+B;YACtD,GAAIb,wBAAwB;gBAACc,aAAad;YAAqB,IAAI,CAAC,CAAC;YACrEe,aAAa;YACbC,gBAAgB;gBAACC,mBAAmB;gBAAOC,gBAAgB;YAAC;QAC9D;IACF;IACA,OAAOO;AACT;AAEA,OAAO,SAASE;IACdF,eAAeL;IACfK,gBAAgBpB;AAClB;AAEA;;;;;;;;CAQC,GACD,OAAO,SAASuB,mBAAmBC,QAA4B,EAAEC,WAAmB;IAClF,OAAO,GAAG/B,aAAaD,OAAOiC,qBAAqB,EAAEF,UAAU,CAAC,EAAEC,aAAa;AACjF;AAEA,2FAA2F;AAC3F,uFAAuF;AACvF,MAAME,mBAAmB,IAAI,OAAO;AACpC,MAAMC,oBAAoB;AAU1B;;;;;;;;;;CAUC,GACD,OAAO,eAAeC,mBAAmBC,MAAgC;IACvE,MAAMC,SAAS,IAAIxC,OAAO;QACxByC,QAAQX;QACRY,UAAUN;QACVO,WAAWN;QACXE,QAAQ;YACNZ,QAAQzB,OAAO0B,qBAAqB;YACpCgB,KAAKL,OAAOM,GAAG;YACfC,MAAMP,OAAOQ,IAAI;YACjBC,aAAa;YACbC,iBAAiB;YACjBC,UAAUX,OAAOY,QAAQ;QAC3B;IACF;IAEA,IAAIZ,OAAOa,UAAU,EAAEZ,OAAOa,EAAE,CAAC,sBAAsBd,OAAOa,UAAU;IACxE,IAAIb,OAAOe,MAAM,EAAE;QACjB,uFAAuF;QACvF,gGAAgG;QAChGf,OAAOe,MAAM,CAACC,gBAAgB,CAC5B,SACA;YACE,KAAKf,OAAOgB,KAAK,GAAGC,KAAK,CAAC,IAAMhD;QAClC,GACA;YAACiD,MAAM;QAAI;QAEb,IAAInB,OAAOe,MAAM,CAACK,OAAO,EAAE;YACzB,MAAMnB,OAAOgB,KAAK,GAAGC,KAAK,CAAC,IAAMhD;YACjC,MAAM,IAAImD,MAAM;QAClB;IACF;IAEA,MAAMpB,OAAOqB,IAAI;AACnB;AAEA;;;CAGC,GACD,OAAO,eAAeC,aAAajB,GAAW;IAC5C,MAAMlC,WAAWe,IAAI,CAAC,IAAIhC,oBAAoB;QAACiC,QAAQzB,OAAO0B,qBAAqB;QAAEgB,KAAKC;IAAG;AAC/F;AAEA;;;;;;CAMC,GACD,OAAO,eAAekB,gBAAgBC,SAAiB;IACrD,MAAMC,aAAa/D,OAAOgE,wBAAwB;IAClD,0FAA0F;IAC1F,6FAA6F;IAC7F,kFAAkF;IAClF,MAAMC,YAAY,IAAIC,KAAKA,KAAKC,GAAG,KAAKJ,aAAa;IACrD,MAAMK,MAAM,MAAMrE,aAChBU,YACA,IAAIf,iBAAiB;QAAC+B,QAAQzB,OAAO0B,qBAAqB;QAAEgB,KAAKoB;IAAS,IAC1E;QAACO,WAAWN;IAAU;IAExB,OAAO;QAACK;QAAKH;IAAS;AACxB;AAEA;;;;;CAKC,GACD,OAAO,eAAeK,sBAAsBC,MAAc;IACxD,IAAIA,WAAW,IAAI;QACjB,MAAM,IAAIb,MACR;IAEJ;IACA,MAAMnB,SAAS9B;IACf,IAAI+D;IACJ,GAAG;QACD,MAAMC,SAAS,MAAMlC,OAAOf,IAAI,CAC9B,IAAI5B,qBAAqB;YACvB6B,QAAQzB,OAAO0B,qBAAqB;YACpCgD,QAAQH;YACRI,mBAAmBH;QACrB;QAEF,MAAMI,UAAU,AAACH,CAAAA,OAAOI,QAAQ,IAAI,EAAE,AAAD,EAAGC,OAAO,CAAC,CAACC,SAC/CA,OAAOrC,GAAG,GAAG;gBAAC;oBAACA,KAAKqC,OAAOrC,GAAG;gBAAA;aAAE,GAAG,EAAE;QAEvC,IAAIkC,QAAQI,MAAM,GAAG,GAAG;YACtB,MAAMC,UAAU,MAAM1C,OAAOf,IAAI,CAC/B,IAAI/B,qBAAqB;gBACvBgC,QAAQzB,OAAO0B,qBAAqB;gBACpCwD,QAAQ;oBAACC,SAASP;oBAASQ,OAAO;gBAAI;YACxC;YAEF,IAAIH,QAAQI,MAAM,IAAIJ,QAAQI,MAAM,CAACL,MAAM,GAAG,GAAG;gBAC/C,MAAM,CAACM,MAAM,GAAGL,QAAQI,MAAM;gBAC9B,MAAM,IAAI3B,MACR,CAAC,iBAAiB,EAAEuB,QAAQI,MAAM,CAACL,MAAM,CAAC,iBAAiB,EAAET,OAAO,EAAE,EAAEe,OAAO5C,IAAI,CAAC,EAAE4C,OAAOC,SAAS;YAE1G;QACF;QACAf,oBAAoBC,OAAOe,WAAW,GAAGf,OAAOgB,qBAAqB,GAAGlF;IAC1E,QAASiE,kBAAmB;AAC9B"}
+{"version":3,"sources":["../../src/api/object-storage.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  DeleteObjectCommand,\n  DeleteObjectsCommand,\n  GetObjectCommand,\n  HeadBucketCommand,\n  ListObjectsV2Command,\n  S3Client,\n} from '@aws-sdk/client-s3';\nimport {Upload} from '@aws-sdk/lib-storage';\nimport {getSignedUrl} from '@aws-sdk/s3-request-presigner';\nimport {reportError} from '@shipfox/node-error-monitoring';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {config} from '#config.js';\nimport {type LogObjectKeyParams, logObjectKey} from '#core/entities/log-object.js';\n\nconst explicitS3Credentials =\n  config.LOG_STORAGE_S3_ACCESS_KEY_ID && config.LOG_STORAGE_S3_SECRET_ACCESS_KEY\n    ? {\n        accessKeyId: config.LOG_STORAGE_S3_ACCESS_KEY_ID,\n        secretAccessKey: config.LOG_STORAGE_S3_SECRET_ACCESS_KEY,\n      }\n    : undefined;\n\nlet _client: S3Client | undefined;\n\n/** Lazily-built S3 client targeting the configured object store (Garage in dev). */\nexport function s3Client(): S3Client {\n  if (!_client) {\n    _client = new S3Client({\n      endpoint: config.LOG_STORAGE_S3_ENDPOINT,\n      region: config.LOG_STORAGE_S3_REGION,\n      forcePathStyle: config.LOG_STORAGE_S3_FORCE_PATH_STYLE,\n      ...(explicitS3Credentials ? {credentials: explicitS3Credentials} : {}),\n      // Fail fast: a slow or black-holed endpoint must not hang callers behind\n      // SDK backoff. One retry, short connect/request timeouts.\n      maxAttempts: 2,\n      requestHandler: {connectionTimeout: 1_000, requestTimeout: 3_000},\n    });\n  }\n  return _client;\n}\n\nexport function closeS3Client(): void {\n  _client?.destroy();\n  _client = undefined;\n}\n\n/** Readiness probe: true when the configured bucket answers a HEAD. Used by the app readiness check, not at process boot. */\nexport async function checkBucketReachable(): Promise<boolean> {\n  try {\n    await s3Client().send(new HeadBucketCommand({Bucket: config.LOG_STORAGE_S3_BUCKET}));\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nlet _uploadClient: S3Client | undefined;\n\n/**\n * S3 client for streaming compaction uploads. `s3Client()` is tuned fail-fast for the\n * readiness HEAD, but each multipart part upload can take as long as the link needs, so\n * the per-request timeout is disabled here. Temporal owns the retry budget, so the SDK\n * keeps only a small attempt count for transient connection blips.\n */\nexport function uploadS3Client(): S3Client {\n  if (!_uploadClient) {\n    _uploadClient = new S3Client({\n      endpoint: config.LOG_STORAGE_S3_ENDPOINT,\n      region: config.LOG_STORAGE_S3_REGION,\n      forcePathStyle: config.LOG_STORAGE_S3_FORCE_PATH_STYLE,\n      ...(explicitS3Credentials ? {credentials: explicitS3Credentials} : {}),\n      maxAttempts: 3,\n      requestHandler: {connectionTimeout: 5_000, requestTimeout: 0},\n    });\n  }\n  return _uploadClient;\n}\n\nexport function closeUploadS3Client(): void {\n  _uploadClient?.destroy();\n  _uploadClient = undefined;\n}\n\n/**\n * Per-attempt object key for a compacted stream: the stream's stable prefix\n * (`logObjectKey` -> `{prefix}/{workspaceId}/{jobId}/{stepId}/{attempt}`, shared with retention\n * and workspace prefix deletes) plus a unique `uploadToken` leaf. Each compaction attempt\n * uploads to its own key and the winner records it atomically, so a slow or zombie attempt\n * can never overwrite a published object. Losing and crashed attempts leave a leaf under the\n * same prefix, reclaimed by the bucket's incomplete-multipart-abort rule and prefix-scoped\n * retention.\n */\nexport function compactedObjectKey(identity: LogObjectKeyParams, uploadToken: string): string {\n  return `${logObjectKey(config.LOG_STORAGE_S3_PREFIX, identity)}/${uploadToken}`;\n}\n\n// lib-storage defaults (5MB parts, queue of 4); we pin a small queue so peak buffer memory\n// stays bounded under concurrent compactions (queueSize x partSize per active upload).\nconst UPLOAD_PART_SIZE = 5 * 1024 * 1024;\nconst UPLOAD_QUEUE_SIZE = 2;\n\nexport interface PutCompactedObjectParams {\n  key: string;\n  body: Readable;\n  metadata: Record<string, string>;\n  signal?: AbortSignal;\n  onProgress?: () => void;\n}\n\n/**\n * Streams `body` (gzip-compressed NDJSON) to the compacted object via a multipart upload.\n * `Content-Encoding: gzip` so a browser-direct presigned GET auto-decompresses. Aborts the\n * multipart upload when `signal` fires, so a cancelled activity leaves no dangling parts.\n *\n * A hard crash (no graceful abort) can still strand incomplete multipart parts, which on\n * pay-per-storage providers (S3, R2) bill until aborted. The worker never sets bucket\n * policy, so the deployed bucket must carry an `AbortIncompleteMultipartUpload` lifecycle\n * rule. `dev/garage/bootstrap.sh` provisions it for local Garage; self-hosters set the same\n * rule on their bucket.\n */\nexport async function putCompactedObject(params: PutCompactedObjectParams): Promise<void> {\n  const upload = new Upload({\n    client: uploadS3Client(),\n    partSize: UPLOAD_PART_SIZE,\n    queueSize: UPLOAD_QUEUE_SIZE,\n    params: {\n      Bucket: config.LOG_STORAGE_S3_BUCKET,\n      Key: params.key,\n      Body: params.body,\n      ContentType: 'application/x-ndjson',\n      ContentEncoding: 'gzip',\n      Metadata: params.metadata,\n    },\n  });\n\n  if (params.onProgress) upload.on('httpUploadProgress', params.onProgress);\n  if (params.signal) {\n    // Subscribe before the aborted-check so a signal firing between them cannot be missed,\n    // then handle the already-aborted case eagerly (an already-aborted signal never fires 'abort').\n    params.signal.addEventListener(\n      'abort',\n      () => {\n        void upload.abort().catch((error) => {\n          logger().error(\n            {err: error, objectKey: params.key},\n            'Failed to abort multipart log upload',\n          );\n          reportError(error, {\n            boundary: 'logs.cleanup',\n            operation: 'abort-multipart-upload',\n            extra: {objectKey: params.key},\n          });\n        });\n      },\n      {once: true},\n    );\n    if (params.signal.aborted) {\n      await upload.abort().catch((error) => {\n        logger().error({err: error, objectKey: params.key}, 'Failed to abort multipart log upload');\n        reportError(error, {\n          boundary: 'logs.cleanup',\n          operation: 'abort-multipart-upload',\n          extra: {objectKey: params.key},\n        });\n      });\n      throw new Error('Compaction upload aborted before it started');\n    }\n  }\n\n  await upload.done();\n}\n\n/**\n * Deletes a compacted object. S3 `DeleteObject` is idempotent (a missing key is a success),\n * so retention and the publish orphan guard can call it freely without a pre-check.\n */\nexport async function deleteObject(key: string): Promise<void> {\n  await s3Client().send(new DeleteObjectCommand({Bucket: config.LOG_STORAGE_S3_BUCKET, Key: key}));\n}\n\n/**\n * Presigns a GET for a compacted log object so the browser can fetch it directly,\n * bypassing API egress. `getSignedUrl` only computes a signature locally (no network\n * call), so the fail-fast `s3Client()` is fine here. `expiresAt` mirrors the URL's\n * `LOG_READ_URL_TTL_SECONDS` lifetime so the caller can hand the client an absolute\n * deadline without re-deriving it.\n */\nexport async function presignedGetUrl(objectKey: string): Promise<{url: string; expiresAt: Date}> {\n  const ttlSeconds = config.LOG_READ_URL_TTL_SECONDS;\n  // Stamp the deadline from before signing: getSignedUrl bakes its expiry from the clock at\n  // sign time, so reading Date.now() after the await could report a deadline slightly past the\n  // real one. Computing it first keeps expiresAt a conservative (never-late) bound.\n  const expiresAt = new Date(Date.now() + ttlSeconds * 1000);\n  const url = await getSignedUrl(\n    s3Client(),\n    new GetObjectCommand({Bucket: config.LOG_STORAGE_S3_BUCKET, Key: objectKey}),\n    {expiresIn: ttlSeconds},\n  );\n  return {url, expiresAt};\n}\n\n/**\n * Deletes all objects under a per-attempt prefix. Retention deletes by prefix, not only the\n * recorded `object_key`, to reclaim orphan leaves from failed compaction attempts.\n *\n * Pass a trailing slash so attempt `1` never matches attempt `10`.\n */\nexport async function deleteObjectsByPrefix(prefix: string): Promise<void> {\n  if (prefix === '') {\n    throw new Error(\n      'deleteObjectsByPrefix refuses an empty prefix (it would target the whole bucket)',\n    );\n  }\n  const client = s3Client();\n  let continuationToken: string | undefined;\n  do {\n    const listed = await client.send(\n      new ListObjectsV2Command({\n        Bucket: config.LOG_STORAGE_S3_BUCKET,\n        Prefix: prefix,\n        ContinuationToken: continuationToken,\n      }),\n    );\n    const objects = (listed.Contents ?? []).flatMap((object) =>\n      object.Key ? [{Key: object.Key}] : [],\n    );\n    if (objects.length > 0) {\n      const deleted = await client.send(\n        new DeleteObjectsCommand({\n          Bucket: config.LOG_STORAGE_S3_BUCKET,\n          Delete: {Objects: objects, Quiet: true},\n        }),\n      );\n      if (deleted.Errors && deleted.Errors.length > 0) {\n        const [first] = deleted.Errors;\n        throw new Error(\n          `Failed to delete ${deleted.Errors.length} object(s) under ${prefix}: ${first?.Key} ${first?.Message}`,\n        );\n      }\n    }\n    continuationToken = listed.IsTruncated ? listed.NextContinuationToken : undefined;\n  } while (continuationToken);\n}\n"],"names":["DeleteObjectCommand","DeleteObjectsCommand","GetObjectCommand","HeadBucketCommand","ListObjectsV2Command","S3Client","Upload","getSignedUrl","reportError","logger","config","logObjectKey","explicitS3Credentials","LOG_STORAGE_S3_ACCESS_KEY_ID","LOG_STORAGE_S3_SECRET_ACCESS_KEY","accessKeyId","secretAccessKey","undefined","_client","s3Client","endpoint","LOG_STORAGE_S3_ENDPOINT","region","LOG_STORAGE_S3_REGION","forcePathStyle","LOG_STORAGE_S3_FORCE_PATH_STYLE","credentials","maxAttempts","requestHandler","connectionTimeout","requestTimeout","closeS3Client","destroy","checkBucketReachable","send","Bucket","LOG_STORAGE_S3_BUCKET","_uploadClient","uploadS3Client","closeUploadS3Client","compactedObjectKey","identity","uploadToken","LOG_STORAGE_S3_PREFIX","UPLOAD_PART_SIZE","UPLOAD_QUEUE_SIZE","putCompactedObject","params","upload","client","partSize","queueSize","Key","key","Body","body","ContentType","ContentEncoding","Metadata","metadata","onProgress","on","signal","addEventListener","abort","catch","error","err","objectKey","boundary","operation","extra","once","aborted","Error","done","deleteObject","presignedGetUrl","ttlSeconds","LOG_READ_URL_TTL_SECONDS","expiresAt","Date","now","url","expiresIn","deleteObjectsByPrefix","prefix","continuationToken","listed","Prefix","ContinuationToken","objects","Contents","flatMap","object","length","deleted","Delete","Objects","Quiet","Errors","first","Message","IsTruncated","NextContinuationToken"],"mappings":"AACA,SACEA,mBAAmB,EACnBC,oBAAoB,EACpBC,gBAAgB,EAChBC,iBAAiB,EACjBC,oBAAoB,EACpBC,QAAQ,QACH,qBAAqB;AAC5B,SAAQC,MAAM,QAAO,uBAAuB;AAC5C,SAAQC,YAAY,QAAO,gCAAgC;AAC3D,SAAQC,WAAW,QAAO,iCAAiC;AAC3D,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAiCC,YAAY,QAAO,+BAA+B;AAEnF,MAAMC,wBACJF,OAAOG,4BAA4B,IAAIH,OAAOI,gCAAgC,GAC1E;IACEC,aAAaL,OAAOG,4BAA4B;IAChDG,iBAAiBN,OAAOI,gCAAgC;AAC1D,IACAG;AAEN,IAAIC;AAEJ,kFAAkF,GAClF,OAAO,SAASC;IACd,IAAI,CAACD,SAAS;QACZA,UAAU,IAAIb,SAAS;YACrBe,UAAUV,OAAOW,uBAAuB;YACxCC,QAAQZ,OAAOa,qBAAqB;YACpCC,gBAAgBd,OAAOe,+BAA+B;YACtD,GAAIb,wBAAwB;gBAACc,aAAad;YAAqB,IAAI,CAAC,CAAC;YACrE,yEAAyE;YACzE,0DAA0D;YAC1De,aAAa;YACbC,gBAAgB;gBAACC,mBAAmB;gBAAOC,gBAAgB;YAAK;QAClE;IACF;IACA,OAAOZ;AACT;AAEA,OAAO,SAASa;IACdb,SAASc;IACTd,UAAUD;AACZ;AAEA,2HAA2H,GAC3H,OAAO,eAAegB;IACpB,IAAI;QACF,MAAMd,WAAWe,IAAI,CAAC,IAAI/B,kBAAkB;YAACgC,QAAQzB,OAAO0B,qBAAqB;QAAA;QACjF,OAAO;IACT,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,IAAIC;AAEJ;;;;;CAKC,GACD,OAAO,SAASC;IACd,IAAI,CAACD,eAAe;QAClBA,gBAAgB,IAAIhC,SAAS;YAC3Be,UAAUV,OAAOW,uBAAuB;YACxCC,QAAQZ,OAAOa,qBAAqB;YACpCC,gBAAgBd,OAAOe,+BAA+B;YACtD,GAAIb,wBAAwB;gBAACc,aAAad;YAAqB,IAAI,CAAC,CAAC;YACrEe,aAAa;YACbC,gBAAgB;gBAACC,mBAAmB;gBAAOC,gBAAgB;YAAC;QAC9D;IACF;IACA,OAAOO;AACT;AAEA,OAAO,SAASE;IACdF,eAAeL;IACfK,gBAAgBpB;AAClB;AAEA;;;;;;;;CAQC,GACD,OAAO,SAASuB,mBAAmBC,QAA4B,EAAEC,WAAmB;IAClF,OAAO,GAAG/B,aAAaD,OAAOiC,qBAAqB,EAAEF,UAAU,CAAC,EAAEC,aAAa;AACjF;AAEA,2FAA2F;AAC3F,uFAAuF;AACvF,MAAME,mBAAmB,IAAI,OAAO;AACpC,MAAMC,oBAAoB;AAU1B;;;;;;;;;;CAUC,GACD,OAAO,eAAeC,mBAAmBC,MAAgC;IACvE,MAAMC,SAAS,IAAI1C,OAAO;QACxB2C,QAAQX;QACRY,UAAUN;QACVO,WAAWN;QACXE,QAAQ;YACNZ,QAAQzB,OAAO0B,qBAAqB;YACpCgB,KAAKL,OAAOM,GAAG;YACfC,MAAMP,OAAOQ,IAAI;YACjBC,aAAa;YACbC,iBAAiB;YACjBC,UAAUX,OAAOY,QAAQ;QAC3B;IACF;IAEA,IAAIZ,OAAOa,UAAU,EAAEZ,OAAOa,EAAE,CAAC,sBAAsBd,OAAOa,UAAU;IACxE,IAAIb,OAAOe,MAAM,EAAE;QACjB,uFAAuF;QACvF,gGAAgG;QAChGf,OAAOe,MAAM,CAACC,gBAAgB,CAC5B,SACA;YACE,KAAKf,OAAOgB,KAAK,GAAGC,KAAK,CAAC,CAACC;gBACzBzD,SAASyD,KAAK,CACZ;oBAACC,KAAKD;oBAAOE,WAAWrB,OAAOM,GAAG;gBAAA,GAClC;gBAEF7C,YAAY0D,OAAO;oBACjBG,UAAU;oBACVC,WAAW;oBACXC,OAAO;wBAACH,WAAWrB,OAAOM,GAAG;oBAAA;gBAC/B;YACF;QACF,GACA;YAACmB,MAAM;QAAI;QAEb,IAAIzB,OAAOe,MAAM,CAACW,OAAO,EAAE;YACzB,MAAMzB,OAAOgB,KAAK,GAAGC,KAAK,CAAC,CAACC;gBAC1BzD,SAASyD,KAAK,CAAC;oBAACC,KAAKD;oBAAOE,WAAWrB,OAAOM,GAAG;gBAAA,GAAG;gBACpD7C,YAAY0D,OAAO;oBACjBG,UAAU;oBACVC,WAAW;oBACXC,OAAO;wBAACH,WAAWrB,OAAOM,GAAG;oBAAA;gBAC/B;YACF;YACA,MAAM,IAAIqB,MAAM;QAClB;IACF;IAEA,MAAM1B,OAAO2B,IAAI;AACnB;AAEA;;;CAGC,GACD,OAAO,eAAeC,aAAavB,GAAW;IAC5C,MAAMlC,WAAWe,IAAI,CAAC,IAAIlC,oBAAoB;QAACmC,QAAQzB,OAAO0B,qBAAqB;QAAEgB,KAAKC;IAAG;AAC/F;AAEA;;;;;;CAMC,GACD,OAAO,eAAewB,gBAAgBT,SAAiB;IACrD,MAAMU,aAAapE,OAAOqE,wBAAwB;IAClD,0FAA0F;IAC1F,6FAA6F;IAC7F,kFAAkF;IAClF,MAAMC,YAAY,IAAIC,KAAKA,KAAKC,GAAG,KAAKJ,aAAa;IACrD,MAAMK,MAAM,MAAM5E,aAChBY,YACA,IAAIjB,iBAAiB;QAACiC,QAAQzB,OAAO0B,qBAAqB;QAAEgB,KAAKgB;IAAS,IAC1E;QAACgB,WAAWN;IAAU;IAExB,OAAO;QAACK;QAAKH;IAAS;AACxB;AAEA;;;;;CAKC,GACD,OAAO,eAAeK,sBAAsBC,MAAc;IACxD,IAAIA,WAAW,IAAI;QACjB,MAAM,IAAIZ,MACR;IAEJ;IACA,MAAMzB,SAAS9B;IACf,IAAIoE;IACJ,GAAG;QACD,MAAMC,SAAS,MAAMvC,OAAOf,IAAI,CAC9B,IAAI9B,qBAAqB;YACvB+B,QAAQzB,OAAO0B,qBAAqB;YACpCqD,QAAQH;YACRI,mBAAmBH;QACrB;QAEF,MAAMI,UAAU,AAACH,CAAAA,OAAOI,QAAQ,IAAI,EAAE,AAAD,EAAGC,OAAO,CAAC,CAACC,SAC/CA,OAAO1C,GAAG,GAAG;gBAAC;oBAACA,KAAK0C,OAAO1C,GAAG;gBAAA;aAAE,GAAG,EAAE;QAEvC,IAAIuC,QAAQI,MAAM,GAAG,GAAG;YACtB,MAAMC,UAAU,MAAM/C,OAAOf,IAAI,CAC/B,IAAIjC,qBAAqB;gBACvBkC,QAAQzB,OAAO0B,qBAAqB;gBACpC6D,QAAQ;oBAACC,SAASP;oBAASQ,OAAO;gBAAI;YACxC;YAEF,IAAIH,QAAQI,MAAM,IAAIJ,QAAQI,MAAM,CAACL,MAAM,GAAG,GAAG;gBAC/C,MAAM,CAACM,MAAM,GAAGL,QAAQI,MAAM;gBAC9B,MAAM,IAAI1B,MACR,CAAC,iBAAiB,EAAEsB,QAAQI,MAAM,CAACL,MAAM,CAAC,iBAAiB,EAAET,OAAO,EAAE,EAAEe,OAAOjD,IAAI,CAAC,EAAEiD,OAAOC,SAAS;YAE1G;QACF;QACAf,oBAAoBC,OAAOe,WAAW,GAAGf,OAAOgB,qBAAqB,GAAGvF;IAC1E,QAASsE,kBAAmB;AAC9B"}

package/dist/core/reap-stale-open-streams.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reap-stale-open-streams.d.ts","sourceRoot":"","sources":["../../src/core/reap-stale-open-streams.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,0BAA0B;IACzC,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,0BAA0B;IACzC;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,0BAA0B,GACjC,OAAO,CAAC,0BAA0B,CAAC,CAwBrC"}
+{"version":3,"file":"reap-stale-open-streams.d.ts","sourceRoot":"","sources":["../../src/core/reap-stale-open-streams.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,0BAA0B;IACzC,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,0BAA0B;IACzC;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,0BAA0B,GACjC,OAAO,CAAC,0BAA0B,CAAC,CAyBrC"}

package/dist/core/reap-stale-open-streams.js

@@ -1,3 +1,4 @@
+import { reportError } from '@shipfox/node-error-monitoring';
 import { logger } from '@shipfox/node-opentelemetry';
 import { db } from '#db/db.js';
 import { listStaleOpenStreams } from '#db/streams.js';
@@ -42,6 +43,12 @@ import { closeStream } from './close-stream.js';
                 err: error,
                 streamId: stream.id
             }, 'Failed to reap stale open log stream');
+            reportError(error, {
+                boundary: 'logs.maintenance',
+                extra: {
+                    streamId: stream.id
+                }
+            });
         }
     }
     return result;

package/dist/core/reap-stale-open-streams.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/core/reap-stale-open-streams.ts"],"sourcesContent":["import {logger} from '@shipfox/node-opentelemetry';\nimport {db} from '#db/db.js';\nimport {listStaleOpenStreams} from '#db/streams.js';\nimport {recordAppendedCount, streamClosedCount} from '#metrics/instance.js';\nimport {closeStream} from './close-stream.js';\n\nexport interface ReapStaleOpenStreamsResult {\n  /** Streams force-closed by this run. */\n  reaped: number;\n  /** Streams whose guarded close threw; logged, skipped, retried next run. */\n  failed: number;\n}\n\nexport interface ReapStaleOpenStreamsParams {\n  /**\n   * A stream open longer than this since `created_at` can no longer be appended to: its job\n   * holds one lease minted at-or-before the stream's creation, so once the lease TTL has\n   * elapsed no append can land and the close is safe. Set above `AUTH_JOB_LEASE_TOKEN_EXPIRES_IN`.\n   */\n  olderThanSeconds: number;\n  batchLimit: number;\n}\n\n/**\n * Force-closes open streams the one-shot job-terminal sweep missed: a stream whose first append\n * landed after that sweep's snapshot would otherwise stay `open` forever, invisible to\n * compaction and retention (both partial on `state = 'closed'`). Reusing the guarded\n * `closeStream` re-drives each into the normal closed -> compact -> retention lifecycle.\n *\n * Each row closes in its own transaction, so a tick/retry overlap is idempotent (the guarded\n * `markStreamClosed` lets only one close win) and one failed row is logged and skipped instead\n * of aborting the batch.\n */\nexport async function reapStaleOpenStreams(\n  params: ReapStaleOpenStreamsParams,\n): Promise<ReapStaleOpenStreamsResult> {\n  const stale = await listStaleOpenStreams({\n    olderThanSeconds: params.olderThanSeconds,\n    limit: params.batchLimit,\n  });\n\n  const result: ReapStaleOpenStreamsResult = {reaped: 0, failed: 0};\n  for (const stream of stale) {\n    try {\n      const closed = await db().transaction((tx) =>\n        closeStream(tx, {streamId: stream.id, reason: 'timeout'}),\n      );\n      if (closed) {\n        result.reaped += 1;\n        recordAppendedCount.add(1, {kind: 'runner_lost'});\n        streamClosedCount.add(1, {reason: 'timeout'});\n      }\n    } catch (error) {\n      result.failed += 1;\n      logger().error({err: error, streamId: stream.id}, 'Failed to reap stale open log stream');\n    }\n  }\n\n  return result;\n}\n"],"names":["logger","db","listStaleOpenStreams","recordAppendedCount","streamClosedCount","closeStream","reapStaleOpenStreams","params","stale","olderThanSeconds","limit","batchLimit","result","reaped","failed","stream","closed","transaction","tx","streamId","id","reason","add","kind","error","err"],"mappings":"AAAA,SAAQA,MAAM,QAAO,8BAA8B;AACnD,SAAQC,EAAE,QAAO,YAAY;AAC7B,SAAQC,oBAAoB,QAAO,iBAAiB;AACpD,SAAQC,mBAAmB,EAAEC,iBAAiB,QAAO,uBAAuB;AAC5E,SAAQC,WAAW,QAAO,oBAAoB;AAmB9C;;;;;;;;;CASC,GACD,OAAO,eAAeC,qBACpBC,MAAkC;IAElC,MAAMC,QAAQ,MAAMN,qBAAqB;QACvCO,kBAAkBF,OAAOE,gBAAgB;QACzCC,OAAOH,OAAOI,UAAU;IAC1B;IAEA,MAAMC,SAAqC;QAACC,QAAQ;QAAGC,QAAQ;IAAC;IAChE,KAAK,MAAMC,UAAUP,MAAO;QAC1B,IAAI;YACF,MAAMQ,SAAS,MAAMf,KAAKgB,WAAW,CAAC,CAACC,KACrCb,YAAYa,IAAI;oBAACC,UAAUJ,OAAOK,EAAE;oBAAEC,QAAQ;gBAAS;YAEzD,IAAIL,QAAQ;gBACVJ,OAAOC,MAAM,IAAI;gBACjBV,oBAAoBmB,GAAG,CAAC,GAAG;oBAACC,MAAM;gBAAa;gBAC/CnB,kBAAkBkB,GAAG,CAAC,GAAG;oBAACD,QAAQ;gBAAS;YAC7C;QACF,EAAE,OAAOG,OAAO;YACdZ,OAAOE,MAAM,IAAI;YACjBd,SAASwB,KAAK,CAAC;gBAACC,KAAKD;gBAAOL,UAAUJ,OAAOK,EAAE;YAAA,GAAG;QACpD;IACF;IAEA,OAAOR;AACT"}
+{"version":3,"sources":["../../src/core/reap-stale-open-streams.ts"],"sourcesContent":["import {reportError} from '@shipfox/node-error-monitoring';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {db} from '#db/db.js';\nimport {listStaleOpenStreams} from '#db/streams.js';\nimport {recordAppendedCount, streamClosedCount} from '#metrics/instance.js';\nimport {closeStream} from './close-stream.js';\n\nexport interface ReapStaleOpenStreamsResult {\n  /** Streams force-closed by this run. */\n  reaped: number;\n  /** Streams whose guarded close threw; logged, skipped, retried next run. */\n  failed: number;\n}\n\nexport interface ReapStaleOpenStreamsParams {\n  /**\n   * A stream open longer than this since `created_at` can no longer be appended to: its job\n   * holds one lease minted at-or-before the stream's creation, so once the lease TTL has\n   * elapsed no append can land and the close is safe. Set above `AUTH_JOB_LEASE_TOKEN_EXPIRES_IN`.\n   */\n  olderThanSeconds: number;\n  batchLimit: number;\n}\n\n/**\n * Force-closes open streams the one-shot job-terminal sweep missed: a stream whose first append\n * landed after that sweep's snapshot would otherwise stay `open` forever, invisible to\n * compaction and retention (both partial on `state = 'closed'`). Reusing the guarded\n * `closeStream` re-drives each into the normal closed -> compact -> retention lifecycle.\n *\n * Each row closes in its own transaction, so a tick/retry overlap is idempotent (the guarded\n * `markStreamClosed` lets only one close win) and one failed row is logged and skipped instead\n * of aborting the batch.\n */\nexport async function reapStaleOpenStreams(\n  params: ReapStaleOpenStreamsParams,\n): Promise<ReapStaleOpenStreamsResult> {\n  const stale = await listStaleOpenStreams({\n    olderThanSeconds: params.olderThanSeconds,\n    limit: params.batchLimit,\n  });\n\n  const result: ReapStaleOpenStreamsResult = {reaped: 0, failed: 0};\n  for (const stream of stale) {\n    try {\n      const closed = await db().transaction((tx) =>\n        closeStream(tx, {streamId: stream.id, reason: 'timeout'}),\n      );\n      if (closed) {\n        result.reaped += 1;\n        recordAppendedCount.add(1, {kind: 'runner_lost'});\n        streamClosedCount.add(1, {reason: 'timeout'});\n      }\n    } catch (error) {\n      result.failed += 1;\n      logger().error({err: error, streamId: stream.id}, 'Failed to reap stale open log stream');\n      reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});\n    }\n  }\n\n  return result;\n}\n"],"names":["reportError","logger","db","listStaleOpenStreams","recordAppendedCount","streamClosedCount","closeStream","reapStaleOpenStreams","params","stale","olderThanSeconds","limit","batchLimit","result","reaped","failed","stream","closed","transaction","tx","streamId","id","reason","add","kind","error","err","boundary","extra"],"mappings":"AAAA,SAAQA,WAAW,QAAO,iCAAiC;AAC3D,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,EAAE,QAAO,YAAY;AAC7B,SAAQC,oBAAoB,QAAO,iBAAiB;AACpD,SAAQC,mBAAmB,EAAEC,iBAAiB,QAAO,uBAAuB;AAC5E,SAAQC,WAAW,QAAO,oBAAoB;AAmB9C;;;;;;;;;CASC,GACD,OAAO,eAAeC,qBACpBC,MAAkC;IAElC,MAAMC,QAAQ,MAAMN,qBAAqB;QACvCO,kBAAkBF,OAAOE,gBAAgB;QACzCC,OAAOH,OAAOI,UAAU;IAC1B;IAEA,MAAMC,SAAqC;QAACC,QAAQ;QAAGC,QAAQ;IAAC;IAChE,KAAK,MAAMC,UAAUP,MAAO;QAC1B,IAAI;YACF,MAAMQ,SAAS,MAAMf,KAAKgB,WAAW,CAAC,CAACC,KACrCb,YAAYa,IAAI;oBAACC,UAAUJ,OAAOK,EAAE;oBAAEC,QAAQ;gBAAS;YAEzD,IAAIL,QAAQ;gBACVJ,OAAOC,MAAM,IAAI;gBACjBV,oBAAoBmB,GAAG,CAAC,GAAG;oBAACC,MAAM;gBAAa;gBAC/CnB,kBAAkBkB,GAAG,CAAC,GAAG;oBAACD,QAAQ;gBAAS;YAC7C;QACF,EAAE,OAAOG,OAAO;YACdZ,OAAOE,MAAM,IAAI;YACjBd,SAASwB,KAAK,CAAC;gBAACC,KAAKD;gBAAOL,UAAUJ,OAAOK,EAAE;YAAA,GAAG;YAClDrB,YAAYyB,OAAO;gBAACE,UAAU;gBAAoBC,OAAO;oBAACR,UAAUJ,OAAOK,EAAE;gBAAA;YAAC;QAChF;IACF;IAEA,OAAOR;AACT"}

package/dist/core/retention.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"retention.d.ts","sourceRoot":"","sources":["../../src/core/retention.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,oBAAoB;IACnC,kDAAkD;IAClD,OAAO,EAAE,MAAM,CAAC;IAChB,mGAAmG;IACnG,KAAK,EAAE,MAAM,CAAC;IACd,uGAAuG;IACvG,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,0FAA0F;IAC1F,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,mFAAmF;IACnF,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,wFAAwF;IACxF,UAAU,CAAC,EAAE,MAAM,IAAI,CAAC;CACzB;AA0BD;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,oBAAoB,CAAC,CA+G/B"}
+{"version":3,"file":"retention.d.ts","sourceRoot":"","sources":["../../src/core/retention.ts"],"names":[],"mappings":"AAeA,MAAM,WAAW,oBAAoB;IACnC,kDAAkD;IAClD,OAAO,EAAE,MAAM,CAAC;IAChB,mGAAmG;IACnG,KAAK,EAAE,MAAM,CAAC;IACd,uGAAuG;IACvG,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,0FAA0F;IAC1F,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,mFAAmF;IACnF,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,wFAAwF;IACxF,UAAU,CAAC,EAAE,MAAM,IAAI,CAAC;CACzB;AA0BD;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,oBAAoB,CAAC,CAmH/B"}

package/dist/core/retention.js

@@ -1,3 +1,4 @@
+import { reportError } from '@shipfox/node-error-monitoring';
 import { logger } from '@shipfox/node-opentelemetry';
 import { deleteObjectsByPrefix } from '#api/object-storage.js';
 import { config } from '#config.js';
@@ -81,6 +82,12 @@ function deleteExpiredStreamRow(params) {
                     err: error,
                     streamId: stream.id
                 }, 'Failed to delete expired log stream objects');
+                reportError(error, {
+                    boundary: 'logs.maintenance',
+                    extra: {
+                        streamId: stream.id
+                    }
+                });
                 continue;
             }
             let outcome;
@@ -96,6 +103,12 @@ function deleteExpiredStreamRow(params) {
                     err: error,
                     streamId: stream.id
                 }, 'Failed to delete expired log stream row');
+                reportError(error, {
+                    boundary: 'logs.maintenance',
+                    extra: {
+                        streamId: stream.id
+                    }
+                });
                 continue;
             }
             if (!outcome.deleted) {
@@ -109,6 +122,12 @@ function deleteExpiredStreamRow(params) {
                         err: error,
                         streamId: stream.id
                     }, 'Failed to reload raced expired log stream');
+                    reportError(error, {
+                        boundary: 'logs.maintenance',
+                        extra: {
+                            streamId: stream.id
+                        }
+                    });
                     continue;
                 }
                 if (current) {
@@ -125,6 +144,12 @@ function deleteExpiredStreamRow(params) {
                             err: error,
                             streamId: stream.id
                         }, 'Failed to delete raced expired log stream');
+                        reportError(error, {
+                            boundary: 'logs.maintenance',
+                            extra: {
+                                streamId: stream.id
+                            }
+                        });
                         continue;
                     }
                 }

package/dist/core/retention.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/core/retention.ts"],"sourcesContent":["import {logger} from '@shipfox/node-opentelemetry';\nimport {deleteObjectsByPrefix} from '#api/object-storage.js';\nimport {config} from '#config.js';\nimport type {AttemptStream} from '#core/entities/attempt-stream.js';\nimport {logObjectKey} from '#core/entities/log-object.js';\nimport {deleteJobAccounting} from '#db/accounting.js';\nimport {db} from '#db/db.js';\nimport {\n  accountingHasNoStreams,\n  deleteExpiredStream,\n  getAttemptStreamById,\n  listExpiredClosedStreams,\n} from '#db/streams.js';\n\nexport interface RetentionSweepResult {\n  /** Streams whose objects and row were deleted. */\n  deleted: number;\n  /** Streams skipped because compaction changed `object_key` after we read it (retried next run). */\n  raced: number;\n  /** Streams whose object cleanup or guarded row delete threw; logged, skipped, and retried next run. */\n  failed: number;\n  accountingPruned: number;\n  iterations: number;\n  /** True when the sweep stopped on its wall-clock budget with backlog likely remaining. */\n  timedOut: boolean;\n}\n\nexport interface RunRetentionSweepParams {\n  retentionDays: number;\n  batchLimit: number;\n  timeBudgetMs: number;\n  maxIterations: number;\n  /** Wall clock; injectable so tests can drive the time budget deterministically. */\n  now?: () => number;\n  /** Liveness signal (e.g. the activity heartbeat); invoked once per processed stream. */\n  onProgress?: () => void;\n}\n\nfunction deleteExpiredStreamObjects(stream: AttemptStream): Promise<void> {\n  return deleteObjectsByPrefix(`${logObjectKey(config.LOG_STORAGE_S3_PREFIX, stream)}/`);\n}\n\nfunction deleteExpiredStreamRow(params: {\n  stream: AttemptStream;\n  retentionDays: number;\n}): Promise<{deleted: boolean; prunedAccounting: boolean}> {\n  return db().transaction(async (tx) => {\n    const {deleted, jobId} = await deleteExpiredStream(tx, {\n      streamId: params.stream.id,\n      observedObjectKey: params.stream.objectKey,\n    });\n    if (deleted && jobId && (await accountingHasNoStreams(tx, jobId))) {\n      const pruned = await deleteJobAccounting(tx, {\n        jobId,\n        retentionDays: params.retentionDays,\n      });\n      return {deleted, prunedAccounting: pruned.deleted};\n    }\n    return {deleted, prunedAccounting: false};\n  });\n}\n\n/**\n * Deletes expired closed streams and prunes accounting for emptied jobs.\n *\n * The loop self-bounds because a Temporal `startToCloseTimeout` marks the activity failed but\n * does not stop already-running JS. Objects are deleted before rows so a cleanup failure leaves\n * the row discoverable for the next sweep; the row delete stays guarded on the observed\n * `object_key`, so a racing compaction publish is re-read before the row is removed.\n */\nexport async function runRetentionSweep(\n  params: RunRetentionSweepParams,\n): Promise<RetentionSweepResult> {\n  const now = params.now ?? Date.now;\n  const deadline = now() + params.timeBudgetMs;\n  const result: RetentionSweepResult = {\n    deleted: 0,\n    raced: 0,\n    failed: 0,\n    accountingPruned: 0,\n    iterations: 0,\n    timedOut: false,\n  };\n\n  // Failed or raced rows retry next run; skipping them here keeps the rest of the backlog moving.\n  const skip = new Set<string>();\n  while (result.iterations < params.maxIterations) {\n    if (now() >= deadline) {\n      result.timedOut = true;\n      break;\n    }\n\n    const batch = await listExpiredClosedStreams({\n      retentionDays: params.retentionDays,\n      limit: params.batchLimit,\n      excludeIds: skip.size > 0 ? [...skip] : undefined,\n    });\n    if (batch.length === 0) break;\n\n    let timedOutMidBatch = false;\n    for (const stream of batch) {\n      // Per-stream checks keep long batches within the heartbeat and wall-clock budgets.\n      params.onProgress?.();\n      if (now() >= deadline) {\n        result.timedOut = true;\n        timedOutMidBatch = true;\n        break;\n      }\n\n      try {\n        await deleteExpiredStreamObjects(stream);\n      } catch (error) {\n        result.failed += 1;\n        skip.add(stream.id);\n        logger().error(\n          {err: error, streamId: stream.id},\n          'Failed to delete expired log stream objects',\n        );\n        continue;\n      }\n\n      let outcome: {deleted: boolean; prunedAccounting: boolean};\n      try {\n        outcome = await deleteExpiredStreamRow({stream, retentionDays: params.retentionDays});\n      } catch (error) {\n        result.failed += 1;\n        skip.add(stream.id);\n        logger().error(\n          {err: error, streamId: stream.id},\n          'Failed to delete expired log stream row',\n        );\n        continue;\n      }\n\n      if (!outcome.deleted) {\n        let current: AttemptStream | null;\n        try {\n          current = await getAttemptStreamById(stream.id);\n        } catch (error) {\n          result.failed += 1;\n          skip.add(stream.id);\n          logger().error(\n            {err: error, streamId: stream.id},\n            'Failed to reload raced expired log stream',\n          );\n          continue;\n        }\n        if (current) {\n          try {\n            await deleteExpiredStreamObjects(current);\n            outcome = await deleteExpiredStreamRow({\n              stream: current,\n              retentionDays: params.retentionDays,\n            });\n          } catch (error) {\n            result.failed += 1;\n            skip.add(stream.id);\n            logger().error(\n              {err: error, streamId: stream.id},\n              'Failed to delete raced expired log stream',\n            );\n            continue;\n          }\n        }\n        if (!current || !outcome.deleted) {\n          // `object_key` changed again or the row disappeared; the next sweep will re-read it.\n          result.raced += 1;\n          skip.add(stream.id);\n          continue;\n        }\n      }\n\n      result.deleted += 1;\n      if (outcome.prunedAccounting) result.accountingPruned += 1;\n    }\n\n    if (timedOutMidBatch) break;\n\n    result.iterations += 1;\n    if (batch.length < params.batchLimit) break;\n  }\n\n  return result;\n}\n"],"names":["logger","deleteObjectsByPrefix","config","logObjectKey","deleteJobAccounting","db","accountingHasNoStreams","deleteExpiredStream","getAttemptStreamById","listExpiredClosedStreams","deleteExpiredStreamObjects","stream","LOG_STORAGE_S3_PREFIX","deleteExpiredStreamRow","params","transaction","tx","deleted","jobId","streamId","id","observedObjectKey","objectKey","pruned","retentionDays","prunedAccounting","runRetentionSweep","now","Date","deadline","timeBudgetMs","result","raced","failed","accountingPruned","iterations","timedOut","skip","Set","maxIterations","batch","limit","batchLimit","excludeIds","size","undefined","length","timedOutMidBatch","onProgress","error","add","err","outcome","current"],"mappings":"AAAA,SAAQA,MAAM,QAAO,8BAA8B;AACnD,SAAQC,qBAAqB,QAAO,yBAAyB;AAC7D,SAAQC,MAAM,QAAO,aAAa;AAElC,SAAQC,YAAY,QAAO,+BAA+B;AAC1D,SAAQC,mBAAmB,QAAO,oBAAoB;AACtD,SAAQC,EAAE,QAAO,YAAY;AAC7B,SACEC,sBAAsB,EACtBC,mBAAmB,EACnBC,oBAAoB,EACpBC,wBAAwB,QACnB,iBAAiB;AA0BxB,SAASC,2BAA2BC,MAAqB;IACvD,OAAOV,sBAAsB,GAAGE,aAAaD,OAAOU,qBAAqB,EAAED,QAAQ,CAAC,CAAC;AACvF;AAEA,SAASE,uBAAuBC,MAG/B;IACC,OAAOT,KAAKU,WAAW,CAAC,OAAOC;QAC7B,MAAM,EAACC,OAAO,EAAEC,KAAK,EAAC,GAAG,MAAMX,oBAAoBS,IAAI;YACrDG,UAAUL,OAAOH,MAAM,CAACS,EAAE;YAC1BC,mBAAmBP,OAAOH,MAAM,CAACW,SAAS;QAC5C;QACA,IAAIL,WAAWC,SAAU,MAAMZ,uBAAuBU,IAAIE,QAAS;YACjE,MAAMK,SAAS,MAAMnB,oBAAoBY,IAAI;gBAC3CE;gBACAM,eAAeV,OAAOU,aAAa;YACrC;YACA,OAAO;gBAACP;gBAASQ,kBAAkBF,OAAON,OAAO;YAAA;QACnD;QACA,OAAO;YAACA;YAASQ,kBAAkB;QAAK;IAC1C;AACF;AAEA;;;;;;;CAOC,GACD,OAAO,eAAeC,kBACpBZ,MAA+B;IAE/B,MAAMa,MAAMb,OAAOa,GAAG,IAAIC,KAAKD,GAAG;IAClC,MAAME,WAAWF,QAAQb,OAAOgB,YAAY;IAC5C,MAAMC,SAA+B;QACnCd,SAAS;QACTe,OAAO;QACPC,QAAQ;QACRC,kBAAkB;QAClBC,YAAY;QACZC,UAAU;IACZ;IAEA,gGAAgG;IAChG,MAAMC,OAAO,IAAIC;IACjB,MAAOP,OAAOI,UAAU,GAAGrB,OAAOyB,aAAa,CAAE;QAC/C,IAAIZ,SAASE,UAAU;YACrBE,OAAOK,QAAQ,GAAG;YAClB;QACF;QAEA,MAAMI,QAAQ,MAAM/B,yBAAyB;YAC3Ce,eAAeV,OAAOU,aAAa;YACnCiB,OAAO3B,OAAO4B,UAAU;YACxBC,YAAYN,KAAKO,IAAI,GAAG,IAAI;mBAAIP;aAAK,GAAGQ;QAC1C;QACA,IAAIL,MAAMM,MAAM,KAAK,GAAG;QAExB,IAAIC,mBAAmB;QACvB,KAAK,MAAMpC,UAAU6B,MAAO;YAC1B,mFAAmF;YACnF1B,OAAOkC,UAAU;YACjB,IAAIrB,SAASE,UAAU;gBACrBE,OAAOK,QAAQ,GAAG;gBAClBW,mBAAmB;gBACnB;YACF;YAEA,IAAI;gBACF,MAAMrC,2BAA2BC;YACnC,EAAE,OAAOsC,OAAO;gBACdlB,OAAOE,MAAM,IAAI;gBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;gBAClBpB,SAASiD,KAAK,CACZ;oBAACE,KAAKF;oBAAO9B,UAAUR,OAAOS,EAAE;gBAAA,GAChC;gBAEF;YACF;YAEA,IAAIgC;YACJ,IAAI;gBACFA,UAAU,MAAMvC,uBAAuB;oBAACF;oBAAQa,eAAeV,OAAOU,aAAa;gBAAA;YACrF,EAAE,OAAOyB,OAAO;gBACdlB,OAAOE,MAAM,IAAI;gBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;gBAClBpB,SAASiD,KAAK,CACZ;oBAACE,KAAKF;oBAAO9B,UAAUR,OAAOS,EAAE;gBAAA,GAChC;gBAEF;YACF;YAEA,IAAI,CAACgC,QAAQnC,OAAO,EAAE;gBACpB,IAAIoC;gBACJ,IAAI;oBACFA,UAAU,MAAM7C,qBAAqBG,OAAOS,EAAE;gBAChD,EAAE,OAAO6B,OAAO;oBACdlB,OAAOE,MAAM,IAAI;oBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;oBAClBpB,SAASiD,KAAK,CACZ;wBAACE,KAAKF;wBAAO9B,UAAUR,OAAOS,EAAE;oBAAA,GAChC;oBAEF;gBACF;gBACA,IAAIiC,SAAS;oBACX,IAAI;wBACF,MAAM3C,2BAA2B2C;wBACjCD,UAAU,MAAMvC,uBAAuB;4BACrCF,QAAQ0C;4BACR7B,eAAeV,OAAOU,aAAa;wBACrC;oBACF,EAAE,OAAOyB,OAAO;wBACdlB,OAAOE,MAAM,IAAI;wBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;wBAClBpB,SAASiD,KAAK,CACZ;4BAACE,KAAKF;4BAAO9B,UAAUR,OAAOS,EAAE;wBAAA,GAChC;wBAEF;oBACF;gBACF;gBACA,IAAI,CAACiC,WAAW,CAACD,QAAQnC,OAAO,EAAE;oBAChC,qFAAqF;oBACrFc,OAAOC,KAAK,IAAI;oBAChBK,KAAKa,GAAG,CAACvC,OAAOS,EAAE;oBAClB;gBACF;YACF;YAEAW,OAAOd,OAAO,IAAI;YAClB,IAAImC,QAAQ3B,gBAAgB,EAAEM,OAAOG,gBAAgB,IAAI;QAC3D;QAEA,IAAIa,kBAAkB;QAEtBhB,OAAOI,UAAU,IAAI;QACrB,IAAIK,MAAMM,MAAM,GAAGhC,OAAO4B,UAAU,EAAE;IACxC;IAEA,OAAOX;AACT"}
+{"version":3,"sources":["../../src/core/retention.ts"],"sourcesContent":["import {reportError} from '@shipfox/node-error-monitoring';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {deleteObjectsByPrefix} from '#api/object-storage.js';\nimport {config} from '#config.js';\nimport type {AttemptStream} from '#core/entities/attempt-stream.js';\nimport {logObjectKey} from '#core/entities/log-object.js';\nimport {deleteJobAccounting} from '#db/accounting.js';\nimport {db} from '#db/db.js';\nimport {\n  accountingHasNoStreams,\n  deleteExpiredStream,\n  getAttemptStreamById,\n  listExpiredClosedStreams,\n} from '#db/streams.js';\n\nexport interface RetentionSweepResult {\n  /** Streams whose objects and row were deleted. */\n  deleted: number;\n  /** Streams skipped because compaction changed `object_key` after we read it (retried next run). */\n  raced: number;\n  /** Streams whose object cleanup or guarded row delete threw; logged, skipped, and retried next run. */\n  failed: number;\n  accountingPruned: number;\n  iterations: number;\n  /** True when the sweep stopped on its wall-clock budget with backlog likely remaining. */\n  timedOut: boolean;\n}\n\nexport interface RunRetentionSweepParams {\n  retentionDays: number;\n  batchLimit: number;\n  timeBudgetMs: number;\n  maxIterations: number;\n  /** Wall clock; injectable so tests can drive the time budget deterministically. */\n  now?: () => number;\n  /** Liveness signal (e.g. the activity heartbeat); invoked once per processed stream. */\n  onProgress?: () => void;\n}\n\nfunction deleteExpiredStreamObjects(stream: AttemptStream): Promise<void> {\n  return deleteObjectsByPrefix(`${logObjectKey(config.LOG_STORAGE_S3_PREFIX, stream)}/`);\n}\n\nfunction deleteExpiredStreamRow(params: {\n  stream: AttemptStream;\n  retentionDays: number;\n}): Promise<{deleted: boolean; prunedAccounting: boolean}> {\n  return db().transaction(async (tx) => {\n    const {deleted, jobId} = await deleteExpiredStream(tx, {\n      streamId: params.stream.id,\n      observedObjectKey: params.stream.objectKey,\n    });\n    if (deleted && jobId && (await accountingHasNoStreams(tx, jobId))) {\n      const pruned = await deleteJobAccounting(tx, {\n        jobId,\n        retentionDays: params.retentionDays,\n      });\n      return {deleted, prunedAccounting: pruned.deleted};\n    }\n    return {deleted, prunedAccounting: false};\n  });\n}\n\n/**\n * Deletes expired closed streams and prunes accounting for emptied jobs.\n *\n * The loop self-bounds because a Temporal `startToCloseTimeout` marks the activity failed but\n * does not stop already-running JS. Objects are deleted before rows so a cleanup failure leaves\n * the row discoverable for the next sweep; the row delete stays guarded on the observed\n * `object_key`, so a racing compaction publish is re-read before the row is removed.\n */\nexport async function runRetentionSweep(\n  params: RunRetentionSweepParams,\n): Promise<RetentionSweepResult> {\n  const now = params.now ?? Date.now;\n  const deadline = now() + params.timeBudgetMs;\n  const result: RetentionSweepResult = {\n    deleted: 0,\n    raced: 0,\n    failed: 0,\n    accountingPruned: 0,\n    iterations: 0,\n    timedOut: false,\n  };\n\n  // Failed or raced rows retry next run; skipping them here keeps the rest of the backlog moving.\n  const skip = new Set<string>();\n  while (result.iterations < params.maxIterations) {\n    if (now() >= deadline) {\n      result.timedOut = true;\n      break;\n    }\n\n    const batch = await listExpiredClosedStreams({\n      retentionDays: params.retentionDays,\n      limit: params.batchLimit,\n      excludeIds: skip.size > 0 ? [...skip] : undefined,\n    });\n    if (batch.length === 0) break;\n\n    let timedOutMidBatch = false;\n    for (const stream of batch) {\n      // Per-stream checks keep long batches within the heartbeat and wall-clock budgets.\n      params.onProgress?.();\n      if (now() >= deadline) {\n        result.timedOut = true;\n        timedOutMidBatch = true;\n        break;\n      }\n\n      try {\n        await deleteExpiredStreamObjects(stream);\n      } catch (error) {\n        result.failed += 1;\n        skip.add(stream.id);\n        logger().error(\n          {err: error, streamId: stream.id},\n          'Failed to delete expired log stream objects',\n        );\n        reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});\n        continue;\n      }\n\n      let outcome: {deleted: boolean; prunedAccounting: boolean};\n      try {\n        outcome = await deleteExpiredStreamRow({stream, retentionDays: params.retentionDays});\n      } catch (error) {\n        result.failed += 1;\n        skip.add(stream.id);\n        logger().error(\n          {err: error, streamId: stream.id},\n          'Failed to delete expired log stream row',\n        );\n        reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});\n        continue;\n      }\n\n      if (!outcome.deleted) {\n        let current: AttemptStream | null;\n        try {\n          current = await getAttemptStreamById(stream.id);\n        } catch (error) {\n          result.failed += 1;\n          skip.add(stream.id);\n          logger().error(\n            {err: error, streamId: stream.id},\n            'Failed to reload raced expired log stream',\n          );\n          reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});\n          continue;\n        }\n        if (current) {\n          try {\n            await deleteExpiredStreamObjects(current);\n            outcome = await deleteExpiredStreamRow({\n              stream: current,\n              retentionDays: params.retentionDays,\n            });\n          } catch (error) {\n            result.failed += 1;\n            skip.add(stream.id);\n            logger().error(\n              {err: error, streamId: stream.id},\n              'Failed to delete raced expired log stream',\n            );\n            reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});\n            continue;\n          }\n        }\n        if (!current || !outcome.deleted) {\n          // `object_key` changed again or the row disappeared; the next sweep will re-read it.\n          result.raced += 1;\n          skip.add(stream.id);\n          continue;\n        }\n      }\n\n      result.deleted += 1;\n      if (outcome.prunedAccounting) result.accountingPruned += 1;\n    }\n\n    if (timedOutMidBatch) break;\n\n    result.iterations += 1;\n    if (batch.length < params.batchLimit) break;\n  }\n\n  return result;\n}\n"],"names":["reportError","logger","deleteObjectsByPrefix","config","logObjectKey","deleteJobAccounting","db","accountingHasNoStreams","deleteExpiredStream","getAttemptStreamById","listExpiredClosedStreams","deleteExpiredStreamObjects","stream","LOG_STORAGE_S3_PREFIX","deleteExpiredStreamRow","params","transaction","tx","deleted","jobId","streamId","id","observedObjectKey","objectKey","pruned","retentionDays","prunedAccounting","runRetentionSweep","now","Date","deadline","timeBudgetMs","result","raced","failed","accountingPruned","iterations","timedOut","skip","Set","maxIterations","batch","limit","batchLimit","excludeIds","size","undefined","length","timedOutMidBatch","onProgress","error","add","err","boundary","extra","outcome","current"],"mappings":"AAAA,SAAQA,WAAW,QAAO,iCAAiC;AAC3D,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,qBAAqB,QAAO,yBAAyB;AAC7D,SAAQC,MAAM,QAAO,aAAa;AAElC,SAAQC,YAAY,QAAO,+BAA+B;AAC1D,SAAQC,mBAAmB,QAAO,oBAAoB;AACtD,SAAQC,EAAE,QAAO,YAAY;AAC7B,SACEC,sBAAsB,EACtBC,mBAAmB,EACnBC,oBAAoB,EACpBC,wBAAwB,QACnB,iBAAiB;AA0BxB,SAASC,2BAA2BC,MAAqB;IACvD,OAAOV,sBAAsB,GAAGE,aAAaD,OAAOU,qBAAqB,EAAED,QAAQ,CAAC,CAAC;AACvF;AAEA,SAASE,uBAAuBC,MAG/B;IACC,OAAOT,KAAKU,WAAW,CAAC,OAAOC;QAC7B,MAAM,EAACC,OAAO,EAAEC,KAAK,EAAC,GAAG,MAAMX,oBAAoBS,IAAI;YACrDG,UAAUL,OAAOH,MAAM,CAACS,EAAE;YAC1BC,mBAAmBP,OAAOH,MAAM,CAACW,SAAS;QAC5C;QACA,IAAIL,WAAWC,SAAU,MAAMZ,uBAAuBU,IAAIE,QAAS;YACjE,MAAMK,SAAS,MAAMnB,oBAAoBY,IAAI;gBAC3CE;gBACAM,eAAeV,OAAOU,aAAa;YACrC;YACA,OAAO;gBAACP;gBAASQ,kBAAkBF,OAAON,OAAO;YAAA;QACnD;QACA,OAAO;YAACA;YAASQ,kBAAkB;QAAK;IAC1C;AACF;AAEA;;;;;;;CAOC,GACD,OAAO,eAAeC,kBACpBZ,MAA+B;IAE/B,MAAMa,MAAMb,OAAOa,GAAG,IAAIC,KAAKD,GAAG;IAClC,MAAME,WAAWF,QAAQb,OAAOgB,YAAY;IAC5C,MAAMC,SAA+B;QACnCd,SAAS;QACTe,OAAO;QACPC,QAAQ;QACRC,kBAAkB;QAClBC,YAAY;QACZC,UAAU;IACZ;IAEA,gGAAgG;IAChG,MAAMC,OAAO,IAAIC;IACjB,MAAOP,OAAOI,UAAU,GAAGrB,OAAOyB,aAAa,CAAE;QAC/C,IAAIZ,SAASE,UAAU;YACrBE,OAAOK,QAAQ,GAAG;YAClB;QACF;QAEA,MAAMI,QAAQ,MAAM/B,yBAAyB;YAC3Ce,eAAeV,OAAOU,aAAa;YACnCiB,OAAO3B,OAAO4B,UAAU;YACxBC,YAAYN,KAAKO,IAAI,GAAG,IAAI;mBAAIP;aAAK,GAAGQ;QAC1C;QACA,IAAIL,MAAMM,MAAM,KAAK,GAAG;QAExB,IAAIC,mBAAmB;QACvB,KAAK,MAAMpC,UAAU6B,MAAO;YAC1B,mFAAmF;YACnF1B,OAAOkC,UAAU;YACjB,IAAIrB,SAASE,UAAU;gBACrBE,OAAOK,QAAQ,GAAG;gBAClBW,mBAAmB;gBACnB;YACF;YAEA,IAAI;gBACF,MAAMrC,2BAA2BC;YACnC,EAAE,OAAOsC,OAAO;gBACdlB,OAAOE,MAAM,IAAI;gBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;gBAClBpB,SAASiD,KAAK,CACZ;oBAACE,KAAKF;oBAAO9B,UAAUR,OAAOS,EAAE;gBAAA,GAChC;gBAEFrB,YAAYkD,OAAO;oBAACG,UAAU;oBAAoBC,OAAO;wBAAClC,UAAUR,OAAOS,EAAE;oBAAA;gBAAC;gBAC9E;YACF;YAEA,IAAIkC;YACJ,IAAI;gBACFA,UAAU,MAAMzC,uBAAuB;oBAACF;oBAAQa,eAAeV,OAAOU,aAAa;gBAAA;YACrF,EAAE,OAAOyB,OAAO;gBACdlB,OAAOE,MAAM,IAAI;gBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;gBAClBpB,SAASiD,KAAK,CACZ;oBAACE,KAAKF;oBAAO9B,UAAUR,OAAOS,EAAE;gBAAA,GAChC;gBAEFrB,YAAYkD,OAAO;oBAACG,UAAU;oBAAoBC,OAAO;wBAAClC,UAAUR,OAAOS,EAAE;oBAAA;gBAAC;gBAC9E;YACF;YAEA,IAAI,CAACkC,QAAQrC,OAAO,EAAE;gBACpB,IAAIsC;gBACJ,IAAI;oBACFA,UAAU,MAAM/C,qBAAqBG,OAAOS,EAAE;gBAChD,EAAE,OAAO6B,OAAO;oBACdlB,OAAOE,MAAM,IAAI;oBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;oBAClBpB,SAASiD,KAAK,CACZ;wBAACE,KAAKF;wBAAO9B,UAAUR,OAAOS,EAAE;oBAAA,GAChC;oBAEFrB,YAAYkD,OAAO;wBAACG,UAAU;wBAAoBC,OAAO;4BAAClC,UAAUR,OAAOS,EAAE;wBAAA;oBAAC;oBAC9E;gBACF;gBACA,IAAImC,SAAS;oBACX,IAAI;wBACF,MAAM7C,2BAA2B6C;wBACjCD,UAAU,MAAMzC,uBAAuB;4BACrCF,QAAQ4C;4BACR/B,eAAeV,OAAOU,aAAa;wBACrC;oBACF,EAAE,OAAOyB,OAAO;wBACdlB,OAAOE,MAAM,IAAI;wBACjBI,KAAKa,GAAG,CAACvC,OAAOS,EAAE;wBAClBpB,SAASiD,KAAK,CACZ;4BAACE,KAAKF;4BAAO9B,UAAUR,OAAOS,EAAE;wBAAA,GAChC;wBAEFrB,YAAYkD,OAAO;4BAACG,UAAU;4BAAoBC,OAAO;gCAAClC,UAAUR,OAAOS,EAAE;4BAAA;wBAAC;wBAC9E;oBACF;gBACF;gBACA,IAAI,CAACmC,WAAW,CAACD,QAAQrC,OAAO,EAAE;oBAChC,qFAAqF;oBACrFc,OAAOC,KAAK,IAAI;oBAChBK,KAAKa,GAAG,CAACvC,OAAOS,EAAE;oBAClB;gBACF;YACF;YAEAW,OAAOd,OAAO,IAAI;YAClB,IAAIqC,QAAQ7B,gBAAgB,EAAEM,OAAOG,gBAAgB,IAAI;QAC3D;QAEA,IAAIa,kBAAkB;QAEtBhB,OAAOI,UAAU,IAAI;QACrB,IAAIK,MAAMM,MAAM,GAAGhC,OAAO4B,UAAU,EAAE;IACxC;IAEA,OAAOX;AACT"}

package/dist/temporal/activities/compact-stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compact-stream.d.ts","sourceRoot":"","sources":["../../../src/temporal/activities/compact-stream.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAK,KAAK,WAAW,EAAC,MAAM,WAAW,CAAC;AAI/C,MAAM,MAAM,mBAAmB,GAC3B;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GACjB;IAAC,OAAO,EAAE,mBAAmB,CAAA;CAAC,GAC9B;IAAC,OAAO,EAAE,YAAY,CAAA;CAAC,GACvB;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAC,GAC5B;IAAC,OAAO,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAA;CAAC,CAAC;AAE7F,UAAU,yBAAyB;IACjC,mBAAmB,EAAE,OAAO,mBAAmB,CAAC;IAChD,2BAA2B,EAAE,CAC3B,EAAE,EAAE,WAAW,EACf,MAAM,EAAE;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAC,KAC1C,OAAO,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAC,CAAC,CAAC;CAClC;AAsFD,wBAAgB,2BAA2B,CACzC,YAAY,GAAE,yBAA+C,GAC5D,CAAC,MAAM,EAAE;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,KAAK,OAAO,CAAC,mBAAmB,CAAC,CAW9D;AAED,eAAO,MAAM,qBAAqB,WAbtB;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,KAAK,OAAO,CAAC,mBAAmB,CAaI,CAAC"}
+{"version":3,"file":"compact-stream.d.ts","sourceRoot":"","sources":["../../../src/temporal/activities/compact-stream.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAK,KAAK,WAAW,EAAC,MAAM,WAAW,CAAC;AAI/C,MAAM,MAAM,mBAAmB,GAC3B;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GACjB;IAAC,OAAO,EAAE,mBAAmB,CAAA;CAAC,GAC9B;IAAC,OAAO,EAAE,YAAY,CAAA;CAAC,GACvB;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAC,GAC5B;IAAC,OAAO,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAA;CAAC,CAAC;AAE7F,UAAU,yBAAyB;IACjC,mBAAmB,EAAE,OAAO,mBAAmB,CAAC;IAChD,2BAA2B,EAAE,CAC3B,EAAE,EAAE,WAAW,EACf,MAAM,EAAE;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAC,KAC1C,OAAO,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAC,CAAC,CAAC;CAClC;AAgGD,wBAAgB,2BAA2B,CACzC,YAAY,GAAE,yBAA+C,GAC5D,CAAC,MAAM,EAAE;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,KAAK,OAAO,CAAC,mBAAmB,CAAC,CAW9D;AAED,eAAO,MAAM,qBAAqB,WAbtB;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,KAAK,OAAO,CAAC,mBAAmB,CAaI,CAAC"}

package/dist/temporal/activities/compact-stream.js

@@ -1,3 +1,5 @@
+import { reportError } from '@shipfox/node-error-monitoring';
+import { logger } from '@shipfox/node-opentelemetry';
 import { Context } from '@temporalio/activity';
 import { compactedObjectKey, deleteObject, putCompactedObject } from '#api/object-storage.js';
 import { compactedGzipStream } from '#core/compaction.js';
@@ -55,7 +57,21 @@ const defaultDependencies = {
         }
     });
     if (stats.chunkCount !== expected.count || stats.lastSeq !== expected.maxSeq || stats.uncompressedBytes !== expected.uncompressedBytes) {
-        await deleteObject(uploadKey).catch(()=>undefined);
+        await deleteObject(uploadKey).catch((error)=>{
+            logger().error({
+                err: error,
+                streamId: stream.id,
+                objectKey: uploadKey
+            }, 'Failed to delete invalid compacted log object');
+            reportError(error, {
+                boundary: 'logs.cleanup',
+                operation: 'delete-invalid-compaction-object',
+                extra: {
+                    streamId: stream.id,
+                    objectKey: uploadKey
+                }
+            });
+        });
         throw new Error(`Compaction integrity check failed for stream ${stream.id}: streamed ${stats.chunkCount} chunks / ${stats.uncompressedBytes} bytes up to seq ${stats.lastSeq}, table holds ${expected.count} / ${expected.uncompressedBytes} bytes up to seq ${expected.maxSeq}`);
     }
     const { updated } = await db().transaction((tx)=>dependencies.setObjectKeyAndDeleteChunks(tx, {

package/dist/temporal/activities/compact-stream.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/temporal/activities/compact-stream.ts"],"sourcesContent":["import {Context} from '@temporalio/activity';\nimport {compactedObjectKey, deleteObject, putCompactedObject} from '#api/object-storage.js';\nimport {compactedGzipStream} from '#core/compaction.js';\nimport {chunkStats} from '#db/chunks.js';\nimport {db, type Transaction} from '#db/db.js';\nimport {getAttemptStreamById, setObjectKeyAndDeleteChunks} from '#db/streams.js';\nimport {type CompactionMetricOutcome, compactionCount} from '#metrics/instance.js';\n\nexport type CompactStreamResult =\n  | {outcome: 'gone'}\n  | {outcome: 'already-compacted'}\n  | {outcome: 'superseded'}\n  | {outcome: 'retention-raced'}\n  | {outcome: 'compacted'; objectKey: string; chunkCount: number; uncompressedBytes: number};\n\ninterface CompactStreamDependencies {\n  compactedGzipStream: typeof compactedGzipStream;\n  setObjectKeyAndDeleteChunks: (\n    tx: Transaction,\n    params: {streamId: string; objectKey: string},\n  ) => Promise<{updated: boolean}>;\n}\n\nconst defaultDependencies: CompactStreamDependencies = {\n  compactedGzipStream,\n  setObjectKeyAndDeleteChunks,\n};\n\n/**\n * Compacts one closed stream into a single gzip NDJSON object, then deletes its chunk rows.\n *\n *   load stream ──┬─ gone ─────────────► no-op (retention raced)\n *                 ├─ object_key set ───► no-op (idempotent re-run)\n *                 └─ else: upload to a per-attempt key ─► gzip ─► multipart (abort-aware, heartbeat)\n *                          │\n *                 verify streamed count/maxSeq/bytes == table  (mismatch ─► delete upload, throw, retry)\n *                          │\n *                 tx: set object_key (state='closed' AND object_key IS NULL) + delete chunks\n *                          └─ 0 rows ─► delete this attempt's upload, then re-read the row:\n *                                         gone ─► retention raced · keyed ─► superseded by another attempt\n *\n * Each attempt uploads to its own `compactedObjectKey(stream, uuid)`, so a slow or zombie\n * attempt can never overwrite a published object. The single-winner publish (object_key set\n * + chunk delete, atomic) drops chunks only once a complete object is durable; the integrity\n * check (count, maxSeq, and byte total) guards a read bug from publishing a truncated object\n * before the only copy of the source is gone (S3 part checksums cover byte transfer).\n */\nasync function compactStream(\n  params: {streamId: string},\n  dependencies: CompactStreamDependencies,\n): Promise<CompactStreamResult> {\n  const stream = await getAttemptStreamById(params.streamId);\n  if (!stream) return {outcome: 'gone'};\n  if (stream.objectKey) return {outcome: 'already-compacted'};\n\n  const ctx = Context.current();\n  const uploadKey = compactedObjectKey(stream, crypto.randomUUID());\n  const expected = await chunkStats(stream.id);\n  const {body, stats} = dependencies.compactedGzipStream({\n    streamId: stream.id,\n    onPage: () => ctx.heartbeat(),\n  });\n\n  await putCompactedObject({\n    key: uploadKey,\n    body,\n    signal: ctx.cancellationSignal,\n    onProgress: () => ctx.heartbeat(),\n    metadata: {\n      stream_id: stream.id,\n      chunk_count: String(expected.count),\n      uncompressed_bytes: String(expected.uncompressedBytes),\n      last_seq: String(expected.maxSeq),\n    },\n  });\n\n  if (\n    stats.chunkCount !== expected.count ||\n    stats.lastSeq !== expected.maxSeq ||\n    stats.uncompressedBytes !== expected.uncompressedBytes\n  ) {\n    await deleteObject(uploadKey).catch(() => undefined);\n    throw new Error(\n      `Compaction integrity check failed for stream ${stream.id}: streamed ${stats.chunkCount} chunks / ${stats.uncompressedBytes} bytes up to seq ${stats.lastSeq}, table holds ${expected.count} / ${expected.uncompressedBytes} bytes up to seq ${expected.maxSeq}`,\n    );\n  }\n\n  const {updated} = await db().transaction((tx) =>\n    dependencies.setObjectKeyAndDeleteChunks(tx, {\n      streamId: stream.id,\n      objectKey: uploadKey,\n    }),\n  );\n  if (!updated) {\n    await deleteObject(uploadKey);\n    const current = await getAttemptStreamById(stream.id);\n    return {outcome: current ? 'superseded' : 'retention-raced'};\n  }\n\n  return {\n    outcome: 'compacted',\n    objectKey: uploadKey,\n    chunkCount: stats.chunkCount,\n    uncompressedBytes: stats.uncompressedBytes,\n  };\n}\n\nexport function createCompactStreamActivity(\n  dependencies: CompactStreamDependencies = defaultDependencies,\n): (params: {streamId: string}) => Promise<CompactStreamResult> {\n  return async (params) => {\n    let outcome: CompactionMetricOutcome = 'failed';\n    try {\n      const result = await compactStream(params, dependencies);\n      outcome = result.outcome;\n      return result;\n    } finally {\n      compactionCount.add(1, {outcome});\n    }\n  };\n}\n\nexport const compactStreamActivity = createCompactStreamActivity();\n"],"names":["Context","compactedObjectKey","deleteObject","putCompactedObject","compactedGzipStream","chunkStats","db","getAttemptStreamById","setObjectKeyAndDeleteChunks","compactionCount","defaultDependencies","compactStream","params","dependencies","stream","streamId","outcome","objectKey","ctx","current","uploadKey","crypto","randomUUID","expected","id","body","stats","onPage","heartbeat","key","signal","cancellationSignal","onProgress","metadata","stream_id","chunk_count","String","count","uncompressed_bytes","uncompressedBytes","last_seq","maxSeq","chunkCount","lastSeq","catch","undefined","Error","updated","transaction","tx","createCompactStreamActivity","result","add","compactStreamActivity"],"mappings":"AAAA,SAAQA,OAAO,QAAO,uBAAuB;AAC7C,SAAQC,kBAAkB,EAAEC,YAAY,EAAEC,kBAAkB,QAAO,yBAAyB;AAC5F,SAAQC,mBAAmB,QAAO,sBAAsB;AACxD,SAAQC,UAAU,QAAO,gBAAgB;AACzC,SAAQC,EAAE,QAAyB,YAAY;AAC/C,SAAQC,oBAAoB,EAAEC,2BAA2B,QAAO,iBAAiB;AACjF,SAAsCC,eAAe,QAAO,uBAAuB;AAiBnF,MAAMC,sBAAiD;IACrDN;IACAI;AACF;AAEA;;;;;;;;;;;;;;;;;;CAkBC,GACD,eAAeG,cACbC,MAA0B,EAC1BC,YAAuC;IAEvC,MAAMC,SAAS,MAAMP,qBAAqBK,OAAOG,QAAQ;IACzD,IAAI,CAACD,QAAQ,OAAO;QAACE,SAAS;IAAM;IACpC,IAAIF,OAAOG,SAAS,EAAE,OAAO;QAACD,SAAS;IAAmB;IAE1D,MAAME,MAAMlB,QAAQmB,OAAO;IAC3B,MAAMC,YAAYnB,mBAAmBa,QAAQO,OAAOC,UAAU;IAC9D,MAAMC,WAAW,MAAMlB,WAAWS,OAAOU,EAAE;IAC3C,MAAM,EAACC,IAAI,EAAEC,KAAK,EAAC,GAAGb,aAAaT,mBAAmB,CAAC;QACrDW,UAAUD,OAAOU,EAAE;QACnBG,QAAQ,IAAMT,IAAIU,SAAS;IAC7B;IAEA,MAAMzB,mBAAmB;QACvB0B,KAAKT;QACLK;QACAK,QAAQZ,IAAIa,kBAAkB;QAC9BC,YAAY,IAAMd,IAAIU,SAAS;QAC/BK,UAAU;YACRC,WAAWpB,OAAOU,EAAE;YACpBW,aAAaC,OAAOb,SAASc,KAAK;YAClCC,oBAAoBF,OAAOb,SAASgB,iBAAiB;YACrDC,UAAUJ,OAAOb,SAASkB,MAAM;QAClC;IACF;IAEA,IACEf,MAAMgB,UAAU,KAAKnB,SAASc,KAAK,IACnCX,MAAMiB,OAAO,KAAKpB,SAASkB,MAAM,IACjCf,MAAMa,iBAAiB,KAAKhB,SAASgB,iBAAiB,EACtD;QACA,MAAMrC,aAAakB,WAAWwB,KAAK,CAAC,IAAMC;QAC1C,MAAM,IAAIC,MACR,CAAC,6CAA6C,EAAEhC,OAAOU,EAAE,CAAC,WAAW,EAAEE,MAAMgB,UAAU,CAAC,UAAU,EAAEhB,MAAMa,iBAAiB,CAAC,iBAAiB,EAAEb,MAAMiB,OAAO,CAAC,cAAc,EAAEpB,SAASc,KAAK,CAAC,GAAG,EAAEd,SAASgB,iBAAiB,CAAC,iBAAiB,EAAEhB,SAASkB,MAAM,EAAE;IAEpQ;IAEA,MAAM,EAACM,OAAO,EAAC,GAAG,MAAMzC,KAAK0C,WAAW,CAAC,CAACC,KACxCpC,aAAaL,2BAA2B,CAACyC,IAAI;YAC3ClC,UAAUD,OAAOU,EAAE;YACnBP,WAAWG;QACb;IAEF,IAAI,CAAC2B,SAAS;QACZ,MAAM7C,aAAakB;QACnB,MAAMD,UAAU,MAAMZ,qBAAqBO,OAAOU,EAAE;QACpD,OAAO;YAACR,SAASG,UAAU,eAAe;QAAiB;IAC7D;IAEA,OAAO;QACLH,SAAS;QACTC,WAAWG;QACXsB,YAAYhB,MAAMgB,UAAU;QAC5BH,mBAAmBb,MAAMa,iBAAiB;IAC5C;AACF;AAEA,OAAO,SAASW,4BACdrC,eAA0CH,mBAAmB;IAE7D,OAAO,OAAOE;QACZ,IAAII,UAAmC;QACvC,IAAI;YACF,MAAMmC,SAAS,MAAMxC,cAAcC,QAAQC;YAC3CG,UAAUmC,OAAOnC,OAAO;YACxB,OAAOmC;QACT,SAAU;YACR1C,gBAAgB2C,GAAG,CAAC,GAAG;gBAACpC;YAAO;QACjC;IACF;AACF;AAEA,OAAO,MAAMqC,wBAAwBH,8BAA8B"}
+{"version":3,"sources":["../../../src/temporal/activities/compact-stream.ts"],"sourcesContent":["import {reportError} from '@shipfox/node-error-monitoring';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {Context} from '@temporalio/activity';\nimport {compactedObjectKey, deleteObject, putCompactedObject} from '#api/object-storage.js';\nimport {compactedGzipStream} from '#core/compaction.js';\nimport {chunkStats} from '#db/chunks.js';\nimport {db, type Transaction} from '#db/db.js';\nimport {getAttemptStreamById, setObjectKeyAndDeleteChunks} from '#db/streams.js';\nimport {type CompactionMetricOutcome, compactionCount} from '#metrics/instance.js';\n\nexport type CompactStreamResult =\n  | {outcome: 'gone'}\n  | {outcome: 'already-compacted'}\n  | {outcome: 'superseded'}\n  | {outcome: 'retention-raced'}\n  | {outcome: 'compacted'; objectKey: string; chunkCount: number; uncompressedBytes: number};\n\ninterface CompactStreamDependencies {\n  compactedGzipStream: typeof compactedGzipStream;\n  setObjectKeyAndDeleteChunks: (\n    tx: Transaction,\n    params: {streamId: string; objectKey: string},\n  ) => Promise<{updated: boolean}>;\n}\n\nconst defaultDependencies: CompactStreamDependencies = {\n  compactedGzipStream,\n  setObjectKeyAndDeleteChunks,\n};\n\n/**\n * Compacts one closed stream into a single gzip NDJSON object, then deletes its chunk rows.\n *\n *   load stream ──┬─ gone ─────────────► no-op (retention raced)\n *                 ├─ object_key set ───► no-op (idempotent re-run)\n *                 └─ else: upload to a per-attempt key ─► gzip ─► multipart (abort-aware, heartbeat)\n *                          │\n *                 verify streamed count/maxSeq/bytes == table  (mismatch ─► delete upload, throw, retry)\n *                          │\n *                 tx: set object_key (state='closed' AND object_key IS NULL) + delete chunks\n *                          └─ 0 rows ─► delete this attempt's upload, then re-read the row:\n *                                         gone ─► retention raced · keyed ─► superseded by another attempt\n *\n * Each attempt uploads to its own `compactedObjectKey(stream, uuid)`, so a slow or zombie\n * attempt can never overwrite a published object. The single-winner publish (object_key set\n * + chunk delete, atomic) drops chunks only once a complete object is durable; the integrity\n * check (count, maxSeq, and byte total) guards a read bug from publishing a truncated object\n * before the only copy of the source is gone (S3 part checksums cover byte transfer).\n */\nasync function compactStream(\n  params: {streamId: string},\n  dependencies: CompactStreamDependencies,\n): Promise<CompactStreamResult> {\n  const stream = await getAttemptStreamById(params.streamId);\n  if (!stream) return {outcome: 'gone'};\n  if (stream.objectKey) return {outcome: 'already-compacted'};\n\n  const ctx = Context.current();\n  const uploadKey = compactedObjectKey(stream, crypto.randomUUID());\n  const expected = await chunkStats(stream.id);\n  const {body, stats} = dependencies.compactedGzipStream({\n    streamId: stream.id,\n    onPage: () => ctx.heartbeat(),\n  });\n\n  await putCompactedObject({\n    key: uploadKey,\n    body,\n    signal: ctx.cancellationSignal,\n    onProgress: () => ctx.heartbeat(),\n    metadata: {\n      stream_id: stream.id,\n      chunk_count: String(expected.count),\n      uncompressed_bytes: String(expected.uncompressedBytes),\n      last_seq: String(expected.maxSeq),\n    },\n  });\n\n  if (\n    stats.chunkCount !== expected.count ||\n    stats.lastSeq !== expected.maxSeq ||\n    stats.uncompressedBytes !== expected.uncompressedBytes\n  ) {\n    await deleteObject(uploadKey).catch((error) => {\n      logger().error(\n        {err: error, streamId: stream.id, objectKey: uploadKey},\n        'Failed to delete invalid compacted log object',\n      );\n      reportError(error, {\n        boundary: 'logs.cleanup',\n        operation: 'delete-invalid-compaction-object',\n        extra: {streamId: stream.id, objectKey: uploadKey},\n      });\n    });\n    throw new Error(\n      `Compaction integrity check failed for stream ${stream.id}: streamed ${stats.chunkCount} chunks / ${stats.uncompressedBytes} bytes up to seq ${stats.lastSeq}, table holds ${expected.count} / ${expected.uncompressedBytes} bytes up to seq ${expected.maxSeq}`,\n    );\n  }\n\n  const {updated} = await db().transaction((tx) =>\n    dependencies.setObjectKeyAndDeleteChunks(tx, {\n      streamId: stream.id,\n      objectKey: uploadKey,\n    }),\n  );\n  if (!updated) {\n    await deleteObject(uploadKey);\n    const current = await getAttemptStreamById(stream.id);\n    return {outcome: current ? 'superseded' : 'retention-raced'};\n  }\n\n  return {\n    outcome: 'compacted',\n    objectKey: uploadKey,\n    chunkCount: stats.chunkCount,\n    uncompressedBytes: stats.uncompressedBytes,\n  };\n}\n\nexport function createCompactStreamActivity(\n  dependencies: CompactStreamDependencies = defaultDependencies,\n): (params: {streamId: string}) => Promise<CompactStreamResult> {\n  return async (params) => {\n    let outcome: CompactionMetricOutcome = 'failed';\n    try {\n      const result = await compactStream(params, dependencies);\n      outcome = result.outcome;\n      return result;\n    } finally {\n      compactionCount.add(1, {outcome});\n    }\n  };\n}\n\nexport const compactStreamActivity = createCompactStreamActivity();\n"],"names":["reportError","logger","Context","compactedObjectKey","deleteObject","putCompactedObject","compactedGzipStream","chunkStats","db","getAttemptStreamById","setObjectKeyAndDeleteChunks","compactionCount","defaultDependencies","compactStream","params","dependencies","stream","streamId","outcome","objectKey","ctx","current","uploadKey","crypto","randomUUID","expected","id","body","stats","onPage","heartbeat","key","signal","cancellationSignal","onProgress","metadata","stream_id","chunk_count","String","count","uncompressed_bytes","uncompressedBytes","last_seq","maxSeq","chunkCount","lastSeq","catch","error","err","boundary","operation","extra","Error","updated","transaction","tx","createCompactStreamActivity","result","add","compactStreamActivity"],"mappings":"AAAA,SAAQA,WAAW,QAAO,iCAAiC;AAC3D,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,OAAO,QAAO,uBAAuB;AAC7C,SAAQC,kBAAkB,EAAEC,YAAY,EAAEC,kBAAkB,QAAO,yBAAyB;AAC5F,SAAQC,mBAAmB,QAAO,sBAAsB;AACxD,SAAQC,UAAU,QAAO,gBAAgB;AACzC,SAAQC,EAAE,QAAyB,YAAY;AAC/C,SAAQC,oBAAoB,EAAEC,2BAA2B,QAAO,iBAAiB;AACjF,SAAsCC,eAAe,QAAO,uBAAuB;AAiBnF,MAAMC,sBAAiD;IACrDN;IACAI;AACF;AAEA;;;;;;;;;;;;;;;;;;CAkBC,GACD,eAAeG,cACbC,MAA0B,EAC1BC,YAAuC;IAEvC,MAAMC,SAAS,MAAMP,qBAAqBK,OAAOG,QAAQ;IACzD,IAAI,CAACD,QAAQ,OAAO;QAACE,SAAS;IAAM;IACpC,IAAIF,OAAOG,SAAS,EAAE,OAAO;QAACD,SAAS;IAAmB;IAE1D,MAAME,MAAMlB,QAAQmB,OAAO;IAC3B,MAAMC,YAAYnB,mBAAmBa,QAAQO,OAAOC,UAAU;IAC9D,MAAMC,WAAW,MAAMlB,WAAWS,OAAOU,EAAE;IAC3C,MAAM,EAACC,IAAI,EAAEC,KAAK,EAAC,GAAGb,aAAaT,mBAAmB,CAAC;QACrDW,UAAUD,OAAOU,EAAE;QACnBG,QAAQ,IAAMT,IAAIU,SAAS;IAC7B;IAEA,MAAMzB,mBAAmB;QACvB0B,KAAKT;QACLK;QACAK,QAAQZ,IAAIa,kBAAkB;QAC9BC,YAAY,IAAMd,IAAIU,SAAS;QAC/BK,UAAU;YACRC,WAAWpB,OAAOU,EAAE;YACpBW,aAAaC,OAAOb,SAASc,KAAK;YAClCC,oBAAoBF,OAAOb,SAASgB,iBAAiB;YACrDC,UAAUJ,OAAOb,SAASkB,MAAM;QAClC;IACF;IAEA,IACEf,MAAMgB,UAAU,KAAKnB,SAASc,KAAK,IACnCX,MAAMiB,OAAO,KAAKpB,SAASkB,MAAM,IACjCf,MAAMa,iBAAiB,KAAKhB,SAASgB,iBAAiB,EACtD;QACA,MAAMrC,aAAakB,WAAWwB,KAAK,CAAC,CAACC;YACnC9C,SAAS8C,KAAK,CACZ;gBAACC,KAAKD;gBAAO9B,UAAUD,OAAOU,EAAE;gBAAEP,WAAWG;YAAS,GACtD;YAEFtB,YAAY+C,OAAO;gBACjBE,UAAU;gBACVC,WAAW;gBACXC,OAAO;oBAAClC,UAAUD,OAAOU,EAAE;oBAAEP,WAAWG;gBAAS;YACnD;QACF;QACA,MAAM,IAAI8B,MACR,CAAC,6CAA6C,EAAEpC,OAAOU,EAAE,CAAC,WAAW,EAAEE,MAAMgB,UAAU,CAAC,UAAU,EAAEhB,MAAMa,iBAAiB,CAAC,iBAAiB,EAAEb,MAAMiB,OAAO,CAAC,cAAc,EAAEpB,SAASc,KAAK,CAAC,GAAG,EAAEd,SAASgB,iBAAiB,CAAC,iBAAiB,EAAEhB,SAASkB,MAAM,EAAE;IAEpQ;IAEA,MAAM,EAACU,OAAO,EAAC,GAAG,MAAM7C,KAAK8C,WAAW,CAAC,CAACC,KACxCxC,aAAaL,2BAA2B,CAAC6C,IAAI;YAC3CtC,UAAUD,OAAOU,EAAE;YACnBP,WAAWG;QACb;IAEF,IAAI,CAAC+B,SAAS;QACZ,MAAMjD,aAAakB;QACnB,MAAMD,UAAU,MAAMZ,qBAAqBO,OAAOU,EAAE;QACpD,OAAO;YAACR,SAASG,UAAU,eAAe;QAAiB;IAC7D;IAEA,OAAO;QACLH,SAAS;QACTC,WAAWG;QACXsB,YAAYhB,MAAMgB,UAAU;QAC5BH,mBAAmBb,MAAMa,iBAAiB;IAC5C;AACF;AAEA,OAAO,SAASe,4BACdzC,eAA0CH,mBAAmB;IAE7D,OAAO,OAAOE;QACZ,IAAII,UAAmC;QACvC,IAAI;YACF,MAAMuC,SAAS,MAAM5C,cAAcC,QAAQC;YAC3CG,UAAUuC,OAAOvC,OAAO;YACxB,OAAOuC;QACT,SAAU;YACR9C,gBAAgB+C,GAAG,CAAC,GAAG;gBAACxC;YAAO;QACjC;IACF;AACF;AAEA,OAAO,MAAMyC,wBAAwBH,8BAA8B"}

package/dist/temporal/activities/compaction-reconcile.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compaction-reconcile.d.ts","sourceRoot":"","sources":["../../../src/temporal/activities/compaction-reconcile.ts"],"names":[],"mappings":"AASA;;;;;;;;;;GAUG;AACH,wBAAsB,2BAA2B,IAAI,OAAO,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAC,CAAC,CA2BhG"}
+{"version":3,"file":"compaction-reconcile.d.ts","sourceRoot":"","sources":["../../../src/temporal/activities/compaction-reconcile.ts"],"names":[],"mappings":"AAUA;;;;;;;;;;GAUG;AACH,wBAAsB,2BAA2B,IAAI,OAAO,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAC,CAAC,CA4BhG"}

package/dist/temporal/activities/compaction-reconcile.js

@@ -1,3 +1,4 @@
+import { reportError } from '@shipfox/node-error-monitoring';
 import { logger } from '@shipfox/node-opentelemetry';
 import { temporalClient } from '@shipfox/node-temporal';
 import { config } from '#config.js';
@@ -41,6 +42,12 @@ const RECONCILE_BATCH_LIMIT = 100;
                 err: error,
                 streamId: stream.id
             }, 'Failed to re-drive stale stream compaction');
+            reportError(error, {
+                boundary: 'logs.maintenance',
+                extra: {
+                    streamId: stream.id
+                }
+            });
         }
     }
     return {

package/dist/temporal/activities/compaction-reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/temporal/activities/compaction-reconcile.ts"],"sourcesContent":["import {logger} from '@shipfox/node-opentelemetry';\nimport {temporalClient} from '@shipfox/node-temporal';\nimport {config} from '#config.js';\nimport {listStaleUncompactedStreams} from '#db/streams.js';\nimport {LOGS_COMPACTION_TASK_QUEUE} from '#temporal/constants.js';\n\n// Bounded per tick; remaining stale streams are picked up on the next cron run.\nconst RECONCILE_BATCH_LIMIT = 100;\n\n/**\n * Backstop for the event-triggered path: finds closed streams that never got an object\n * key past the stale window and re-starts compaction for each. Starting by the same\n * `logs-compact:{streamId}` workflow id re-drives a stream whose bounded-retry run already\n * failed and closed, while skipping one whose run is still RUNNING (AlreadyStarted). Starts\n * independent workflows, not children, so a re-driven run is not tied to this cron tick.\n *\n * One stream's unexpected start failure is logged and skipped, never thrown: a single poison\n * stream must not abort the batch and leave the rest of the backlog un-re-driven (the next\n * tick retries it). The activity only fails on the initial DB query.\n */\nexport async function compactionReconcileActivity(): Promise<{restarted: number; failed: number}> {\n  const stale = await listStaleUncompactedStreams({\n    olderThanSeconds: config.LOG_COMPACTION_RECONCILE_STALE_SECONDS,\n    limit: RECONCILE_BATCH_LIMIT,\n  });\n\n  let restarted = 0;\n  let failed = 0;\n  for (const stream of stale) {\n    try {\n      await temporalClient().workflow.start('compactStream', {\n        taskQueue: LOGS_COMPACTION_TASK_QUEUE,\n        workflowId: `logs-compact:${stream.id}`,\n        args: [{streamId: stream.id}],\n      });\n      restarted += 1;\n    } catch (error) {\n      if (error instanceof Error && error.name === 'WorkflowExecutionAlreadyStartedError') continue;\n      failed += 1;\n      logger().error(\n        {err: error, streamId: stream.id},\n        'Failed to re-drive stale stream compaction',\n      );\n    }\n  }\n\n  return {restarted, failed};\n}\n"],"names":["logger","temporalClient","config","listStaleUncompactedStreams","LOGS_COMPACTION_TASK_QUEUE","RECONCILE_BATCH_LIMIT","compactionReconcileActivity","stale","olderThanSeconds","LOG_COMPACTION_RECONCILE_STALE_SECONDS","limit","restarted","failed","stream","workflow","start","taskQueue","workflowId","id","args","streamId","error","Error","name","err"],"mappings":"AAAA,SAAQA,MAAM,QAAO,8BAA8B;AACnD,SAAQC,cAAc,QAAO,yBAAyB;AACtD,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAQC,2BAA2B,QAAO,iBAAiB;AAC3D,SAAQC,0BAA0B,QAAO,yBAAyB;AAElE,gFAAgF;AAChF,MAAMC,wBAAwB;AAE9B;;;;;;;;;;CAUC,GACD,OAAO,eAAeC;IACpB,MAAMC,QAAQ,MAAMJ,4BAA4B;QAC9CK,kBAAkBN,OAAOO,sCAAsC;QAC/DC,OAAOL;IACT;IAEA,IAAIM,YAAY;IAChB,IAAIC,SAAS;IACb,KAAK,MAAMC,UAAUN,MAAO;QAC1B,IAAI;YACF,MAAMN,iBAAiBa,QAAQ,CAACC,KAAK,CAAC,iBAAiB;gBACrDC,WAAWZ;gBACXa,YAAY,CAAC,aAAa,EAAEJ,OAAOK,EAAE,EAAE;gBACvCC,MAAM;oBAAC;wBAACC,UAAUP,OAAOK,EAAE;oBAAA;iBAAE;YAC/B;YACAP,aAAa;QACf,EAAE,OAAOU,OAAO;YACd,IAAIA,iBAAiBC,SAASD,MAAME,IAAI,KAAK,wCAAwC;YACrFX,UAAU;YACVZ,SAASqB,KAAK,CACZ;gBAACG,KAAKH;gBAAOD,UAAUP,OAAOK,EAAE;YAAA,GAChC;QAEJ;IACF;IAEA,OAAO;QAACP;QAAWC;IAAM;AAC3B"}
+{"version":3,"sources":["../../../src/temporal/activities/compaction-reconcile.ts"],"sourcesContent":["import {reportError} from '@shipfox/node-error-monitoring';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {temporalClient} from '@shipfox/node-temporal';\nimport {config} from '#config.js';\nimport {listStaleUncompactedStreams} from '#db/streams.js';\nimport {LOGS_COMPACTION_TASK_QUEUE} from '#temporal/constants.js';\n\n// Bounded per tick; remaining stale streams are picked up on the next cron run.\nconst RECONCILE_BATCH_LIMIT = 100;\n\n/**\n * Backstop for the event-triggered path: finds closed streams that never got an object\n * key past the stale window and re-starts compaction for each. Starting by the same\n * `logs-compact:{streamId}` workflow id re-drives a stream whose bounded-retry run already\n * failed and closed, while skipping one whose run is still RUNNING (AlreadyStarted). Starts\n * independent workflows, not children, so a re-driven run is not tied to this cron tick.\n *\n * One stream's unexpected start failure is logged and skipped, never thrown: a single poison\n * stream must not abort the batch and leave the rest of the backlog un-re-driven (the next\n * tick retries it). The activity only fails on the initial DB query.\n */\nexport async function compactionReconcileActivity(): Promise<{restarted: number; failed: number}> {\n  const stale = await listStaleUncompactedStreams({\n    olderThanSeconds: config.LOG_COMPACTION_RECONCILE_STALE_SECONDS,\n    limit: RECONCILE_BATCH_LIMIT,\n  });\n\n  let restarted = 0;\n  let failed = 0;\n  for (const stream of stale) {\n    try {\n      await temporalClient().workflow.start('compactStream', {\n        taskQueue: LOGS_COMPACTION_TASK_QUEUE,\n        workflowId: `logs-compact:${stream.id}`,\n        args: [{streamId: stream.id}],\n      });\n      restarted += 1;\n    } catch (error) {\n      if (error instanceof Error && error.name === 'WorkflowExecutionAlreadyStartedError') continue;\n      failed += 1;\n      logger().error(\n        {err: error, streamId: stream.id},\n        'Failed to re-drive stale stream compaction',\n      );\n      reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});\n    }\n  }\n\n  return {restarted, failed};\n}\n"],"names":["reportError","logger","temporalClient","config","listStaleUncompactedStreams","LOGS_COMPACTION_TASK_QUEUE","RECONCILE_BATCH_LIMIT","compactionReconcileActivity","stale","olderThanSeconds","LOG_COMPACTION_RECONCILE_STALE_SECONDS","limit","restarted","failed","stream","workflow","start","taskQueue","workflowId","id","args","streamId","error","Error","name","err","boundary","extra"],"mappings":"AAAA,SAAQA,WAAW,QAAO,iCAAiC;AAC3D,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,cAAc,QAAO,yBAAyB;AACtD,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAQC,2BAA2B,QAAO,iBAAiB;AAC3D,SAAQC,0BAA0B,QAAO,yBAAyB;AAElE,gFAAgF;AAChF,MAAMC,wBAAwB;AAE9B;;;;;;;;;;CAUC,GACD,OAAO,eAAeC;IACpB,MAAMC,QAAQ,MAAMJ,4BAA4B;QAC9CK,kBAAkBN,OAAOO,sCAAsC;QAC/DC,OAAOL;IACT;IAEA,IAAIM,YAAY;IAChB,IAAIC,SAAS;IACb,KAAK,MAAMC,UAAUN,MAAO;QAC1B,IAAI;YACF,MAAMN,iBAAiBa,QAAQ,CAACC,KAAK,CAAC,iBAAiB;gBACrDC,WAAWZ;gBACXa,YAAY,CAAC,aAAa,EAAEJ,OAAOK,EAAE,EAAE;gBACvCC,MAAM;oBAAC;wBAACC,UAAUP,OAAOK,EAAE;oBAAA;iBAAE;YAC/B;YACAP,aAAa;QACf,EAAE,OAAOU,OAAO;YACd,IAAIA,iBAAiBC,SAASD,MAAME,IAAI,KAAK,wCAAwC;YACrFX,UAAU;YACVZ,SAASqB,KAAK,CACZ;gBAACG,KAAKH;gBAAOD,UAAUP,OAAOK,EAAE;YAAA,GAChC;YAEFnB,YAAYsB,OAAO;gBAACI,UAAU;gBAAoBC,OAAO;oBAACN,UAAUP,OAAOK,EAAE;gBAAA;YAAC;QAChF;IACF;IAEA,OAAO;QAACP;QAAWC;IAAM;AAC3B"}