@shipfox/api-logs 2.0.0

package/dist/api/object-storage.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/api/object-storage.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  DeleteObjectCommand,\n  DeleteObjectsCommand,\n  GetObjectCommand,\n  HeadBucketCommand,\n  ListObjectsV2Command,\n  S3Client,\n} from '@aws-sdk/client-s3';\nimport {Upload} from '@aws-sdk/lib-storage';\nimport {getSignedUrl} from '@aws-sdk/s3-request-presigner';\nimport {config} from '#config.js';\nimport {type LogObjectKeyParams, logObjectKey} from '#core/entities/log-object.js';\n\nconst explicitS3Credentials =\n  config.LOG_STORAGE_S3_ACCESS_KEY_ID && config.LOG_STORAGE_S3_SECRET_ACCESS_KEY\n    ? {\n        accessKeyId: config.LOG_STORAGE_S3_ACCESS_KEY_ID,\n        secretAccessKey: config.LOG_STORAGE_S3_SECRET_ACCESS_KEY,\n      }\n    : undefined;\n\nlet _client: S3Client | undefined;\n\n/** Lazily-built S3 client targeting the configured object store (Garage in dev). */\nexport function s3Client(): S3Client {\n  if (!_client) {\n    _client = new S3Client({\n      endpoint: config.LOG_STORAGE_S3_ENDPOINT,\n      region: config.LOG_STORAGE_S3_REGION,\n      forcePathStyle: config.LOG_STORAGE_S3_FORCE_PATH_STYLE,\n      ...(explicitS3Credentials ? {credentials: explicitS3Credentials} : {}),\n      // Fail fast: a slow or black-holed endpoint must not hang callers behind\n      // SDK backoff. One retry, short connect/request timeouts.\n      maxAttempts: 2,\n      requestHandler: {connectionTimeout: 1_000, requestTimeout: 3_000},\n    });\n  }\n  return _client;\n}\n\nexport function closeS3Client(): void {\n  _client?.destroy();\n  _client = undefined;\n}\n\n/** Readiness probe: true when the configured bucket answers a HEAD. Used by the app readiness check, not at process boot. */\nexport async function checkBucketReachable(): Promise<boolean> {\n  try {\n    await s3Client().send(new HeadBucketCommand({Bucket: config.LOG_STORAGE_S3_BUCKET}));\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nlet _uploadClient: S3Client | undefined;\n\n/**\n * S3 client for streaming compaction uploads. `s3Client()` is tuned fail-fast for the\n * readiness HEAD, but each multipart part upload can take as long as the link needs, so\n * the per-request timeout is disabled here. Temporal owns the retry budget, so the SDK\n * keeps only a small attempt count for transient connection blips.\n */\nexport function uploadS3Client(): S3Client {\n  if (!_uploadClient) {\n    _uploadClient = new S3Client({\n      endpoint: config.LOG_STORAGE_S3_ENDPOINT,\n      region: config.LOG_STORAGE_S3_REGION,\n      forcePathStyle: config.LOG_STORAGE_S3_FORCE_PATH_STYLE,\n      ...(explicitS3Credentials ? {credentials: explicitS3Credentials} : {}),\n      maxAttempts: 3,\n      requestHandler: {connectionTimeout: 5_000, requestTimeout: 0},\n    });\n  }\n  return _uploadClient;\n}\n\nexport function closeUploadS3Client(): void {\n  _uploadClient?.destroy();\n  _uploadClient = undefined;\n}\n\n/**\n * Per-attempt object key for a compacted stream: the stream's stable prefix\n * (`logObjectKey` -> `{prefix}/{workspaceId}/{jobId}/{stepId}/{attempt}`, shared with retention\n * and workspace prefix deletes) plus a unique `uploadToken` leaf. Each compaction attempt\n * uploads to its own key and the winner records it atomically, so a slow or zombie attempt\n * can never overwrite a published object. Losing and crashed attempts leave a leaf under the\n * same prefix, reclaimed by the bucket's incomplete-multipart-abort rule and prefix-scoped\n * retention.\n */\nexport function compactedObjectKey(identity: LogObjectKeyParams, uploadToken: string): string {\n  return `${logObjectKey(config.LOG_STORAGE_S3_PREFIX, identity)}/${uploadToken}`;\n}\n\n// lib-storage defaults (5MB parts, queue of 4); we pin a small queue so peak buffer memory\n// stays bounded under concurrent compactions (queueSize x partSize per active upload).\nconst UPLOAD_PART_SIZE = 5 * 1024 * 1024;\nconst UPLOAD_QUEUE_SIZE = 2;\n\nexport interface PutCompactedObjectParams {\n  key: string;\n  body: Readable;\n  metadata: Record<string, string>;\n  signal?: AbortSignal;\n  onProgress?: () => void;\n}\n\n/**\n * Streams `body` (gzip-compressed NDJSON) to the compacted object via a multipart upload.\n * `Content-Encoding: gzip` so a browser-direct presigned GET auto-decompresses. Aborts the\n * multipart upload when `signal` fires, so a cancelled activity leaves no dangling parts.\n *\n * A hard crash (no graceful abort) can still strand incomplete multipart parts, which on\n * pay-per-storage providers (S3, R2) bill until aborted. The worker never sets bucket\n * policy, so the deployed bucket must carry an `AbortIncompleteMultipartUpload` lifecycle\n * rule. `dev/garage/bootstrap.sh` provisions it for local Garage; self-hosters set the same\n * rule on their bucket.\n */\nexport async function putCompactedObject(params: PutCompactedObjectParams): Promise<void> {\n  const upload = new Upload({\n    client: uploadS3Client(),\n    partSize: UPLOAD_PART_SIZE,\n    queueSize: UPLOAD_QUEUE_SIZE,\n    params: {\n      Bucket: config.LOG_STORAGE_S3_BUCKET,\n      Key: params.key,\n      Body: params.body,\n      ContentType: 'application/x-ndjson',\n      ContentEncoding: 'gzip',\n      Metadata: params.metadata,\n    },\n  });\n\n  if (params.onProgress) upload.on('httpUploadProgress', params.onProgress);\n  if (params.signal) {\n    // Subscribe before the aborted-check so a signal firing between them cannot be missed,\n    // then handle the already-aborted case eagerly (an already-aborted signal never fires 'abort').\n    params.signal.addEventListener(\n      'abort',\n      () => {\n        void upload.abort().catch(() => undefined);\n      },\n      {once: true},\n    );\n    if (params.signal.aborted) {\n      await upload.abort().catch(() => undefined);\n      throw new Error('Compaction upload aborted before it started');\n    }\n  }\n\n  await upload.done();\n}\n\n/**\n * Deletes a compacted object. S3 `DeleteObject` is idempotent (a missing key is a success),\n * so retention and the publish orphan guard can call it freely without a pre-check.\n */\nexport async function deleteObject(key: string): Promise<void> {\n  await s3Client().send(new DeleteObjectCommand({Bucket: config.LOG_STORAGE_S3_BUCKET, Key: key}));\n}\n\n/**\n * Presigns a GET for a compacted log object so the browser can fetch it directly,\n * bypassing API egress. `getSignedUrl` only computes a signature locally (no network\n * call), so the fail-fast `s3Client()` is fine here. `expiresAt` mirrors the URL's\n * `LOG_READ_URL_TTL_SECONDS` lifetime so the caller can hand the client an absolute\n * deadline without re-deriving it.\n */\nexport async function presignedGetUrl(objectKey: string): Promise<{url: string; expiresAt: Date}> {\n  const ttlSeconds = config.LOG_READ_URL_TTL_SECONDS;\n  // Stamp the deadline from before signing: getSignedUrl bakes its expiry from the clock at\n  // sign time, so reading Date.now() after the await could report a deadline slightly past the\n  // real one. Computing it first keeps expiresAt a conservative (never-late) bound.\n  const expiresAt = new Date(Date.now() + ttlSeconds * 1000);\n  const url = await getSignedUrl(\n    s3Client(),\n    new GetObjectCommand({Bucket: config.LOG_STORAGE_S3_BUCKET, Key: objectKey}),\n    {expiresIn: ttlSeconds},\n  );\n  return {url, expiresAt};\n}\n\n/**\n * Deletes all objects under a per-attempt prefix. Retention deletes by prefix, not only the\n * recorded `object_key`, to reclaim orphan leaves from failed compaction attempts.\n *\n * Pass a trailing slash so attempt `1` never matches attempt `10`.\n */\nexport async function deleteObjectsByPrefix(prefix: string): Promise<void> {\n  if (prefix === '') {\n    throw new Error(\n      'deleteObjectsByPrefix refuses an empty prefix (it would target the whole bucket)',\n    );\n  }\n  const client = s3Client();\n  let continuationToken: string | undefined;\n  do {\n    const listed = await client.send(\n      new ListObjectsV2Command({\n        Bucket: config.LOG_STORAGE_S3_BUCKET,\n        Prefix: prefix,\n        ContinuationToken: continuationToken,\n      }),\n    );\n    const objects = (listed.Contents ?? []).flatMap((object) =>\n      object.Key ? [{Key: object.Key}] : [],\n    );\n    if (objects.length > 0) {\n      const deleted = await client.send(\n        new DeleteObjectsCommand({\n          Bucket: config.LOG_STORAGE_S3_BUCKET,\n          Delete: {Objects: objects, Quiet: true},\n        }),\n      );\n      if (deleted.Errors && deleted.Errors.length > 0) {\n        const [first] = deleted.Errors;\n        throw new Error(\n          `Failed to delete ${deleted.Errors.length} object(s) under ${prefix}: ${first?.Key} ${first?.Message}`,\n        );\n      }\n    }\n    continuationToken = listed.IsTruncated ? listed.NextContinuationToken : undefined;\n  } while (continuationToken);\n}\n"],"names":["DeleteObjectCommand","DeleteObjectsCommand","GetObjectCommand","HeadBucketCommand","ListObjectsV2Command","S3Client","Upload","getSignedUrl","config","logObjectKey","explicitS3Credentials","LOG_STORAGE_S3_ACCESS_KEY_ID","LOG_STORAGE_S3_SECRET_ACCESS_KEY","accessKeyId","secretAccessKey","undefined","_client","s3Client","endpoint","LOG_STORAGE_S3_ENDPOINT","region","LOG_STORAGE_S3_REGION","forcePathStyle","LOG_STORAGE_S3_FORCE_PATH_STYLE","credentials","maxAttempts","requestHandler","connectionTimeout","requestTimeout","closeS3Client","destroy","checkBucketReachable","send","Bucket","LOG_STORAGE_S3_BUCKET","_uploadClient","uploadS3Client","closeUploadS3Client","compactedObjectKey","identity","uploadToken","LOG_STORAGE_S3_PREFIX","UPLOAD_PART_SIZE","UPLOAD_QUEUE_SIZE","putCompactedObject","params","upload","client","partSize","queueSize","Key","key","Body","body","ContentType","ContentEncoding","Metadata","metadata","onProgress","on","signal","addEventListener","abort","catch","once","aborted","Error","done","deleteObject","presignedGetUrl","objectKey","ttlSeconds","LOG_READ_URL_TTL_SECONDS","expiresAt","Date","now","url","expiresIn","deleteObjectsByPrefix","prefix","continuationToken","listed","Prefix","ContinuationToken","objects","Contents","flatMap","object","length","deleted","Delete","Objects","Quiet","Errors","first","Message","IsTruncated","NextContinuationToken"],"mappings":"AACA,SACEA,mBAAmB,EACnBC,oBAAoB,EACpBC,gBAAgB,EAChBC,iBAAiB,EACjBC,oBAAoB,EACpBC,QAAQ,QACH,qBAAqB;AAC5B,SAAQC,MAAM,QAAO,uBAAuB;AAC5C,SAAQC,YAAY,QAAO,gCAAgC;AAC3D,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAiCC,YAAY,QAAO,+BAA+B;AAEnF,MAAMC,wBACJF,OAAOG,4BAA4B,IAAIH,OAAOI,gCAAgC,GAC1E;IACEC,aAAaL,OAAOG,4BAA4B;IAChDG,iBAAiBN,OAAOI,gCAAgC;AAC1D,IACAG;AAEN,IAAIC;AAEJ,kFAAkF,GAClF,OAAO,SAASC;IACd,IAAI,CAACD,SAAS;QACZA,UAAU,IAAIX,SAAS;YACrBa,UAAUV,OAAOW,uBAAuB;YACxCC,QAAQZ,OAAOa,qBAAqB;YACpCC,gBAAgBd,OAAOe,+BAA+B;YACtD,GAAIb,wBAAwB;gBAACc,aAAad;YAAqB,IAAI,CAAC,CAAC;YACrE,yEAAyE;YACzE,0DAA0D;YAC1De,aAAa;YACbC,gBAAgB;gBAACC,mBAAmB;gBAAOC,gBAAgB;YAAK;QAClE;IACF;IACA,OAAOZ;AACT;AAEA,OAAO,SAASa;IACdb,SAASc;IACTd,UAAUD;AACZ;AAEA,2HAA2H,GAC3H,OAAO,eAAegB;IACpB,IAAI;QACF,MAAMd,WAAWe,IAAI,CAAC,IAAI7B,kBAAkB;YAAC8B,QAAQzB,OAAO0B,qBAAqB;QAAA;QACjF,OAAO;IACT,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,IAAIC;AAEJ;;;;;CAKC,GACD,OAAO,SAASC;IACd,IAAI,CAACD,eAAe;QAClBA,gBAAgB,IAAI9B,SAAS;YAC3Ba,UAAUV,OAAOW,uBAAuB;YACxCC,QAAQZ,OAAOa,qBAAqB;YACpCC,gBAAgBd,OAAOe,+BAA+B;YACtD,GAAIb,wBAAwB;gBAACc,aAAad;YAAqB,IAAI,CAAC,CAAC;YACrEe,aAAa;YACbC,gBAAgB;gBAACC,mBAAmB;gBAAOC,gBAAgB;YAAC;QAC9D;IACF;IACA,OAAOO;AACT;AAEA,OAAO,SAASE;IACdF,eAAeL;IACfK,gBAAgBpB;AAClB;AAEA;;;;;;;;CAQC,GACD,OAAO,SAASuB,mBAAmBC,QAA4B,EAAEC,WAAmB;IAClF,OAAO,GAAG/B,aAAaD,OAAOiC,qBAAqB,EAAEF,UAAU,CAAC,EAAEC,aAAa;AACjF;AAEA,2FAA2F;AAC3F,uFAAuF;AACvF,MAAME,mBAAmB,IAAI,OAAO;AACpC,MAAMC,oBAAoB;AAU1B;;;;;;;;;;CAUC,GACD,OAAO,eAAeC,mBAAmBC,MAAgC;IACvE,MAAMC,SAAS,IAAIxC,OAAO;QACxByC,QAAQX;QACRY,UAAUN;QACVO,WAAWN;QACXE,QAAQ;YACNZ,QAAQzB,OAAO0B,qBAAqB;YACpCgB,KAAKL,OAAOM,GAAG;YACfC,MAAMP,OAAOQ,IAAI;YACjBC,aAAa;YACbC,iBAAiB;YACjBC,UAAUX,OAAOY,QAAQ;QAC3B;IACF;IAEA,IAAIZ,OAAOa,UAAU,EAAEZ,OAAOa,EAAE,CAAC,sBAAsBd,OAAOa,UAAU;IACxE,IAAIb,OAAOe,MAAM,EAAE;QACjB,uFAAuF;QACvF,gGAAgG;QAChGf,OAAOe,MAAM,CAACC,gBAAgB,CAC5B,SACA;YACE,KAAKf,OAAOgB,KAAK,GAAGC,KAAK,CAAC,IAAMhD;QAClC,GACA;YAACiD,MAAM;QAAI;QAEb,IAAInB,OAAOe,MAAM,CAACK,OAAO,EAAE;YACzB,MAAMnB,OAAOgB,KAAK,GAAGC,KAAK,CAAC,IAAMhD;YACjC,MAAM,IAAImD,MAAM;QAClB;IACF;IAEA,MAAMpB,OAAOqB,IAAI;AACnB;AAEA;;;CAGC,GACD,OAAO,eAAeC,aAAajB,GAAW;IAC5C,MAAMlC,WAAWe,IAAI,CAAC,IAAIhC,oBAAoB;QAACiC,QAAQzB,OAAO0B,qBAAqB;QAAEgB,KAAKC;IAAG;AAC/F;AAEA;;;;;;CAMC,GACD,OAAO,eAAekB,gBAAgBC,SAAiB;IACrD,MAAMC,aAAa/D,OAAOgE,wBAAwB;IAClD,0FAA0F;IAC1F,6FAA6F;IAC7F,kFAAkF;IAClF,MAAMC,YAAY,IAAIC,KAAKA,KAAKC,GAAG,KAAKJ,aAAa;IACrD,MAAMK,MAAM,MAAMrE,aAChBU,YACA,IAAIf,iBAAiB;QAAC+B,QAAQzB,OAAO0B,qBAAqB;QAAEgB,KAAKoB;IAAS,IAC1E;QAACO,WAAWN;IAAU;IAExB,OAAO;QAACK;QAAKH;IAAS;AACxB;AAEA;;;;;CAKC,GACD,OAAO,eAAeK,sBAAsBC,MAAc;IACxD,IAAIA,WAAW,IAAI;QACjB,MAAM,IAAIb,MACR;IAEJ;IACA,MAAMnB,SAAS9B;IACf,IAAI+D;IACJ,GAAG;QACD,MAAMC,SAAS,MAAMlC,OAAOf,IAAI,CAC9B,IAAI5B,qBAAqB;YACvB6B,QAAQzB,OAAO0B,qBAAqB;YACpCgD,QAAQH;YACRI,mBAAmBH;QACrB;QAEF,MAAMI,UAAU,AAACH,CAAAA,OAAOI,QAAQ,IAAI,EAAE,AAAD,EAAGC,OAAO,CAAC,CAACC,SAC/CA,OAAOrC,GAAG,GAAG;gBAAC;oBAACA,KAAKqC,OAAOrC,GAAG;gBAAA;aAAE,GAAG,EAAE;QAEvC,IAAIkC,QAAQI,MAAM,GAAG,GAAG;YACtB,MAAMC,UAAU,MAAM1C,OAAOf,IAAI,CAC/B,IAAI/B,qBAAqB;gBACvBgC,QAAQzB,OAAO0B,qBAAqB;gBACpCwD,QAAQ;oBAACC,SAASP;oBAASQ,OAAO;gBAAI;YACxC;YAEF,IAAIH,QAAQI,MAAM,IAAIJ,QAAQI,MAAM,CAACL,MAAM,GAAG,GAAG;gBAC/C,MAAM,CAACM,MAAM,GAAGL,QAAQI,MAAM;gBAC9B,MAAM,IAAI3B,MACR,CAAC,iBAAiB,EAAEuB,QAAQI,MAAM,CAACL,MAAM,CAAC,iBAAiB,EAAET,OAAO,EAAE,EAAEe,OAAO5C,IAAI,CAAC,EAAE4C,OAAOC,SAAS;YAE1G;QACF;QACAf,oBAAoBC,OAAOe,WAAW,GAAGf,OAAOgB,qBAAqB,GAAGlF;IAC1E,QAASiE,kBAAmB;AAC9B"}

package/dist/config.d.ts

@@ -0,0 +1,20 @@
+export declare const config: Readonly<{
+    LOG_STORAGE_S3_ENDPOINT: string;
+    LOG_STORAGE_S3_REGION: string;
+    LOG_STORAGE_S3_BUCKET: string;
+    LOG_STORAGE_S3_PREFIX: string;
+    LOG_STORAGE_S3_ACCESS_KEY_ID: string | undefined;
+    LOG_STORAGE_S3_SECRET_ACCESS_KEY: string | undefined;
+    LOG_STORAGE_S3_FORCE_PATH_STYLE: boolean;
+    LOG_BUDGET_BASE_BYTES: number;
+    LOG_BUDGET_RATE_BYTES_PER_MINUTE: number;
+    LOG_STREAM_CLOSE_GRACE_SECONDS: number;
+    LOG_COMPACTION_RECONCILE_STALE_SECONDS: number;
+    LOG_STREAM_REAP_AFTER_SECONDS: number;
+    LOG_APPEND_BODY_LIMIT_BYTES: number;
+    LOG_MAX_SESSION_LINE_BYTES: number;
+    LOG_READ_URL_TTL_SECONDS: number;
+    LOG_READ_INLINE_MAX_BYTES: number;
+    LOG_RETENTION_DAYS: number;
+} & import("@shipfox/config").CleanedEnvAccessors>;
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;kDAqEjB,CAAC"}

package/dist/config.js

@@ -0,0 +1,108 @@
+import { config as authConfig } from '@shipfox/api-auth/config';
+import { bool, createConfig, num, str, url } from '@shipfox/config';
+import { durationToSeconds } from '@shipfox/node-jwt';
+export const config = createConfig({
+    LOG_STORAGE_S3_ENDPOINT: url({
+        desc: 'Endpoint URL of the S3-compatible object store that holds compacted logs. Defaults to the bundled local-development Garage (http://localhost:3900); set it to your object store endpoint for production.',
+        default: 'http://localhost:3900'
+    }),
+    LOG_STORAGE_S3_REGION: str({
+        desc: 'Region passed to the S3 client. Any value works for Garage; set the real region for AWS S3. Defaults to garage for local development.',
+        default: 'garage'
+    }),
+    LOG_STORAGE_S3_BUCKET: str({
+        desc: 'Name of the bucket that stores compacted log objects. Defaults to shipfox-logs (created by dev/garage/bootstrap.sh); create the bucket and set this for production.',
+        default: 'shipfox-logs'
+    }),
+    LOG_STORAGE_S3_PREFIX: str({
+        desc: 'Key prefix under which compacted log objects are stored in the bucket. Set this to host several modules in one bucket, each under its own prefix. Use a value without a leading or trailing slash. Defaults to logs.',
+        default: 'logs'
+    }),
+    LOG_STORAGE_S3_ACCESS_KEY_ID: str({
+        desc: 'Optional access key ID used to authenticate to the object store. Set it together with LOG_STORAGE_S3_SECRET_ACCESS_KEY for an explicit credential pair, or leave both unset to use the standard AWS SDK credential provider chain.',
+        default: undefined
+    }),
+    LOG_STORAGE_S3_SECRET_ACCESS_KEY: str({
+        desc: 'Optional secret access key used to authenticate to the object store. Set it together with LOG_STORAGE_S3_ACCESS_KEY_ID for an explicit credential pair, or leave both unset to use the standard AWS SDK credential provider chain.',
+        default: undefined
+    }),
+    LOG_STORAGE_S3_FORCE_PATH_STYLE: bool({
+        desc: 'Whether to address the bucket as a path (endpoint/bucket) instead of a subdomain. Set it to true for Garage and MinIO; false works for AWS S3.',
+        default: true
+    }),
+    LOG_BUDGET_BASE_BYTES: num({
+        desc: 'Base of the per-job log accrual budget, in stored bytes (normalized NDJSON the server keeps, framing included). A job may always store this much before the time-based rate is added. Sized to hold a few inline agent_session entries (such as base64 images) before the shared per-job cap trips. Defaults to 32 MiB.',
+        default: 33_554_432
+    }),
+    LOG_BUDGET_RATE_BYTES_PER_MINUTE: num({
+        desc: 'Stored bytes added to the per-job log budget for each minute since the first log append. Defaults to 1 MiB per minute.',
+        default: 1_048_576
+    }),
+    LOG_STREAM_CLOSE_GRACE_SECONDS: num({
+        desc: 'How long to wait after a job reaches a terminal state before force-closing any of its log streams the runner never ended itself (it died, was capped, or its log spool failed). The wait lets a last in-flight chunk land before the stream is marked truncated. Defaults to 120 seconds.',
+        default: 120
+    }),
+    LOG_COMPACTION_RECONCILE_STALE_SECONDS: num({
+        desc: 'How long a closed log stream may stay uncompacted before the reconcile cron re-drives it. The event-triggered compaction normally runs within seconds of close; this backstop only re-drives streams whose compaction never started or permanently failed. Defaults to 900 seconds (15 minutes).',
+        default: 900
+    }),
+    LOG_STREAM_REAP_AFTER_SECONDS: num({
+        desc: 'How long a log stream may stay open before the reaper cron force-closes it as abandoned. This is the backstop for a stream a runner started after its job already went terminal (the one-shot close sweep had already run); the reaper marks it truncated so it re-enters the compaction and retention lifecycle. Must be greater than AUTH_JOB_LEASE_TOKEN_EXPIRES_IN (the job lease lifetime, default 90 minutes) so a still-valid lease can never be force-closed mid-append, which would silently truncate live logs. Defaults to 7200 seconds (2 hours).',
+        default: 7200
+    }),
+    LOG_APPEND_BODY_LIMIT_BYTES: num({
+        desc: 'Maximum size of a single log append request body, in bytes. Output records cap at 16 KiB each, but one agent_session record carries a whole session entry (which can inline base64 images), so the body must hold a full LOG_MAX_SESSION_LINE_BYTES line plus framing. This also bounds how far a job can overshoot its log budget: the one append that crosses the budget is stored in full before the job is capped. Defaults to 8 MiB.',
+        default: 8_388_608
+    }),
+    LOG_MAX_SESSION_LINE_BYTES: num({
+        desc: 'Maximum size of one agent_session log line (a verbatim agent session entry), in bytes. A larger line is rejected with 400, and the runner drops it with a gap marker instead of sending it. Sized generously for entries that inline large content such as base64 images. Must be at most LOG_APPEND_BODY_LIMIT_BYTES, since a line that cannot fit in one request body could never be accepted. Defaults to 4 MiB.',
+        default: 4_194_304
+    }),
+    LOG_READ_URL_TTL_SECONDS: num({
+        desc: 'Lifetime, in seconds, of a presigned GET URL handed out for a compacted (cold) log read. The browser must finish fetching the object before it expires; raise it for slow links or large logs. Must be between 1 and 604800 (7 days, the longest a presigned URL can live). Defaults to 3600 (one hour).',
+        default: 3600
+    }),
+    LOG_READ_INLINE_MAX_BYTES: num({
+        desc: 'Maximum number of stored log bytes returned in a single inline (hot) read response. A long-running step can buffer far more than this in Postgres before compaction, so a read is capped here and the client re-polls (following has_more) to drain the backlog before it tails. Bounds per-request memory. Must be at least 1. Defaults to 1 MiB.',
+        default: 1_048_576
+    }),
+    LOG_RETENTION_DAYS: num({
+        desc: 'How many days a closed log stream is kept before the retention cron hard-deletes its stored object and database row. Our own worker enforces this (not bucket lifecycle rules), so behavior is identical across object stores. Must be a whole number of days, 1 or greater. Defaults to 90 days.',
+        default: 90
+    })
+});
+const hasS3AccessKeyId = Boolean(config.LOG_STORAGE_S3_ACCESS_KEY_ID);
+const hasS3SecretAccessKey = Boolean(config.LOG_STORAGE_S3_SECRET_ACCESS_KEY);
+if (hasS3AccessKeyId !== hasS3SecretAccessKey) {
+    throw new Error('LOG_STORAGE_S3_ACCESS_KEY_ID and LOG_STORAGE_S3_SECRET_ACCESS_KEY must be set together or both left unset.');
+}
+// SigV4 caps a presigned URL's lifetime at 7 days, so a larger TTL would fail at signing
+// time on every cold read. Reject out-of-range values at startup rather than per request.
+const MAX_PRESIGN_TTL_SECONDS = 604_800;
+if (config.LOG_READ_URL_TTL_SECONDS < 1 || config.LOG_READ_URL_TTL_SECONDS > MAX_PRESIGN_TTL_SECONDS) {
+    throw new Error(`LOG_READ_URL_TTL_SECONDS must be between 1 and ${MAX_PRESIGN_TTL_SECONDS}; got ${config.LOG_READ_URL_TTL_SECONDS}`);
+}
+if (config.LOG_READ_INLINE_MAX_BYTES < 1) {
+    throw new Error(`LOG_READ_INLINE_MAX_BYTES must be at least 1; got ${config.LOG_READ_INLINE_MAX_BYTES}`);
+}
+// A whole agent_session line is one record in one body. If the body limit is below the
+// line cap, a legitimate large session line could never fit in one request body — Fastify
+// would 413 it before the per-line validator runs. Fail fast at startup instead.
+if (config.LOG_APPEND_BODY_LIMIT_BYTES < config.LOG_MAX_SESSION_LINE_BYTES) {
+    throw new Error(`LOG_APPEND_BODY_LIMIT_BYTES (${config.LOG_APPEND_BODY_LIMIT_BYTES}) must be >= LOG_MAX_SESSION_LINE_BYTES (${config.LOG_MAX_SESSION_LINE_BYTES}).`);
+}
+// `num` accepts zero, negative, and fractional values; reject them before they reach
+// `make_interval(days => ...)` and hard-delete the wrong streams.
+if (!Number.isInteger(config.LOG_RETENTION_DAYS) || config.LOG_RETENTION_DAYS < 1) {
+    throw new Error(`LOG_RETENTION_DAYS (${config.LOG_RETENTION_DAYS}) must be a whole number of days >= 1.`);
+}
+// A stream still accepts appends until its job lease expires, so reaping it any sooner would
+// truncate live logs. Validate against the real lease TTL read from auth's config (not a
+// hardcoded floor), so raising AUTH_JOB_LEASE_TOKEN_EXPIRES_IN can never silently outrun the
+// reaper window.
+const jobLeaseTtlSeconds = durationToSeconds(authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN);
+if (!Number.isFinite(config.LOG_STREAM_REAP_AFTER_SECONDS) || config.LOG_STREAM_REAP_AFTER_SECONDS <= jobLeaseTtlSeconds) {
+    throw new Error(`LOG_STREAM_REAP_AFTER_SECONDS (${config.LOG_STREAM_REAP_AFTER_SECONDS}) must be greater than the job lease TTL (${jobLeaseTtlSeconds}s, from AUTH_JOB_LEASE_TOKEN_EXPIRES_IN=${authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN}); a smaller value would let the reaper force-close a stream a still-valid lease is appending to and silently truncate live logs.`);
+}
+
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/config.ts"],"sourcesContent":["import {config as authConfig} from '@shipfox/api-auth/config';\nimport {bool, createConfig, num, str, url} from '@shipfox/config';\nimport {durationToSeconds} from '@shipfox/node-jwt';\n\nexport const config = createConfig({\n  LOG_STORAGE_S3_ENDPOINT: url({\n    desc: 'Endpoint URL of the S3-compatible object store that holds compacted logs. Defaults to the bundled local-development Garage (http://localhost:3900); set it to your object store endpoint for production.',\n    default: 'http://localhost:3900',\n  }),\n  LOG_STORAGE_S3_REGION: str({\n    desc: 'Region passed to the S3 client. Any value works for Garage; set the real region for AWS S3. Defaults to garage for local development.',\n    default: 'garage',\n  }),\n  LOG_STORAGE_S3_BUCKET: str({\n    desc: 'Name of the bucket that stores compacted log objects. Defaults to shipfox-logs (created by dev/garage/bootstrap.sh); create the bucket and set this for production.',\n    default: 'shipfox-logs',\n  }),\n  LOG_STORAGE_S3_PREFIX: str({\n    desc: 'Key prefix under which compacted log objects are stored in the bucket. Set this to host several modules in one bucket, each under its own prefix. Use a value without a leading or trailing slash. Defaults to logs.',\n    default: 'logs',\n  }),\n  LOG_STORAGE_S3_ACCESS_KEY_ID: str({\n    desc: 'Optional access key ID used to authenticate to the object store. Set it together with LOG_STORAGE_S3_SECRET_ACCESS_KEY for an explicit credential pair, or leave both unset to use the standard AWS SDK credential provider chain.',\n    default: undefined,\n  }),\n  LOG_STORAGE_S3_SECRET_ACCESS_KEY: str({\n    desc: 'Optional secret access key used to authenticate to the object store. Set it together with LOG_STORAGE_S3_ACCESS_KEY_ID for an explicit credential pair, or leave both unset to use the standard AWS SDK credential provider chain.',\n    default: undefined,\n  }),\n  LOG_STORAGE_S3_FORCE_PATH_STYLE: bool({\n    desc: 'Whether to address the bucket as a path (endpoint/bucket) instead of a subdomain. Set it to true for Garage and MinIO; false works for AWS S3.',\n    default: true,\n  }),\n  LOG_BUDGET_BASE_BYTES: num({\n    desc: 'Base of the per-job log accrual budget, in stored bytes (normalized NDJSON the server keeps, framing included). A job may always store this much before the time-based rate is added. Sized to hold a few inline agent_session entries (such as base64 images) before the shared per-job cap trips. Defaults to 32 MiB.',\n    default: 33_554_432,\n  }),\n  LOG_BUDGET_RATE_BYTES_PER_MINUTE: num({\n    desc: 'Stored bytes added to the per-job log budget for each minute since the first log append. Defaults to 1 MiB per minute.',\n    default: 1_048_576,\n  }),\n  LOG_STREAM_CLOSE_GRACE_SECONDS: num({\n    desc: 'How long to wait after a job reaches a terminal state before force-closing any of its log streams the runner never ended itself (it died, was capped, or its log spool failed). The wait lets a last in-flight chunk land before the stream is marked truncated. Defaults to 120 seconds.',\n    default: 120,\n  }),\n  LOG_COMPACTION_RECONCILE_STALE_SECONDS: num({\n    desc: 'How long a closed log stream may stay uncompacted before the reconcile cron re-drives it. The event-triggered compaction normally runs within seconds of close; this backstop only re-drives streams whose compaction never started or permanently failed. Defaults to 900 seconds (15 minutes).',\n    default: 900,\n  }),\n  LOG_STREAM_REAP_AFTER_SECONDS: num({\n    desc: 'How long a log stream may stay open before the reaper cron force-closes it as abandoned. This is the backstop for a stream a runner started after its job already went terminal (the one-shot close sweep had already run); the reaper marks it truncated so it re-enters the compaction and retention lifecycle. Must be greater than AUTH_JOB_LEASE_TOKEN_EXPIRES_IN (the job lease lifetime, default 90 minutes) so a still-valid lease can never be force-closed mid-append, which would silently truncate live logs. Defaults to 7200 seconds (2 hours).',\n    default: 7200,\n  }),\n  LOG_APPEND_BODY_LIMIT_BYTES: num({\n    desc: 'Maximum size of a single log append request body, in bytes. Output records cap at 16 KiB each, but one agent_session record carries a whole session entry (which can inline base64 images), so the body must hold a full LOG_MAX_SESSION_LINE_BYTES line plus framing. This also bounds how far a job can overshoot its log budget: the one append that crosses the budget is stored in full before the job is capped. Defaults to 8 MiB.',\n    default: 8_388_608,\n  }),\n  LOG_MAX_SESSION_LINE_BYTES: num({\n    desc: 'Maximum size of one agent_session log line (a verbatim agent session entry), in bytes. A larger line is rejected with 400, and the runner drops it with a gap marker instead of sending it. Sized generously for entries that inline large content such as base64 images. Must be at most LOG_APPEND_BODY_LIMIT_BYTES, since a line that cannot fit in one request body could never be accepted. Defaults to 4 MiB.',\n    default: 4_194_304,\n  }),\n  LOG_READ_URL_TTL_SECONDS: num({\n    desc: 'Lifetime, in seconds, of a presigned GET URL handed out for a compacted (cold) log read. The browser must finish fetching the object before it expires; raise it for slow links or large logs. Must be between 1 and 604800 (7 days, the longest a presigned URL can live). Defaults to 3600 (one hour).',\n    default: 3600,\n  }),\n  LOG_READ_INLINE_MAX_BYTES: num({\n    desc: 'Maximum number of stored log bytes returned in a single inline (hot) read response. A long-running step can buffer far more than this in Postgres before compaction, so a read is capped here and the client re-polls (following has_more) to drain the backlog before it tails. Bounds per-request memory. Must be at least 1. Defaults to 1 MiB.',\n    default: 1_048_576,\n  }),\n  LOG_RETENTION_DAYS: num({\n    desc: 'How many days a closed log stream is kept before the retention cron hard-deletes its stored object and database row. Our own worker enforces this (not bucket lifecycle rules), so behavior is identical across object stores. Must be a whole number of days, 1 or greater. Defaults to 90 days.',\n    default: 90,\n  }),\n});\n\nconst hasS3AccessKeyId = Boolean(config.LOG_STORAGE_S3_ACCESS_KEY_ID);\nconst hasS3SecretAccessKey = Boolean(config.LOG_STORAGE_S3_SECRET_ACCESS_KEY);\nif (hasS3AccessKeyId !== hasS3SecretAccessKey) {\n  throw new Error(\n    'LOG_STORAGE_S3_ACCESS_KEY_ID and LOG_STORAGE_S3_SECRET_ACCESS_KEY must be set together or both left unset.',\n  );\n}\n\n// SigV4 caps a presigned URL's lifetime at 7 days, so a larger TTL would fail at signing\n// time on every cold read. Reject out-of-range values at startup rather than per request.\nconst MAX_PRESIGN_TTL_SECONDS = 604_800;\nif (\n  config.LOG_READ_URL_TTL_SECONDS < 1 ||\n  config.LOG_READ_URL_TTL_SECONDS > MAX_PRESIGN_TTL_SECONDS\n) {\n  throw new Error(\n    `LOG_READ_URL_TTL_SECONDS must be between 1 and ${MAX_PRESIGN_TTL_SECONDS}; got ${config.LOG_READ_URL_TTL_SECONDS}`,\n  );\n}\nif (config.LOG_READ_INLINE_MAX_BYTES < 1) {\n  throw new Error(\n    `LOG_READ_INLINE_MAX_BYTES must be at least 1; got ${config.LOG_READ_INLINE_MAX_BYTES}`,\n  );\n}\n\n// A whole agent_session line is one record in one body. If the body limit is below the\n// line cap, a legitimate large session line could never fit in one request body — Fastify\n// would 413 it before the per-line validator runs. Fail fast at startup instead.\nif (config.LOG_APPEND_BODY_LIMIT_BYTES < config.LOG_MAX_SESSION_LINE_BYTES) {\n  throw new Error(\n    `LOG_APPEND_BODY_LIMIT_BYTES (${config.LOG_APPEND_BODY_LIMIT_BYTES}) must be >= LOG_MAX_SESSION_LINE_BYTES (${config.LOG_MAX_SESSION_LINE_BYTES}).`,\n  );\n}\n\n// `num` accepts zero, negative, and fractional values; reject them before they reach\n// `make_interval(days => ...)` and hard-delete the wrong streams.\nif (!Number.isInteger(config.LOG_RETENTION_DAYS) || config.LOG_RETENTION_DAYS < 1) {\n  throw new Error(\n    `LOG_RETENTION_DAYS (${config.LOG_RETENTION_DAYS}) must be a whole number of days >= 1.`,\n  );\n}\n\n// A stream still accepts appends until its job lease expires, so reaping it any sooner would\n// truncate live logs. Validate against the real lease TTL read from auth's config (not a\n// hardcoded floor), so raising AUTH_JOB_LEASE_TOKEN_EXPIRES_IN can never silently outrun the\n// reaper window.\nconst jobLeaseTtlSeconds = durationToSeconds(authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN);\nif (\n  !Number.isFinite(config.LOG_STREAM_REAP_AFTER_SECONDS) ||\n  config.LOG_STREAM_REAP_AFTER_SECONDS <= jobLeaseTtlSeconds\n) {\n  throw new Error(\n    `LOG_STREAM_REAP_AFTER_SECONDS (${config.LOG_STREAM_REAP_AFTER_SECONDS}) must be greater than the job lease TTL (${jobLeaseTtlSeconds}s, from AUTH_JOB_LEASE_TOKEN_EXPIRES_IN=${authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN}); a smaller value would let the reaper force-close a stream a still-valid lease is appending to and silently truncate live logs.`,\n  );\n}\n"],"names":["config","authConfig","bool","createConfig","num","str","url","durationToSeconds","LOG_STORAGE_S3_ENDPOINT","desc","default","LOG_STORAGE_S3_REGION","LOG_STORAGE_S3_BUCKET","LOG_STORAGE_S3_PREFIX","LOG_STORAGE_S3_ACCESS_KEY_ID","undefined","LOG_STORAGE_S3_SECRET_ACCESS_KEY","LOG_STORAGE_S3_FORCE_PATH_STYLE","LOG_BUDGET_BASE_BYTES","LOG_BUDGET_RATE_BYTES_PER_MINUTE","LOG_STREAM_CLOSE_GRACE_SECONDS","LOG_COMPACTION_RECONCILE_STALE_SECONDS","LOG_STREAM_REAP_AFTER_SECONDS","LOG_APPEND_BODY_LIMIT_BYTES","LOG_MAX_SESSION_LINE_BYTES","LOG_READ_URL_TTL_SECONDS","LOG_READ_INLINE_MAX_BYTES","LOG_RETENTION_DAYS","hasS3AccessKeyId","Boolean","hasS3SecretAccessKey","Error","MAX_PRESIGN_TTL_SECONDS","Number","isInteger","jobLeaseTtlSeconds","AUTH_JOB_LEASE_TOKEN_EXPIRES_IN","isFinite"],"mappings":"AAAA,SAAQA,UAAUC,UAAU,QAAO,2BAA2B;AAC9D,SAAQC,IAAI,EAAEC,YAAY,EAAEC,GAAG,EAAEC,GAAG,EAAEC,GAAG,QAAO,kBAAkB;AAClE,SAAQC,iBAAiB,QAAO,oBAAoB;AAEpD,OAAO,MAAMP,SAASG,aAAa;IACjCK,yBAAyBF,IAAI;QAC3BG,MAAM;QACNC,SAAS;IACX;IACAC,uBAAuBN,IAAI;QACzBI,MAAM;QACNC,SAAS;IACX;IACAE,uBAAuBP,IAAI;QACzBI,MAAM;QACNC,SAAS;IACX;IACAG,uBAAuBR,IAAI;QACzBI,MAAM;QACNC,SAAS;IACX;IACAI,8BAA8BT,IAAI;QAChCI,MAAM;QACNC,SAASK;IACX;IACAC,kCAAkCX,IAAI;QACpCI,MAAM;QACNC,SAASK;IACX;IACAE,iCAAiCf,KAAK;QACpCO,MAAM;QACNC,SAAS;IACX;IACAQ,uBAAuBd,IAAI;QACzBK,MAAM;QACNC,SAAS;IACX;IACAS,kCAAkCf,IAAI;QACpCK,MAAM;QACNC,SAAS;IACX;IACAU,gCAAgChB,IAAI;QAClCK,MAAM;QACNC,SAAS;IACX;IACAW,wCAAwCjB,IAAI;QAC1CK,MAAM;QACNC,SAAS;IACX;IACAY,+BAA+BlB,IAAI;QACjCK,MAAM;QACNC,SAAS;IACX;IACAa,6BAA6BnB,IAAI;QAC/BK,MAAM;QACNC,SAAS;IACX;IACAc,4BAA4BpB,IAAI;QAC9BK,MAAM;QACNC,SAAS;IACX;IACAe,0BAA0BrB,IAAI;QAC5BK,MAAM;QACNC,SAAS;IACX;IACAgB,2BAA2BtB,IAAI;QAC7BK,MAAM;QACNC,SAAS;IACX;IACAiB,oBAAoBvB,IAAI;QACtBK,MAAM;QACNC,SAAS;IACX;AACF,GAAG;AAEH,MAAMkB,mBAAmBC,QAAQ7B,OAAOc,4BAA4B;AACpE,MAAMgB,uBAAuBD,QAAQ7B,OAAOgB,gCAAgC;AAC5E,IAAIY,qBAAqBE,sBAAsB;IAC7C,MAAM,IAAIC,MACR;AAEJ;AAEA,yFAAyF;AACzF,0FAA0F;AAC1F,MAAMC,0BAA0B;AAChC,IACEhC,OAAOyB,wBAAwB,GAAG,KAClCzB,OAAOyB,wBAAwB,GAAGO,yBAClC;IACA,MAAM,IAAID,MACR,CAAC,+CAA+C,EAAEC,wBAAwB,MAAM,EAAEhC,OAAOyB,wBAAwB,EAAE;AAEvH;AACA,IAAIzB,OAAO0B,yBAAyB,GAAG,GAAG;IACxC,MAAM,IAAIK,MACR,CAAC,kDAAkD,EAAE/B,OAAO0B,yBAAyB,EAAE;AAE3F;AAEA,uFAAuF;AACvF,0FAA0F;AAC1F,iFAAiF;AACjF,IAAI1B,OAAOuB,2BAA2B,GAAGvB,OAAOwB,0BAA0B,EAAE;IAC1E,MAAM,IAAIO,MACR,CAAC,6BAA6B,EAAE/B,OAAOuB,2BAA2B,CAAC,yCAAyC,EAAEvB,OAAOwB,0BAA0B,CAAC,EAAE,CAAC;AAEvJ;AAEA,qFAAqF;AACrF,kEAAkE;AAClE,IAAI,CAACS,OAAOC,SAAS,CAAClC,OAAO2B,kBAAkB,KAAK3B,OAAO2B,kBAAkB,GAAG,GAAG;IACjF,MAAM,IAAII,MACR,CAAC,oBAAoB,EAAE/B,OAAO2B,kBAAkB,CAAC,sCAAsC,CAAC;AAE5F;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,6FAA6F;AAC7F,iBAAiB;AACjB,MAAMQ,qBAAqB5B,kBAAkBN,WAAWmC,+BAA+B;AACvF,IACE,CAACH,OAAOI,QAAQ,CAACrC,OAAOsB,6BAA6B,KACrDtB,OAAOsB,6BAA6B,IAAIa,oBACxC;IACA,MAAM,IAAIJ,MACR,CAAC,+BAA+B,EAAE/B,OAAOsB,6BAA6B,CAAC,0CAA0C,EAAEa,mBAAmB,wCAAwC,EAAElC,WAAWmC,+BAA+B,CAAC,iIAAiI,CAAC;AAEjW"}

package/dist/core/append-logs.d.ts

@@ -0,0 +1,28 @@
+import { Buffer } from 'node:buffer';
+type StepLookup = (stepId: string) => Promise<{
+    config: Record<string, unknown>;
+} | undefined>;
+export declare function setStepLookupForTesting(lookup: StepLookup): () => void;
+export interface AppendLogsParams {
+    jobId: string;
+    workspaceId: string;
+    projectId: string;
+    workflowRunAttemptId: string;
+    stepId: string;
+    attempt: number;
+    offset: number;
+    body: Buffer;
+}
+export interface AppendLogsResult {
+    committedLength: number;
+    capped: boolean;
+}
+/**
+ * Concurrency is serialized through Postgres row locks taken implicitly by the
+ * conditional UPDATEs, not an explicit `SELECT ... FOR UPDATE`. Appends for one
+ * job contend on its single accounting row, so the path is multi-instance safe
+ * but not lock-free.
+ */
+export declare function appendLogs(params: AppendLogsParams): Promise<AppendLogsResult>;
+export {};
+//# sourceMappingURL=append-logs.d.ts.map

package/dist/core/append-logs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"append-logs.d.ts","sourceRoot":"","sources":["../../src/core/append-logs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AAgCnC,KAAK,UAAU,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAC,GAAG,SAAS,CAAC,CAAC;AAI7F,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,IAAI,CAMtE;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACjB;AAmND;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAmGpF"}

package/dist/core/append-logs.js

@@ -0,0 +1,318 @@
+import { Buffer } from 'node:buffer';
+import { DEFAULT_HARNESS, harnessSchema } from '@shipfox/api-agent-dto';
+import { parseLogRecordLine, parseRawLogRecordLine } from '@shipfox/api-logs-dto';
+import { getStepById } from '@shipfox/api-workflows';
+import { logger } from '@shipfox/node-opentelemetry';
+import { config } from '#config.js';
+import { accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped } from '#db/accounting.js';
+import { insertChunk } from '#db/chunks.js';
+import { db } from '#db/db.js';
+import { casExtendCommittedLength, getAttemptStream, getOrCreateAttemptStreamWithStatus, setDeclaredTotalBytes } from '#db/streams.js';
+import { recordAppendedCount, streamClosedCount, streamOpenedCount } from '#metrics/instance.js';
+import { allowedBudget } from './budget.js';
+import { closeStream, controlTombstone } from './close-stream.js';
+import { MalformedLogChunkError, OffsetGapError } from './errors.js';
+import { parseSessionRecord } from './session/parse-session.js';
+let stepLookup = getStepById;
+export function setStepLookupForTesting(lookup) {
+    const previous = stepLookup;
+    stepLookup = lookup;
+    return ()=>{
+        stepLookup = previous;
+    };
+}
+/**
+ * Pure pre-transaction parse. Requires whole, newline-terminated lines so
+ * `committed_length` always lands on a line boundary (one body = one chunk = whole
+ * lines; no line ever spans two chunks). Runs before any lock is taken, so a
+ * malformed or large body never holds a row. The offset CAS uses the raw append byte length; the
+ * budget charges the normalized body built from these parsed records.
+ *
+ * Each line is validated against the raw record union (a forged server-only
+ * `capped`/`runner_lost` fails here); the declared total is pulled from an `end`
+ * record. A line that is a valid server-only record under the read union surfaces
+ * its type via `forgedType` for the narrowed audit warn.
+ */ function parseAppendBody(body) {
+    if (body.length === 0) return {
+        records: [],
+        hasAgentSessionRecord: false
+    };
+    const text = body.toString('utf8');
+    if (!text.endsWith('\n')) {
+        throw new MalformedLogChunkError('append body must end with a newline (whole records only)');
+    }
+    const lines = text.split('\n');
+    lines.pop();
+    let declaredTotalBytes;
+    const records = [];
+    let hasAgentSessionRecord = false;
+    for (const line of lines){
+        let record;
+        try {
+            record = parseRawLogRecordLine(line);
+        } catch  {
+            throw new MalformedLogChunkError('append body contains an invalid NDJSON record', detectForgedType(line));
+        }
+        records.push(record);
+        if (record.type === 'end') {
+            declaredTotalBytes = record.total_bytes;
+        }
+        // An agent_session line is one whole entry in one record; bound its size here (the DTO
+        // schema leaves `data` uncapped because it cannot read this runtime config). An over-cap
+        // line is rejected so a single entry can never blow the request body or the spool window.
+        if (record.type === 'agent_session' && Buffer.byteLength(line, 'utf8') > config.LOG_MAX_SESSION_LINE_BYTES) {
+            throw new MalformedLogChunkError(`agent_session line exceeds ${config.LOG_MAX_SESSION_LINE_BYTES} bytes`);
+        }
+        if (record.type === 'agent_session') {
+            hasAgentSessionRecord = true;
+        }
+    }
+    return declaredTotalBytes === undefined ? {
+        records,
+        hasAgentSessionRecord
+    } : {
+        declaredTotalBytes,
+        records,
+        hasAgentSessionRecord
+    };
+}
+/**
+ * A line that fails the raw write union but is a valid record under the read union
+ * can only be a server-only `capped`/`runner_lost` tombstone — i.e. a forgery
+ * attempt. Returns its type for the audit warn, or undefined for plain garbage.
+ */ function detectForgedType(line) {
+    try {
+        return parseLogRecordLine(line).type;
+    } catch  {
+        return undefined;
+    }
+}
+async function getSessionHarness(stepId) {
+    const step = await stepLookup(stepId);
+    const parsed = harnessSchema.safeParse(step?.config.harness);
+    return parsed.success ? parsed.data : DEFAULT_HARNESS;
+}
+/**
+ * Empty-body heartbeat: report the current committed length without materializing
+ * a stream, so a runner cannot mint unbounded rows with empty appends.
+ */ async function readHeartbeat(tx, params) {
+    const existing = await getAttemptStream(tx, {
+        jobId: params.jobId,
+        stepId: params.stepId,
+        attempt: params.attempt
+    });
+    return {
+        committedLength: existing?.committedLength ?? 0,
+        capped: await isJobCapped(tx, params.jobId)
+    };
+}
+/**
+ * Accrues the stored bytes, persists the chunk, and trips the per-job cap when
+ * this append crosses the budget. Runs only after the offset-CAS extended
+ * `committed_length`, so the committed length already reflects the accepted bytes.
+ */ async function storeChunk(tx, { params, streamId, body, committedLength, declaredTotalBytes }) {
+    await ensureJobAccounting(tx, {
+        jobId: params.jobId,
+        workspaceId: params.workspaceId
+    });
+    const storedByteLen = body.length;
+    const accrued = await accrueStoredBytes(tx, {
+        jobId: params.jobId,
+        delta: storedByteLen
+    });
+    // Already capped: accept-and-drop. committed_length has advanced so the runner
+    // drains its spool cleanly instead of retry-looping; nothing is stored.
+    if (!accrued) return {
+        committedLength,
+        capped: true,
+        stored: false,
+        recordCounts: {}
+    };
+    await insertChunk(tx, {
+        streamId,
+        streamOffset: params.offset,
+        byteLen: storedByteLen,
+        data: body,
+        origin: 'runner'
+    });
+    if (declaredTotalBytes !== undefined) {
+        await setDeclaredTotalBytes(tx, {
+            streamId,
+            declaredTotalBytes
+        });
+    }
+    const allowed = allowedBudget({
+        baseBytes: config.LOG_BUDGET_BASE_BYTES,
+        ratePerMinuteBytes: config.LOG_BUDGET_RATE_BYTES_PER_MINUTE,
+        elapsedMs: Date.now() - accrued.startedAt.getTime()
+    });
+    if (accrued.used <= allowed) {
+        return {
+            committedLength,
+            capped: false,
+            stored: true,
+            recordCounts: {}
+        };
+    }
+    // Over budget. No hard ceiling: this crossing append is stored in full (overshoot
+    // bounded by one body). Claim the cap once and inject an in-band `capped` tombstone
+    // for the winner.
+    const won = await claimCap(tx, params.jobId);
+    if (won) {
+        const tombstone = controlTombstone('capped');
+        await insertChunk(tx, {
+            streamId,
+            streamOffset: committedLength,
+            byteLen: tombstone.length,
+            data: tombstone,
+            origin: 'control'
+        });
+    }
+    return {
+        committedLength,
+        capped: true,
+        stored: true,
+        recordCounts: won ? {
+            capped: 1
+        } : {}
+    };
+}
+function buildStoredBody(records, harness) {
+    const storedRecords = [];
+    for (const record of records){
+        if (record.type !== 'agent_session') {
+            storedRecords.push(record);
+            continue;
+        }
+        for (const row of parseSessionRecord(agentSessionRecord(record), harness)){
+            storedRecords.push({
+                v: 1,
+                ts: row.timestamp,
+                type: 'agent_session',
+                row
+            });
+        }
+    }
+    const body = Buffer.from(storedRecords.map((record)=>`${JSON.stringify(record)}\n`).join(''));
+    const recordCounts = {};
+    for (const record of storedRecords){
+        recordCounts[record.type] = (recordCounts[record.type] ?? 0) + 1;
+    }
+    return {
+        body,
+        recordCounts
+    };
+}
+function agentSessionRecord(record) {
+    return {
+        data: record.data,
+        ts: record.ts
+    };
+}
+/**
+ * Concurrency is serialized through Postgres row locks taken implicitly by the
+ * conditional UPDATEs, not an explicit `SELECT ... FOR UPDATE`. Appends for one
+ * job contend on its single accounting row, so the path is multi-instance safe
+ * but not lock-free.
+ */ export async function appendLogs(params) {
+    let parsed;
+    try {
+        parsed = parseAppendBody(params.body);
+    } catch (error) {
+        // Narrowed audit: only the detectable forgery case, never the payload or a token.
+        if (error instanceof MalformedLogChunkError && error.forgedType !== undefined) {
+            logger().warn({
+                jobId: params.jobId,
+                stepId: params.stepId,
+                offendingType: error.forgedType
+            }, 'Rejected forged server-only log record on append');
+        }
+        throw error;
+    }
+    const { declaredTotalBytes } = parsed;
+    const sessionHarness = parsed.hasAgentSessionRecord ? await getSessionHarness(params.stepId) : DEFAULT_HARNESS;
+    const stored = buildStoredBody(parsed.records, sessionHarness);
+    const commitByteLen = params.body.length;
+    const metrics = {
+        recordCounts: {},
+        streamClosedReason: undefined,
+        streamOpened: false
+    };
+    const result = await db().transaction(async (tx)=>{
+        if (commitByteLen === 0) return readHeartbeat(tx, params);
+        const { created, stream } = await getOrCreateAttemptStreamWithStatus(tx, {
+            jobId: params.jobId,
+            stepId: params.stepId,
+            attempt: params.attempt,
+            workspaceId: params.workspaceId,
+            projectId: params.projectId,
+            workflowRunAttemptId: params.workflowRunAttemptId
+        });
+        metrics.streamOpened = created;
+        // Closed stream (the runner's end already landed, or the job-terminated sweep ran):
+        // accept-and-drop so a late chunk can never race compaction. committed_length is
+        // frozen at close, so this reports the final offset and the runner stops cleanly.
+        if (stream.state === 'closed') {
+            return {
+                committedLength: stream.committedLength,
+                capped: await isJobCapped(tx, params.jobId)
+            };
+        }
+        const cas = await casExtendCommittedLength(tx, {
+            streamId: stream.id,
+            offset: params.offset,
+            byteLen: commitByteLen
+        });
+        if (cas.outcome === 'gap') throw new OffsetGapError(cas.committedLength);
+        if (cas.outcome === 'retry') {
+            return {
+                committedLength: cas.committedLength,
+                capped: await isJobCapped(tx, params.jobId)
+            };
+        }
+        const { recordCounts, stored: chunkStored, ...result } = await storeChunk(tx, {
+            params,
+            streamId: stream.id,
+            body: stored.body,
+            committedLength: cas.committedLength,
+            declaredTotalBytes
+        });
+        if (chunkStored) {
+            addRecordCounts(metrics.recordCounts, stored.recordCounts);
+        }
+        addRecordCounts(metrics.recordCounts, recordCounts);
+        // The runner's end record was committed in this append (offset-CAS guarantees
+        // everything before it is already committed), so the stream is whole. Declared-close
+        // it in-band so compaction starts at once instead of waiting for the timeout sweep.
+        // Only when the chunk was actually stored: an end body dropped because the job was
+        // already capped persists nothing, so the stream is not whole and stays open for the
+        // timeout sweep to close it as truncated.
+        if (declaredTotalBytes !== undefined && chunkStored) {
+            const closed = await closeStream(tx, {
+                streamId: stream.id,
+                reason: 'declared'
+            });
+            if (closed) metrics.streamClosedReason = 'declared';
+        }
+        return result;
+    });
+    if (metrics.streamOpened) streamOpenedCount.add(1);
+    for (const [kind, count] of Object.entries(metrics.recordCounts)){
+        if (count > 0) recordAppendedCount.add(count, {
+            kind: kind
+        });
+    }
+    if (metrics.streamClosedReason) {
+        streamClosedCount.add(1, {
+            reason: metrics.streamClosedReason
+        });
+    }
+    return result;
+}
+function addRecordCounts(target, source) {
+    for (const [kind, count] of Object.entries(source)){
+        target[kind] = (target[kind] ?? 0) + (count ?? 0);
+    }
+}
+
+//# sourceMappingURL=append-logs.js.map

package/dist/core/append-logs.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/core/append-logs.ts"],"sourcesContent":["import {Buffer} from 'node:buffer';\nimport {DEFAULT_HARNESS, type Harness, harnessSchema} from '@shipfox/api-agent-dto';\nimport {\n  type LogRecord,\n  parseLogRecordLine,\n  parseRawLogRecordLine,\n  type RawLogRecord,\n} from '@shipfox/api-logs-dto';\nimport {getStepById} from '@shipfox/api-workflows';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {config} from '#config.js';\nimport {accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped} from '#db/accounting.js';\nimport {insertChunk} from '#db/chunks.js';\nimport {db, type Transaction} from '#db/db.js';\nimport {\n  casExtendCommittedLength,\n  getAttemptStream,\n  getOrCreateAttemptStreamWithStatus,\n  setDeclaredTotalBytes,\n} from '#db/streams.js';\nimport {\n  type LogRecordMetricKind,\n  recordAppendedCount,\n  streamClosedCount,\n  streamOpenedCount,\n} from '#metrics/instance.js';\nimport {allowedBudget} from './budget.js';\nimport {closeStream, controlTombstone} from './close-stream.js';\nimport {MalformedLogChunkError, OffsetGapError} from './errors.js';\nimport {parseSessionRecord} from './session/parse-session.js';\nimport type {AgentSessionRecord} from './session/session-record.js';\n\ntype StepLookup = (stepId: string) => Promise<{config: Record<string, unknown>} | undefined>;\n\nlet stepLookup: StepLookup = getStepById;\n\nexport function setStepLookupForTesting(lookup: StepLookup): () => void {\n  const previous = stepLookup;\n  stepLookup = lookup;\n  return () => {\n    stepLookup = previous;\n  };\n}\n\nexport interface AppendLogsParams {\n  jobId: string;\n  workspaceId: string;\n  projectId: string;\n  workflowRunAttemptId: string;\n  stepId: string;\n  attempt: number;\n  offset: number;\n  body: Buffer;\n}\n\nexport interface AppendLogsResult {\n  committedLength: number;\n  capped: boolean;\n}\n\ninterface ParsedBody {\n  declaredTotalBytes?: number;\n  records: RawLogRecord[];\n  hasAgentSessionRecord: boolean;\n}\n\n/**\n * Pure pre-transaction parse. Requires whole, newline-terminated lines so\n * `committed_length` always lands on a line boundary (one body = one chunk = whole\n * lines; no line ever spans two chunks). Runs before any lock is taken, so a\n * malformed or large body never holds a row. The offset CAS uses the raw append byte length; the\n * budget charges the normalized body built from these parsed records.\n *\n * Each line is validated against the raw record union (a forged server-only\n * `capped`/`runner_lost` fails here); the declared total is pulled from an `end`\n * record. A line that is a valid server-only record under the read union surfaces\n * its type via `forgedType` for the narrowed audit warn.\n */\nfunction parseAppendBody(body: Buffer): ParsedBody {\n  if (body.length === 0) return {records: [], hasAgentSessionRecord: false};\n\n  const text = body.toString('utf8');\n  if (!text.endsWith('\\n')) {\n    throw new MalformedLogChunkError('append body must end with a newline (whole records only)');\n  }\n  const lines = text.split('\\n');\n  lines.pop();\n\n  let declaredTotalBytes: number | undefined;\n  const records: RawLogRecord[] = [];\n  let hasAgentSessionRecord = false;\n  for (const line of lines) {\n    let record: ReturnType<typeof parseRawLogRecordLine>;\n    try {\n      record = parseRawLogRecordLine(line);\n    } catch {\n      throw new MalformedLogChunkError(\n        'append body contains an invalid NDJSON record',\n        detectForgedType(line),\n      );\n    }\n    records.push(record);\n    if (record.type === 'end') {\n      declaredTotalBytes = record.total_bytes;\n    }\n    // An agent_session line is one whole entry in one record; bound its size here (the DTO\n    // schema leaves `data` uncapped because it cannot read this runtime config). An over-cap\n    // line is rejected so a single entry can never blow the request body or the spool window.\n    if (\n      record.type === 'agent_session' &&\n      Buffer.byteLength(line, 'utf8') > config.LOG_MAX_SESSION_LINE_BYTES\n    ) {\n      throw new MalformedLogChunkError(\n        `agent_session line exceeds ${config.LOG_MAX_SESSION_LINE_BYTES} bytes`,\n      );\n    }\n    if (record.type === 'agent_session') {\n      hasAgentSessionRecord = true;\n    }\n  }\n\n  return declaredTotalBytes === undefined\n    ? {records, hasAgentSessionRecord}\n    : {declaredTotalBytes, records, hasAgentSessionRecord};\n}\n\n/**\n * A line that fails the raw write union but is a valid record under the read union\n * can only be a server-only `capped`/`runner_lost` tombstone — i.e. a forgery\n * attempt. Returns its type for the audit warn, or undefined for plain garbage.\n */\nfunction detectForgedType(line: string): string | undefined {\n  try {\n    return parseLogRecordLine(line).type;\n  } catch {\n    return undefined;\n  }\n}\n\nasync function getSessionHarness(stepId: string): Promise<Harness> {\n  const step = await stepLookup(stepId);\n  const parsed = harnessSchema.safeParse(step?.config.harness);\n  return parsed.success ? parsed.data : DEFAULT_HARNESS;\n}\n\n/**\n * Empty-body heartbeat: report the current committed length without materializing\n * a stream, so a runner cannot mint unbounded rows with empty appends.\n */\nasync function readHeartbeat(tx: Transaction, params: AppendLogsParams): Promise<AppendLogsResult> {\n  const existing = await getAttemptStream(tx, {\n    jobId: params.jobId,\n    stepId: params.stepId,\n    attempt: params.attempt,\n  });\n  return {\n    committedLength: existing?.committedLength ?? 0,\n    capped: await isJobCapped(tx, params.jobId),\n  };\n}\n\ninterface StoreChunkParams {\n  params: AppendLogsParams;\n  streamId: string;\n  body: Buffer;\n  committedLength: number;\n  declaredTotalBytes: number | undefined;\n}\n\ninterface StoreChunkResult extends AppendLogsResult {\n  /**\n   * Whether the runner chunk was persisted. False only when the job was already\n   * capped and the body (including any `end` record) was dropped, so the stream is\n   * not whole and must not be declared-closed.\n   */\n  stored: boolean;\n  recordCounts: Partial<Record<LogRecord['type'], number>>;\n}\n\n/**\n * Accrues the stored bytes, persists the chunk, and trips the per-job cap when\n * this append crosses the budget. Runs only after the offset-CAS extended\n * `committed_length`, so the committed length already reflects the accepted bytes.\n */\nasync function storeChunk(\n  tx: Transaction,\n  {params, streamId, body, committedLength, declaredTotalBytes}: StoreChunkParams,\n): Promise<StoreChunkResult> {\n  await ensureJobAccounting(tx, {jobId: params.jobId, workspaceId: params.workspaceId});\n  const storedByteLen = body.length;\n  const accrued = await accrueStoredBytes(tx, {jobId: params.jobId, delta: storedByteLen});\n\n  // Already capped: accept-and-drop. committed_length has advanced so the runner\n  // drains its spool cleanly instead of retry-looping; nothing is stored.\n  if (!accrued) return {committedLength, capped: true, stored: false, recordCounts: {}};\n\n  await insertChunk(tx, {\n    streamId,\n    streamOffset: params.offset,\n    byteLen: storedByteLen,\n    data: body,\n    origin: 'runner',\n  });\n  if (declaredTotalBytes !== undefined) {\n    await setDeclaredTotalBytes(tx, {streamId, declaredTotalBytes});\n  }\n\n  const allowed = allowedBudget({\n    baseBytes: config.LOG_BUDGET_BASE_BYTES,\n    ratePerMinuteBytes: config.LOG_BUDGET_RATE_BYTES_PER_MINUTE,\n    elapsedMs: Date.now() - accrued.startedAt.getTime(),\n  });\n  if (accrued.used <= allowed) {\n    return {committedLength, capped: false, stored: true, recordCounts: {}};\n  }\n\n  // Over budget. No hard ceiling: this crossing append is stored in full (overshoot\n  // bounded by one body). Claim the cap once and inject an in-band `capped` tombstone\n  // for the winner.\n  const won = await claimCap(tx, params.jobId);\n  if (won) {\n    const tombstone = controlTombstone('capped');\n    await insertChunk(tx, {\n      streamId,\n      streamOffset: committedLength,\n      byteLen: tombstone.length,\n      data: tombstone,\n      origin: 'control',\n    });\n  }\n  return {\n    committedLength,\n    capped: true,\n    stored: true,\n    recordCounts: won ? {capped: 1} : {},\n  };\n}\n\nfunction buildStoredBody(\n  records: readonly RawLogRecord[],\n  harness: Harness,\n): {body: Buffer; recordCounts: Partial<Record<LogRecord['type'], number>>} {\n  const storedRecords: LogRecord[] = [];\n  for (const record of records) {\n    if (record.type !== 'agent_session') {\n      storedRecords.push(record);\n      continue;\n    }\n\n    for (const row of parseSessionRecord(agentSessionRecord(record), harness)) {\n      storedRecords.push({v: 1, ts: row.timestamp, type: 'agent_session', row});\n    }\n  }\n\n  const body = Buffer.from(storedRecords.map((record) => `${JSON.stringify(record)}\\n`).join(''));\n  const recordCounts: Partial<Record<LogRecord['type'], number>> = {};\n  for (const record of storedRecords) {\n    recordCounts[record.type] = (recordCounts[record.type] ?? 0) + 1;\n  }\n\n  return {body, recordCounts};\n}\n\nfunction agentSessionRecord(\n  record: Extract<RawLogRecord, {type: 'agent_session'}>,\n): AgentSessionRecord {\n  return {data: record.data, ts: record.ts};\n}\n\n/**\n * Concurrency is serialized through Postgres row locks taken implicitly by the\n * conditional UPDATEs, not an explicit `SELECT ... FOR UPDATE`. Appends for one\n * job contend on its single accounting row, so the path is multi-instance safe\n * but not lock-free.\n */\nexport async function appendLogs(params: AppendLogsParams): Promise<AppendLogsResult> {\n  let parsed: ParsedBody;\n  try {\n    parsed = parseAppendBody(params.body);\n  } catch (error) {\n    // Narrowed audit: only the detectable forgery case, never the payload or a token.\n    if (error instanceof MalformedLogChunkError && error.forgedType !== undefined) {\n      logger().warn(\n        {\n          jobId: params.jobId,\n          stepId: params.stepId,\n          offendingType: error.forgedType,\n        },\n        'Rejected forged server-only log record on append',\n      );\n    }\n    throw error;\n  }\n  const {declaredTotalBytes} = parsed;\n  const sessionHarness = parsed.hasAgentSessionRecord\n    ? await getSessionHarness(params.stepId)\n    : DEFAULT_HARNESS;\n  const stored = buildStoredBody(parsed.records, sessionHarness);\n  const commitByteLen = params.body.length;\n  const metrics = {\n    recordCounts: {} as Partial<Record<LogRecordMetricKind, number>>,\n    streamClosedReason: undefined as 'declared' | undefined,\n    streamOpened: false,\n  };\n\n  const result = await db().transaction(async (tx) => {\n    if (commitByteLen === 0) return readHeartbeat(tx, params);\n\n    const {created, stream} = await getOrCreateAttemptStreamWithStatus(tx, {\n      jobId: params.jobId,\n      stepId: params.stepId,\n      attempt: params.attempt,\n      workspaceId: params.workspaceId,\n      projectId: params.projectId,\n      workflowRunAttemptId: params.workflowRunAttemptId,\n    });\n    metrics.streamOpened = created;\n\n    // Closed stream (the runner's end already landed, or the job-terminated sweep ran):\n    // accept-and-drop so a late chunk can never race compaction. committed_length is\n    // frozen at close, so this reports the final offset and the runner stops cleanly.\n    if (stream.state === 'closed') {\n      return {committedLength: stream.committedLength, capped: await isJobCapped(tx, params.jobId)};\n    }\n\n    const cas = await casExtendCommittedLength(tx, {\n      streamId: stream.id,\n      offset: params.offset,\n      byteLen: commitByteLen,\n    });\n    if (cas.outcome === 'gap') throw new OffsetGapError(cas.committedLength);\n    if (cas.outcome === 'retry') {\n      return {committedLength: cas.committedLength, capped: await isJobCapped(tx, params.jobId)};\n    }\n\n    const {\n      recordCounts,\n      stored: chunkStored,\n      ...result\n    } = await storeChunk(tx, {\n      params,\n      streamId: stream.id,\n      body: stored.body,\n      committedLength: cas.committedLength,\n      declaredTotalBytes,\n    });\n    if (chunkStored) {\n      addRecordCounts(metrics.recordCounts, stored.recordCounts);\n    }\n    addRecordCounts(metrics.recordCounts, recordCounts);\n\n    // The runner's end record was committed in this append (offset-CAS guarantees\n    // everything before it is already committed), so the stream is whole. Declared-close\n    // it in-band so compaction starts at once instead of waiting for the timeout sweep.\n    // Only when the chunk was actually stored: an end body dropped because the job was\n    // already capped persists nothing, so the stream is not whole and stays open for the\n    // timeout sweep to close it as truncated.\n    if (declaredTotalBytes !== undefined && chunkStored) {\n      const closed = await closeStream(tx, {streamId: stream.id, reason: 'declared'});\n      if (closed) metrics.streamClosedReason = 'declared';\n    }\n\n    return result;\n  });\n\n  if (metrics.streamOpened) streamOpenedCount.add(1);\n  for (const [kind, count] of Object.entries(metrics.recordCounts)) {\n    if (count > 0) recordAppendedCount.add(count, {kind: kind as LogRecordMetricKind});\n  }\n  if (metrics.streamClosedReason) {\n    streamClosedCount.add(1, {reason: metrics.streamClosedReason});\n  }\n\n  return result;\n}\n\nfunction addRecordCounts(\n  target: Partial<Record<LogRecordMetricKind, number>>,\n  source: Partial<Record<LogRecordMetricKind, number>>,\n): void {\n  for (const [kind, count] of Object.entries(source)) {\n    target[kind as LogRecordMetricKind] = (target[kind as LogRecordMetricKind] ?? 0) + (count ?? 0);\n  }\n}\n"],"names":["Buffer","DEFAULT_HARNESS","harnessSchema","parseLogRecordLine","parseRawLogRecordLine","getStepById","logger","config","accrueStoredBytes","claimCap","ensureJobAccounting","isJobCapped","insertChunk","db","casExtendCommittedLength","getAttemptStream","getOrCreateAttemptStreamWithStatus","setDeclaredTotalBytes","recordAppendedCount","streamClosedCount","streamOpenedCount","allowedBudget","closeStream","controlTombstone","MalformedLogChunkError","OffsetGapError","parseSessionRecord","stepLookup","setStepLookupForTesting","lookup","previous","parseAppendBody","body","length","records","hasAgentSessionRecord","text","toString","endsWith","lines","split","pop","declaredTotalBytes","line","record","detectForgedType","push","type","total_bytes","byteLength","LOG_MAX_SESSION_LINE_BYTES","undefined","getSessionHarness","stepId","step","parsed","safeParse","harness","success","data","readHeartbeat","tx","params","existing","jobId","attempt","committedLength","capped","storeChunk","streamId","workspaceId","storedByteLen","accrued","delta","stored","recordCounts","streamOffset","offset","byteLen","origin","allowed","baseBytes","LOG_BUDGET_BASE_BYTES","ratePerMinuteBytes","LOG_BUDGET_RATE_BYTES_PER_MINUTE","elapsedMs","Date","now","startedAt","getTime","used","won","tombstone","buildStoredBody","storedRecords","row","agentSessionRecord","v","ts","timestamp","from","map","JSON","stringify","join","appendLogs","error","forgedType","warn","offendingType","sessionHarness","commitByteLen","metrics","streamClosedReason","streamOpened","result","transaction","created","stream","projectId","workflowRunAttemptId","state","cas","id","outcome","chunkStored","addRecordCounts","closed","reason","add","kind","count","Object","entries","target","source"],"mappings":"AAAA,SAAQA,MAAM,QAAO,cAAc;AACnC,SAAQC,eAAe,EAAgBC,aAAa,QAAO,yBAAyB;AACpF,SAEEC,kBAAkB,EAClBC,qBAAqB,QAEhB,wBAAwB;AAC/B,SAAQC,WAAW,QAAO,yBAAyB;AACnD,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAQC,iBAAiB,EAAEC,QAAQ,EAAEC,mBAAmB,EAAEC,WAAW,QAAO,oBAAoB;AAChG,SAAQC,WAAW,QAAO,gBAAgB;AAC1C,SAAQC,EAAE,QAAyB,YAAY;AAC/C,SACEC,wBAAwB,EACxBC,gBAAgB,EAChBC,kCAAkC,EAClCC,qBAAqB,QAChB,iBAAiB;AACxB,SAEEC,mBAAmB,EACnBC,iBAAiB,EACjBC,iBAAiB,QACZ,uBAAuB;AAC9B,SAAQC,aAAa,QAAO,cAAc;AAC1C,SAAQC,WAAW,EAAEC,gBAAgB,QAAO,oBAAoB;AAChE,SAAQC,sBAAsB,EAAEC,cAAc,QAAO,cAAc;AACnE,SAAQC,kBAAkB,QAAO,6BAA6B;AAK9D,IAAIC,aAAyBtB;AAE7B,OAAO,SAASuB,wBAAwBC,MAAkB;IACxD,MAAMC,WAAWH;IACjBA,aAAaE;IACb,OAAO;QACLF,aAAaG;IACf;AACF;AAwBA;;;;;;;;;;;CAWC,GACD,SAASC,gBAAgBC,IAAY;IACnC,IAAIA,KAAKC,MAAM,KAAK,GAAG,OAAO;QAACC,SAAS,EAAE;QAAEC,uBAAuB;IAAK;IAExE,MAAMC,OAAOJ,KAAKK,QAAQ,CAAC;IAC3B,IAAI,CAACD,KAAKE,QAAQ,CAAC,OAAO;QACxB,MAAM,IAAId,uBAAuB;IACnC;IACA,MAAMe,QAAQH,KAAKI,KAAK,CAAC;IACzBD,MAAME,GAAG;IAET,IAAIC;IACJ,MAAMR,UAA0B,EAAE;IAClC,IAAIC,wBAAwB;IAC5B,KAAK,MAAMQ,QAAQJ,MAAO;QACxB,IAAIK;QACJ,IAAI;YACFA,SAASxC,sBAAsBuC;QACjC,EAAE,OAAM;YACN,MAAM,IAAInB,uBACR,iDACAqB,iBAAiBF;QAErB;QACAT,QAAQY,IAAI,CAACF;QACb,IAAIA,OAAOG,IAAI,KAAK,OAAO;YACzBL,qBAAqBE,OAAOI,WAAW;QACzC;QACA,uFAAuF;QACvF,yFAAyF;QACzF,0FAA0F;QAC1F,IACEJ,OAAOG,IAAI,KAAK,mBAChB/C,OAAOiD,UAAU,CAACN,MAAM,UAAUpC,OAAO2C,0BAA0B,EACnE;YACA,MAAM,IAAI1B,uBACR,CAAC,2BAA2B,EAAEjB,OAAO2C,0BAA0B,CAAC,MAAM,CAAC;QAE3E;QACA,IAAIN,OAAOG,IAAI,KAAK,iBAAiB;YACnCZ,wBAAwB;QAC1B;IACF;IAEA,OAAOO,uBAAuBS,YAC1B;QAACjB;QAASC;IAAqB,IAC/B;QAACO;QAAoBR;QAASC;IAAqB;AACzD;AAEA;;;;CAIC,GACD,SAASU,iBAAiBF,IAAY;IACpC,IAAI;QACF,OAAOxC,mBAAmBwC,MAAMI,IAAI;IACtC,EAAE,OAAM;QACN,OAAOI;IACT;AACF;AAEA,eAAeC,kBAAkBC,MAAc;IAC7C,MAAMC,OAAO,MAAM3B,WAAW0B;IAC9B,MAAME,SAASrD,cAAcsD,SAAS,CAACF,MAAM/C,OAAOkD;IACpD,OAAOF,OAAOG,OAAO,GAAGH,OAAOI,IAAI,GAAG1D;AACxC;AAEA;;;CAGC,GACD,eAAe2D,cAAcC,EAAe,EAAEC,MAAwB;IACpE,MAAMC,WAAW,MAAMhD,iBAAiB8C,IAAI;QAC1CG,OAAOF,OAAOE,KAAK;QACnBX,QAAQS,OAAOT,MAAM;QACrBY,SAASH,OAAOG,OAAO;IACzB;IACA,OAAO;QACLC,iBAAiBH,UAAUG,mBAAmB;QAC9CC,QAAQ,MAAMxD,YAAYkD,IAAIC,OAAOE,KAAK;IAC5C;AACF;AAoBA;;;;CAIC,GACD,eAAeI,WACbP,EAAe,EACf,EAACC,MAAM,EAAEO,QAAQ,EAAErC,IAAI,EAAEkC,eAAe,EAAExB,kBAAkB,EAAmB;IAE/E,MAAMhC,oBAAoBmD,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAEM,aAAaR,OAAOQ,WAAW;IAAA;IACnF,MAAMC,gBAAgBvC,KAAKC,MAAM;IACjC,MAAMuC,UAAU,MAAMhE,kBAAkBqD,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAES,OAAOF;IAAa;IAEtF,+EAA+E;IAC/E,wEAAwE;IACxE,IAAI,CAACC,SAAS,OAAO;QAACN;QAAiBC,QAAQ;QAAMO,QAAQ;QAAOC,cAAc,CAAC;IAAC;IAEpF,MAAM/D,YAAYiD,IAAI;QACpBQ;QACAO,cAAcd,OAAOe,MAAM;QAC3BC,SAASP;QACTZ,MAAM3B;QACN+C,QAAQ;IACV;IACA,IAAIrC,uBAAuBS,WAAW;QACpC,MAAMlC,sBAAsB4C,IAAI;YAACQ;YAAU3B;QAAkB;IAC/D;IAEA,MAAMsC,UAAU3D,cAAc;QAC5B4D,WAAW1E,OAAO2E,qBAAqB;QACvCC,oBAAoB5E,OAAO6E,gCAAgC;QAC3DC,WAAWC,KAAKC,GAAG,KAAKf,QAAQgB,SAAS,CAACC,OAAO;IACnD;IACA,IAAIjB,QAAQkB,IAAI,IAAIV,SAAS;QAC3B,OAAO;YAACd;YAAiBC,QAAQ;YAAOO,QAAQ;YAAMC,cAAc,CAAC;QAAC;IACxE;IAEA,kFAAkF;IAClF,oFAAoF;IACpF,kBAAkB;IAClB,MAAMgB,MAAM,MAAMlF,SAASoD,IAAIC,OAAOE,KAAK;IAC3C,IAAI2B,KAAK;QACP,MAAMC,YAAYrE,iBAAiB;QACnC,MAAMX,YAAYiD,IAAI;YACpBQ;YACAO,cAAcV;YACdY,SAASc,UAAU3D,MAAM;YACzB0B,MAAMiC;YACNb,QAAQ;QACV;IACF;IACA,OAAO;QACLb;QACAC,QAAQ;QACRO,QAAQ;QACRC,cAAcgB,MAAM;YAACxB,QAAQ;QAAC,IAAI,CAAC;IACrC;AACF;AAEA,SAAS0B,gBACP3D,OAAgC,EAChCuB,OAAgB;IAEhB,MAAMqC,gBAA6B,EAAE;IACrC,KAAK,MAAMlD,UAAUV,QAAS;QAC5B,IAAIU,OAAOG,IAAI,KAAK,iBAAiB;YACnC+C,cAAchD,IAAI,CAACF;YACnB;QACF;QAEA,KAAK,MAAMmD,OAAOrE,mBAAmBsE,mBAAmBpD,SAASa,SAAU;YACzEqC,cAAchD,IAAI,CAAC;gBAACmD,GAAG;gBAAGC,IAAIH,IAAII,SAAS;gBAAEpD,MAAM;gBAAiBgD;YAAG;QACzE;IACF;IAEA,MAAM/D,OAAOhC,OAAOoG,IAAI,CAACN,cAAcO,GAAG,CAAC,CAACzD,SAAW,GAAG0D,KAAKC,SAAS,CAAC3D,QAAQ,EAAE,CAAC,EAAE4D,IAAI,CAAC;IAC3F,MAAM7B,eAA2D,CAAC;IAClE,KAAK,MAAM/B,UAAUkD,cAAe;QAClCnB,YAAY,CAAC/B,OAAOG,IAAI,CAAC,GAAG,AAAC4B,CAAAA,YAAY,CAAC/B,OAAOG,IAAI,CAAC,IAAI,CAAA,IAAK;IACjE;IAEA,OAAO;QAACf;QAAM2C;IAAY;AAC5B;AAEA,SAASqB,mBACPpD,MAAsD;IAEtD,OAAO;QAACe,MAAMf,OAAOe,IAAI;QAAEuC,IAAItD,OAAOsD,EAAE;IAAA;AAC1C;AAEA;;;;;CAKC,GACD,OAAO,eAAeO,WAAW3C,MAAwB;IACvD,IAAIP;IACJ,IAAI;QACFA,SAASxB,gBAAgB+B,OAAO9B,IAAI;IACtC,EAAE,OAAO0E,OAAO;QACd,kFAAkF;QAClF,IAAIA,iBAAiBlF,0BAA0BkF,MAAMC,UAAU,KAAKxD,WAAW;YAC7E7C,SAASsG,IAAI,CACX;gBACE5C,OAAOF,OAAOE,KAAK;gBACnBX,QAAQS,OAAOT,MAAM;gBACrBwD,eAAeH,MAAMC,UAAU;YACjC,GACA;QAEJ;QACA,MAAMD;IACR;IACA,MAAM,EAAChE,kBAAkB,EAAC,GAAGa;IAC7B,MAAMuD,iBAAiBvD,OAAOpB,qBAAqB,GAC/C,MAAMiB,kBAAkBU,OAAOT,MAAM,IACrCpD;IACJ,MAAMyE,SAASmB,gBAAgBtC,OAAOrB,OAAO,EAAE4E;IAC/C,MAAMC,gBAAgBjD,OAAO9B,IAAI,CAACC,MAAM;IACxC,MAAM+E,UAAU;QACdrC,cAAc,CAAC;QACfsC,oBAAoB9D;QACpB+D,cAAc;IAChB;IAEA,MAAMC,SAAS,MAAMtG,KAAKuG,WAAW,CAAC,OAAOvD;QAC3C,IAAIkD,kBAAkB,GAAG,OAAOnD,cAAcC,IAAIC;QAElD,MAAM,EAACuD,OAAO,EAAEC,MAAM,EAAC,GAAG,MAAMtG,mCAAmC6C,IAAI;YACrEG,OAAOF,OAAOE,KAAK;YACnBX,QAAQS,OAAOT,MAAM;YACrBY,SAASH,OAAOG,OAAO;YACvBK,aAAaR,OAAOQ,WAAW;YAC/BiD,WAAWzD,OAAOyD,SAAS;YAC3BC,sBAAsB1D,OAAO0D,oBAAoB;QACnD;QACAR,QAAQE,YAAY,GAAGG;QAEvB,oFAAoF;QACpF,iFAAiF;QACjF,kFAAkF;QAClF,IAAIC,OAAOG,KAAK,KAAK,UAAU;YAC7B,OAAO;gBAACvD,iBAAiBoD,OAAOpD,eAAe;gBAAEC,QAAQ,MAAMxD,YAAYkD,IAAIC,OAAOE,KAAK;YAAC;QAC9F;QAEA,MAAM0D,MAAM,MAAM5G,yBAAyB+C,IAAI;YAC7CQ,UAAUiD,OAAOK,EAAE;YACnB9C,QAAQf,OAAOe,MAAM;YACrBC,SAASiC;QACX;QACA,IAAIW,IAAIE,OAAO,KAAK,OAAO,MAAM,IAAInG,eAAeiG,IAAIxD,eAAe;QACvE,IAAIwD,IAAIE,OAAO,KAAK,SAAS;YAC3B,OAAO;gBAAC1D,iBAAiBwD,IAAIxD,eAAe;gBAAEC,QAAQ,MAAMxD,YAAYkD,IAAIC,OAAOE,KAAK;YAAC;QAC3F;QAEA,MAAM,EACJW,YAAY,EACZD,QAAQmD,WAAW,EACnB,GAAGV,QACJ,GAAG,MAAM/C,WAAWP,IAAI;YACvBC;YACAO,UAAUiD,OAAOK,EAAE;YACnB3F,MAAM0C,OAAO1C,IAAI;YACjBkC,iBAAiBwD,IAAIxD,eAAe;YACpCxB;QACF;QACA,IAAImF,aAAa;YACfC,gBAAgBd,QAAQrC,YAAY,EAAED,OAAOC,YAAY;QAC3D;QACAmD,gBAAgBd,QAAQrC,YAAY,EAAEA;QAEtC,8EAA8E;QAC9E,qFAAqF;QACrF,oFAAoF;QACpF,mFAAmF;QACnF,qFAAqF;QACrF,0CAA0C;QAC1C,IAAIjC,uBAAuBS,aAAa0E,aAAa;YACnD,MAAME,SAAS,MAAMzG,YAAYuC,IAAI;gBAACQ,UAAUiD,OAAOK,EAAE;gBAAEK,QAAQ;YAAU;YAC7E,IAAID,QAAQf,QAAQC,kBAAkB,GAAG;QAC3C;QAEA,OAAOE;IACT;IAEA,IAAIH,QAAQE,YAAY,EAAE9F,kBAAkB6G,GAAG,CAAC;IAChD,KAAK,MAAM,CAACC,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACrB,QAAQrC,YAAY,EAAG;QAChE,IAAIwD,QAAQ,GAAGjH,oBAAoB+G,GAAG,CAACE,OAAO;YAACD,MAAMA;QAA2B;IAClF;IACA,IAAIlB,QAAQC,kBAAkB,EAAE;QAC9B9F,kBAAkB8G,GAAG,CAAC,GAAG;YAACD,QAAQhB,QAAQC,kBAAkB;QAAA;IAC9D;IAEA,OAAOE;AACT;AAEA,SAASW,gBACPQ,MAAoD,EACpDC,MAAoD;IAEpD,KAAK,MAAM,CAACL,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACE,QAAS;QAClDD,MAAM,CAACJ,KAA4B,GAAG,AAACI,CAAAA,MAAM,CAACJ,KAA4B,IAAI,CAAA,IAAMC,CAAAA,SAAS,CAAA;IAC/F;AACF"}

package/dist/core/budget.d.ts

@@ -0,0 +1,12 @@
+export interface AllowedBudgetParams {
+    baseBytes: number;
+    ratePerMinuteBytes: number;
+    /** Time since the budget clock origin (first append) in milliseconds. */
+    elapsedMs: number;
+}
+/**
+ * Accrual budget in stored bytes: `base + rate * elapsedMinutes`, floored to an
+ * integer. No hard ceiling — job duration limits bound total volume.
+ */
+export declare function allowedBudget({ baseBytes, ratePerMinuteBytes, elapsedMs, }: AllowedBudgetParams): number;
+//# sourceMappingURL=budget.d.ts.map

package/dist/core/budget.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"budget.d.ts","sourceRoot":"","sources":["../../src/core/budget.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,yEAAyE;IACzE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,EAC5B,SAAS,EACT,kBAAkB,EAClB,SAAS,GACV,EAAE,mBAAmB,GAAG,MAAM,CAG9B"}

package/dist/core/budget.js

@@ -0,0 +1,9 @@
+/**
+ * Accrual budget in stored bytes: `base + rate * elapsedMinutes`, floored to an
+ * integer. No hard ceiling — job duration limits bound total volume.
+ */ export function allowedBudget({ baseBytes, ratePerMinuteBytes, elapsedMs }) {
+    const elapsed = Math.max(0, elapsedMs);
+    return baseBytes + Math.floor(ratePerMinuteBytes * elapsed / 60_000);
+}
+
+//# sourceMappingURL=budget.js.map

package/dist/core/budget.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/core/budget.ts"],"sourcesContent":["export interface AllowedBudgetParams {\n  baseBytes: number;\n  ratePerMinuteBytes: number;\n  /** Time since the budget clock origin (first append) in milliseconds. */\n  elapsedMs: number;\n}\n\n/**\n * Accrual budget in stored bytes: `base + rate * elapsedMinutes`, floored to an\n * integer. No hard ceiling — job duration limits bound total volume.\n */\nexport function allowedBudget({\n  baseBytes,\n  ratePerMinuteBytes,\n  elapsedMs,\n}: AllowedBudgetParams): number {\n  const elapsed = Math.max(0, elapsedMs);\n  return baseBytes + Math.floor((ratePerMinuteBytes * elapsed) / 60_000);\n}\n"],"names":["allowedBudget","baseBytes","ratePerMinuteBytes","elapsedMs","elapsed","Math","max","floor"],"mappings":"AAOA;;;CAGC,GACD,OAAO,SAASA,cAAc,EAC5BC,SAAS,EACTC,kBAAkB,EAClBC,SAAS,EACW;IACpB,MAAMC,UAAUC,KAAKC,GAAG,CAAC,GAAGH;IAC5B,OAAOF,YAAYI,KAAKE,KAAK,CAAC,AAACL,qBAAqBE,UAAW;AACjE"}