@shipfox/api-logs 5.0.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +32 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +5 -10
- package/dist/config.js.map +1 -1
- package/dist/core/append-logs.d.ts +2 -6
- package/dist/core/append-logs.d.ts.map +1 -1
- package/dist/core/append-logs.js +7 -16
- package/dist/core/append-logs.js.map +1 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +63 -59
- package/dist/index.js.map +1 -1
- package/dist/presentation/routes/append-logs.d.ts +2 -1
- package/dist/presentation/routes/append-logs.d.ts.map +1 -1
- package/dist/presentation/routes/append-logs.js +61 -61
- package/dist/presentation/routes/append-logs.js.map +1 -1
- package/dist/presentation/routes/index.d.ts +2 -1
- package/dist/presentation/routes/index.d.ts.map +1 -1
- package/dist/presentation/routes/index.js +25 -23
- package/dist/presentation/routes/index.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +16 -13
- package/src/config.test.ts +8 -32
- package/src/config.ts +10 -15
- package/src/core/append-logs.test.ts +13 -19
- package/src/core/append-logs.ts +12 -20
- package/src/index.ts +63 -51
- package/src/presentation/routes/append-logs.test.ts +3 -2
- package/src/presentation/routes/append-logs.ts +54 -50
- package/src/presentation/routes/index.ts +22 -19
- package/src/presentation/routes/read-logs.test.ts +3 -2
- package/test/env.ts +1 -3
- package/test/fixtures/lease-token.ts +2 -1
- package/test/fixtures/workflows-client.ts +17 -0
- package/tsconfig.build.tsbuildinfo +1 -1
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
$ shipfox-swc && shipfox-temporal-bundle dist/temporal/workflows/index.js
|
|
2
|
-
Successfully compiled: 59 files with swc (
|
|
3
|
-
2026-07-
|
|
2
|
+
Successfully compiled: 59 files with swc (510.82ms)
|
|
3
|
+
2026-07-21T08:11:10.748Z [INFO] asset workflow-bundle-2850a9a125f53b77d607.js 1.46 MiB [emitted] [immutable] (name: main)
|
|
4
4
|
runtime modules 670 bytes 3 modules
|
|
5
5
|
modules by path ../../../node_modules/.pnpm/ 424 KiB
|
|
6
6
|
modules by path ../../../node_modules/.pnpm/@temporalio+common@1.18.1/node_modules/@temporal...(truncated) 155 KiB 34 modules
|
|
@@ -16,6 +16,6 @@ modules by path ./dist/temporal/workflows/ 4.17 KiB
|
|
|
16
16
|
optional modules 30 bytes [optional]
|
|
17
17
|
__temporal_custom_payload_converter (ignored) 15 bytes [optional] [built] [code generated]
|
|
18
18
|
__temporal_custom_failure_converter (ignored) 15 bytes [optional] [built] [code generated]
|
|
19
|
-
webpack 5.107.2 compiled successfully in
|
|
20
|
-
2026-07-
|
|
19
|
+
webpack 5.107.2 compiled successfully in 3800 ms
|
|
20
|
+
2026-07-21T08:11:10.761Z [INFO] Workflow bundle created { size: '1.46MB' }
|
|
21
21
|
/home/runner/work/shipfox/shipfox/libs/api/logs/dist/temporal/workflows/index.js: 1530555 bytes
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,37 @@
|
|
|
1
1
|
# @shipfox/api-logs
|
|
2
2
|
|
|
3
|
+
## 6.0.0
|
|
4
|
+
|
|
5
|
+
### Major Changes
|
|
6
|
+
|
|
7
|
+
- 23a4dc2: Moves Logs and Integrations to injected Workflows inter-module clients with minimal log and leased agent-tool queries.
|
|
8
|
+
|
|
9
|
+
### Patch Changes
|
|
10
|
+
|
|
11
|
+
- 112c0fa: Adds the Auth inter-module token-minting contract and removes Auth implementation and configuration coupling from its consumers.
|
|
12
|
+
- f73da5d: Enforces bounded API context imports and routes inter-module consumers through producer contracts.
|
|
13
|
+
- Updated dependencies [0bb82a4]
|
|
14
|
+
- Updated dependencies [23563de]
|
|
15
|
+
- Updated dependencies [54ce48b]
|
|
16
|
+
- Updated dependencies [f4bc2eb]
|
|
17
|
+
- Updated dependencies [c0162b0]
|
|
18
|
+
- Updated dependencies [7ac43a4]
|
|
19
|
+
- Updated dependencies [a01e917]
|
|
20
|
+
- Updated dependencies [8bdc149]
|
|
21
|
+
- Updated dependencies [3810996]
|
|
22
|
+
- Updated dependencies [23a4dc2]
|
|
23
|
+
- Updated dependencies [b00ed29]
|
|
24
|
+
- Updated dependencies [8aa7cd3]
|
|
25
|
+
- Updated dependencies [81f9544]
|
|
26
|
+
- @shipfox/api-agent-dto@6.0.0
|
|
27
|
+
- @shipfox/api-workflows-dto@6.0.0
|
|
28
|
+
- @shipfox/node-module@0.4.0
|
|
29
|
+
- @shipfox/node-temporal@0.3.2
|
|
30
|
+
- @shipfox/node-drizzle@0.3.2
|
|
31
|
+
- @shipfox/node-outbox@0.2.4
|
|
32
|
+
- @shipfox/api-auth-context@6.0.0
|
|
33
|
+
- @shipfox/node-fastify@0.2.4
|
|
34
|
+
|
|
3
35
|
## 5.0.0
|
|
4
36
|
|
|
5
37
|
### Patch Changes
|
package/dist/config.d.ts
CHANGED
|
@@ -17,4 +17,5 @@ export declare const config: Readonly<{
|
|
|
17
17
|
LOG_READ_INLINE_MAX_BYTES: number;
|
|
18
18
|
LOG_RETENTION_DAYS: number;
|
|
19
19
|
} & import("@shipfox/config").CleanedEnvAccessors>;
|
|
20
|
+
export declare function validateLogStreamReapAfterSeconds(jobLeaseTokenTtlSeconds: number): void;
|
|
20
21
|
//# sourceMappingURL=config.d.ts.map
|
package/dist/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;kDAqEjB,CAAC;AA4CH,wBAAgB,iCAAiC,CAAC,uBAAuB,EAAE,MAAM,GAAG,IAAI,CASvF"}
|
package/dist/config.js
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import { config as authConfig } from '@shipfox/api-auth/config';
|
|
2
1
|
import { bool, createConfig, num, str, url } from '@shipfox/config';
|
|
3
|
-
import { durationToSeconds } from '@shipfox/node-jwt';
|
|
4
2
|
export const config = createConfig({
|
|
5
3
|
LOG_STORAGE_S3_ENDPOINT: url({
|
|
6
4
|
desc: 'Endpoint URL of the S3-compatible object store that holds compacted logs. Defaults to the bundled local-development Garage (http://localhost:3900); set it to your object store endpoint for production.',
|
|
@@ -47,7 +45,7 @@ export const config = createConfig({
|
|
|
47
45
|
default: 900
|
|
48
46
|
}),
|
|
49
47
|
LOG_STREAM_REAP_AFTER_SECONDS: num({
|
|
50
|
-
desc: 'How long a log stream may stay open before the reaper cron force-closes it as abandoned. This is the backstop for a stream a runner started after its job already went terminal (the one-shot close sweep had already run); the reaper marks it truncated so it re-enters the compaction and retention lifecycle.
|
|
48
|
+
desc: 'How long a log stream may stay open before the reaper cron force-closes it as abandoned. This is the backstop for a stream a runner started after its job already went terminal (the one-shot close sweep had already run); the reaper marks it truncated so it re-enters the compaction and retention lifecycle. The application validates that this exceeds the Auth job lease lifetime. Defaults to 7200 seconds (2 hours).',
|
|
51
49
|
default: 7200
|
|
52
50
|
}),
|
|
53
51
|
LOG_APPEND_BODY_LIMIT_BYTES: num({
|
|
@@ -96,13 +94,10 @@ if (config.LOG_APPEND_BODY_LIMIT_BYTES < config.LOG_MAX_SESSION_LINE_BYTES) {
|
|
|
96
94
|
if (!Number.isInteger(config.LOG_RETENTION_DAYS) || config.LOG_RETENTION_DAYS < 1) {
|
|
97
95
|
throw new Error(`LOG_RETENTION_DAYS (${config.LOG_RETENTION_DAYS}) must be a whole number of days >= 1.`);
|
|
98
96
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
const jobLeaseTtlSeconds = durationToSeconds(authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN);
|
|
104
|
-
if (!Number.isFinite(config.LOG_STREAM_REAP_AFTER_SECONDS) || config.LOG_STREAM_REAP_AFTER_SECONDS <= jobLeaseTtlSeconds) {
|
|
105
|
-
throw new Error(`LOG_STREAM_REAP_AFTER_SECONDS (${config.LOG_STREAM_REAP_AFTER_SECONDS}) must be greater than the job lease TTL (${jobLeaseTtlSeconds}s, from AUTH_JOB_LEASE_TOKEN_EXPIRES_IN=${authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN}); a smaller value would let the reaper force-close a stream a still-valid lease is appending to and silently truncate live logs.`);
|
|
97
|
+
export function validateLogStreamReapAfterSeconds(jobLeaseTokenTtlSeconds) {
|
|
98
|
+
if (!Number.isFinite(config.LOG_STREAM_REAP_AFTER_SECONDS) || config.LOG_STREAM_REAP_AFTER_SECONDS <= jobLeaseTokenTtlSeconds) {
|
|
99
|
+
throw new Error(`LOG_STREAM_REAP_AFTER_SECONDS (${config.LOG_STREAM_REAP_AFTER_SECONDS}) must be greater than the Auth job lease TTL (${jobLeaseTokenTtlSeconds}s); a smaller value would let the reaper force-close a stream a still-valid lease is appending to and silently truncate live logs.`);
|
|
100
|
+
}
|
|
106
101
|
}
|
|
107
102
|
|
|
108
103
|
//# sourceMappingURL=config.js.map
|
package/dist/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/config.ts"],"sourcesContent":["import {config as authConfig} from '@shipfox/api-auth/config';\nimport {bool, createConfig, num, str, url} from '@shipfox/config';\nimport {durationToSeconds} from '@shipfox/node-jwt';\n\nexport const config = createConfig({\n LOG_STORAGE_S3_ENDPOINT: url({\n desc: 'Endpoint URL of the S3-compatible object store that holds compacted logs. Defaults to the bundled local-development Garage (http://localhost:3900); set it to your object store endpoint for production.',\n default: 'http://localhost:3900',\n }),\n LOG_STORAGE_S3_REGION: str({\n desc: 'Region passed to the S3 client. Any value works for Garage; set the real region for AWS S3. Defaults to garage for local development.',\n default: 'garage',\n }),\n LOG_STORAGE_S3_BUCKET: str({\n desc: 'Name of the bucket that stores compacted log objects. Defaults to shipfox-logs (created by dev/garage/bootstrap.sh); create the bucket and set this for production.',\n default: 'shipfox-logs',\n }),\n LOG_STORAGE_S3_PREFIX: str({\n desc: 'Key prefix under which compacted log objects are stored in the bucket. Set this to host several modules in one bucket, each under its own prefix. Use a value without a leading or trailing slash. Defaults to logs.',\n default: 'logs',\n }),\n LOG_STORAGE_S3_ACCESS_KEY_ID: str({\n desc: 'Optional access key ID used to authenticate to the object store. Set it together with LOG_STORAGE_S3_SECRET_ACCESS_KEY for an explicit credential pair, or leave both unset to use the standard AWS SDK credential provider chain.',\n default: undefined,\n }),\n LOG_STORAGE_S3_SECRET_ACCESS_KEY: str({\n desc: 'Optional secret access key used to authenticate to the object store. Set it together with LOG_STORAGE_S3_ACCESS_KEY_ID for an explicit credential pair, or leave both unset to use the standard AWS SDK credential provider chain.',\n default: undefined,\n }),\n LOG_STORAGE_S3_FORCE_PATH_STYLE: bool({\n desc: 'Whether to address the bucket as a path (endpoint/bucket) instead of a subdomain. Set it to true for Garage and MinIO; false works for AWS S3.',\n default: true,\n }),\n LOG_BUDGET_BASE_BYTES: num({\n desc: 'Base of the per-job log accrual budget, in stored bytes (normalized NDJSON the server keeps, framing included). A job may always store this much before the time-based rate is added. Sized to hold a few inline agent_session entries (such as base64 images) before the shared per-job cap trips. Defaults to 32 MiB.',\n default: 33_554_432,\n }),\n LOG_BUDGET_RATE_BYTES_PER_MINUTE: num({\n desc: 'Stored bytes added to the per-job log budget for each minute since the first log append. Defaults to 1 MiB per minute.',\n default: 1_048_576,\n }),\n LOG_STREAM_CLOSE_GRACE_SECONDS: num({\n desc: 'How long to wait after a job reaches a terminal state before force-closing any of its log streams the runner never ended itself (it died, was capped, or its log spool failed). The wait lets a last in-flight chunk land before the stream is marked truncated. Defaults to 120 seconds.',\n default: 120,\n }),\n LOG_COMPACTION_RECONCILE_STALE_SECONDS: num({\n desc: 'How long a closed log stream may stay uncompacted before the reconcile cron re-drives it. The event-triggered compaction normally runs within seconds of close; this backstop only re-drives streams whose compaction never started or permanently failed. Defaults to 900 seconds (15 minutes).',\n default: 900,\n }),\n LOG_STREAM_REAP_AFTER_SECONDS: num({\n desc: 'How long a log stream may stay open before the reaper cron force-closes it as abandoned. This is the backstop for a stream a runner started after its job already went terminal (the one-shot close sweep had already run); the reaper marks it truncated so it re-enters the compaction and retention lifecycle. Must be greater than AUTH_JOB_LEASE_TOKEN_EXPIRES_IN (the job lease lifetime, default 90 minutes) so a still-valid lease can never be force-closed mid-append, which would silently truncate live logs. Defaults to 7200 seconds (2 hours).',\n default: 7200,\n }),\n LOG_APPEND_BODY_LIMIT_BYTES: num({\n desc: 'Maximum size of a single log append request body, in bytes. Output records cap at 16 KiB each, but one agent_session record carries a whole session entry (which can inline base64 images), so the body must hold a full LOG_MAX_SESSION_LINE_BYTES line plus framing. This also bounds how far a job can overshoot its log budget: the one append that crosses the budget is stored in full before the job is capped. Defaults to 8 MiB.',\n default: 8_388_608,\n }),\n LOG_MAX_SESSION_LINE_BYTES: num({\n desc: 'Maximum size of one agent_session log line (a verbatim agent session entry), in bytes. A larger line is rejected with 400, and the runner drops it with a gap marker instead of sending it. Sized generously for entries that inline large content such as base64 images. Must be at most LOG_APPEND_BODY_LIMIT_BYTES, since a line that cannot fit in one request body could never be accepted. Defaults to 4 MiB.',\n default: 4_194_304,\n }),\n LOG_READ_URL_TTL_SECONDS: num({\n desc: 'Lifetime, in seconds, of a presigned GET URL handed out for a compacted (cold) log read. The browser must finish fetching the object before it expires; raise it for slow links or large logs. Must be between 1 and 604800 (7 days, the longest a presigned URL can live). Defaults to 3600 (one hour).',\n default: 3600,\n }),\n LOG_READ_INLINE_MAX_BYTES: num({\n desc: 'Maximum number of stored log bytes returned in a single inline (hot) read response. A long-running step can buffer far more than this in Postgres before compaction, so a read is capped here and the client re-polls (following has_more) to drain the backlog before it tails. Bounds per-request memory. Must be at least 1. Defaults to 1 MiB.',\n default: 1_048_576,\n }),\n LOG_RETENTION_DAYS: num({\n desc: 'How many days a closed log stream is kept before the retention cron hard-deletes its stored object and database row. Our own worker enforces this (not bucket lifecycle rules), so behavior is identical across object stores. Must be a whole number of days, 1 or greater. Defaults to 90 days.',\n default: 90,\n }),\n});\n\nconst hasS3AccessKeyId = Boolean(config.LOG_STORAGE_S3_ACCESS_KEY_ID);\nconst hasS3SecretAccessKey = Boolean(config.LOG_STORAGE_S3_SECRET_ACCESS_KEY);\nif (hasS3AccessKeyId !== hasS3SecretAccessKey) {\n throw new Error(\n 'LOG_STORAGE_S3_ACCESS_KEY_ID and LOG_STORAGE_S3_SECRET_ACCESS_KEY must be set together or both left unset.',\n );\n}\n\n// SigV4 caps a presigned URL's lifetime at 7 days, so a larger TTL would fail at signing\n// time on every cold read. Reject out-of-range values at startup rather than per request.\nconst MAX_PRESIGN_TTL_SECONDS = 604_800;\nif (\n config.LOG_READ_URL_TTL_SECONDS < 1 ||\n config.LOG_READ_URL_TTL_SECONDS > MAX_PRESIGN_TTL_SECONDS\n) {\n throw new Error(\n `LOG_READ_URL_TTL_SECONDS must be between 1 and ${MAX_PRESIGN_TTL_SECONDS}; got ${config.LOG_READ_URL_TTL_SECONDS}`,\n );\n}\nif (config.LOG_READ_INLINE_MAX_BYTES < 1) {\n throw new Error(\n `LOG_READ_INLINE_MAX_BYTES must be at least 1; got ${config.LOG_READ_INLINE_MAX_BYTES}`,\n );\n}\n\n// A whole agent_session line is one record in one body. If the body limit is below the\n// line cap, a legitimate large session line could never fit in one request body — Fastify\n// would 413 it before the per-line validator runs. Fail fast at startup instead.\nif (config.LOG_APPEND_BODY_LIMIT_BYTES < config.LOG_MAX_SESSION_LINE_BYTES) {\n throw new Error(\n `LOG_APPEND_BODY_LIMIT_BYTES (${config.LOG_APPEND_BODY_LIMIT_BYTES}) must be >= LOG_MAX_SESSION_LINE_BYTES (${config.LOG_MAX_SESSION_LINE_BYTES}).`,\n );\n}\n\n// `num` accepts zero, negative, and fractional values; reject them before they reach\n// `make_interval(days => ...)` and hard-delete the wrong streams.\nif (!Number.isInteger(config.LOG_RETENTION_DAYS) || config.LOG_RETENTION_DAYS < 1) {\n throw new Error(\n `LOG_RETENTION_DAYS (${config.LOG_RETENTION_DAYS}) must be a whole number of days >= 1.`,\n );\n}\n\n// A stream still accepts appends until its job lease expires, so reaping it any sooner would\n// truncate live logs. Validate against the real lease TTL read from auth's config (not a\n// hardcoded floor), so raising AUTH_JOB_LEASE_TOKEN_EXPIRES_IN can never silently outrun the\n// reaper window.\nconst jobLeaseTtlSeconds = durationToSeconds(authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN);\nif (\n !Number.isFinite(config.LOG_STREAM_REAP_AFTER_SECONDS) ||\n config.LOG_STREAM_REAP_AFTER_SECONDS <= jobLeaseTtlSeconds\n) {\n throw new Error(\n `LOG_STREAM_REAP_AFTER_SECONDS (${config.LOG_STREAM_REAP_AFTER_SECONDS}) must be greater than the job lease TTL (${jobLeaseTtlSeconds}s, from AUTH_JOB_LEASE_TOKEN_EXPIRES_IN=${authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN}); a smaller value would let the reaper force-close a stream a still-valid lease is appending to and silently truncate live logs.`,\n );\n}\n"],"names":["config","authConfig","bool","createConfig","num","str","url","durationToSeconds","LOG_STORAGE_S3_ENDPOINT","desc","default","LOG_STORAGE_S3_REGION","LOG_STORAGE_S3_BUCKET","LOG_STORAGE_S3_PREFIX","LOG_STORAGE_S3_ACCESS_KEY_ID","undefined","LOG_STORAGE_S3_SECRET_ACCESS_KEY","LOG_STORAGE_S3_FORCE_PATH_STYLE","LOG_BUDGET_BASE_BYTES","LOG_BUDGET_RATE_BYTES_PER_MINUTE","LOG_STREAM_CLOSE_GRACE_SECONDS","LOG_COMPACTION_RECONCILE_STALE_SECONDS","LOG_STREAM_REAP_AFTER_SECONDS","LOG_APPEND_BODY_LIMIT_BYTES","LOG_MAX_SESSION_LINE_BYTES","LOG_READ_URL_TTL_SECONDS","LOG_READ_INLINE_MAX_BYTES","LOG_RETENTION_DAYS","hasS3AccessKeyId","Boolean","hasS3SecretAccessKey","Error","MAX_PRESIGN_TTL_SECONDS","Number","isInteger","jobLeaseTtlSeconds","AUTH_JOB_LEASE_TOKEN_EXPIRES_IN","isFinite"],"mappings":"AAAA,SAAQA,UAAUC,UAAU,QAAO,2BAA2B;AAC9D,SAAQC,IAAI,EAAEC,YAAY,EAAEC,GAAG,EAAEC,GAAG,EAAEC,GAAG,QAAO,kBAAkB;AAClE,SAAQC,iBAAiB,QAAO,oBAAoB;AAEpD,OAAO,MAAMP,SAASG,aAAa;IACjCK,yBAAyBF,IAAI;QAC3BG,MAAM;QACNC,SAAS;IACX;IACAC,uBAAuBN,IAAI;QACzBI,MAAM;QACNC,SAAS;IACX;IACAE,uBAAuBP,IAAI;QACzBI,MAAM;QACNC,SAAS;IACX;IACAG,uBAAuBR,IAAI;QACzBI,MAAM;QACNC,SAAS;IACX;IACAI,8BAA8BT,IAAI;QAChCI,MAAM;QACNC,SAASK;IACX;IACAC,kCAAkCX,IAAI;QACpCI,MAAM;QACNC,SAASK;IACX;IACAE,iCAAiCf,KAAK;QACpCO,MAAM;QACNC,SAAS;IACX;IACAQ,uBAAuBd,IAAI;QACzBK,MAAM;QACNC,SAAS;IACX;IACAS,kCAAkCf,IAAI;QACpCK,MAAM;QACNC,SAAS;IACX;IACAU,gCAAgChB,IAAI;QAClCK,MAAM;QACNC,SAAS;IACX;IACAW,wCAAwCjB,IAAI;QAC1CK,MAAM;QACNC,SAAS;IACX;IACAY,+BAA+BlB,IAAI;QACjCK,MAAM;QACNC,SAAS;IACX;IACAa,6BAA6BnB,IAAI;QAC/BK,MAAM;QACNC,SAAS;IACX;IACAc,4BAA4BpB,IAAI;QAC9BK,MAAM;QACNC,SAAS;IACX;IACAe,0BAA0BrB,IAAI;QAC5BK,MAAM;QACNC,SAAS;IACX;IACAgB,2BAA2BtB,IAAI;QAC7BK,MAAM;QACNC,SAAS;IACX;IACAiB,oBAAoBvB,IAAI;QACtBK,MAAM;QACNC,SAAS;IACX;AACF,GAAG;AAEH,MAAMkB,mBAAmBC,QAAQ7B,OAAOc,4BAA4B;AACpE,MAAMgB,uBAAuBD,QAAQ7B,OAAOgB,gCAAgC;AAC5E,IAAIY,qBAAqBE,sBAAsB;IAC7C,MAAM,IAAIC,MACR;AAEJ;AAEA,yFAAyF;AACzF,0FAA0F;AAC1F,MAAMC,0BAA0B;AAChC,IACEhC,OAAOyB,wBAAwB,GAAG,KAClCzB,OAAOyB,wBAAwB,GAAGO,yBAClC;IACA,MAAM,IAAID,MACR,CAAC,+CAA+C,EAAEC,wBAAwB,MAAM,EAAEhC,OAAOyB,wBAAwB,EAAE;AAEvH;AACA,IAAIzB,OAAO0B,yBAAyB,GAAG,GAAG;IACxC,MAAM,IAAIK,MACR,CAAC,kDAAkD,EAAE/B,OAAO0B,yBAAyB,EAAE;AAE3F;AAEA,uFAAuF;AACvF,0FAA0F;AAC1F,iFAAiF;AACjF,IAAI1B,OAAOuB,2BAA2B,GAAGvB,OAAOwB,0BAA0B,EAAE;IAC1E,MAAM,IAAIO,MACR,CAAC,6BAA6B,EAAE/B,OAAOuB,2BAA2B,CAAC,yCAAyC,EAAEvB,OAAOwB,0BAA0B,CAAC,EAAE,CAAC;AAEvJ;AAEA,qFAAqF;AACrF,kEAAkE;AAClE,IAAI,CAACS,OAAOC,SAAS,CAAClC,OAAO2B,kBAAkB,KAAK3B,OAAO2B,kBAAkB,GAAG,GAAG;IACjF,MAAM,IAAII,MACR,CAAC,oBAAoB,EAAE/B,OAAO2B,kBAAkB,CAAC,sCAAsC,CAAC;AAE5F;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,6FAA6F;AAC7F,iBAAiB;AACjB,MAAMQ,qBAAqB5B,kBAAkBN,WAAWmC,+BAA+B;AACvF,IACE,CAACH,OAAOI,QAAQ,CAACrC,OAAOsB,6BAA6B,KACrDtB,OAAOsB,6BAA6B,IAAIa,oBACxC;IACA,MAAM,IAAIJ,MACR,CAAC,+BAA+B,EAAE/B,OAAOsB,6BAA6B,CAAC,0CAA0C,EAAEa,mBAAmB,wCAAwC,EAAElC,WAAWmC,+BAA+B,CAAC,iIAAiI,CAAC;AAEjW"}
|
|
1
|
+
{"version":3,"sources":["../src/config.ts"],"sourcesContent":["import {bool, createConfig, num, str, url} from '@shipfox/config';\n\nexport const config = createConfig({\n LOG_STORAGE_S3_ENDPOINT: url({\n desc: 'Endpoint URL of the S3-compatible object store that holds compacted logs. Defaults to the bundled local-development Garage (http://localhost:3900); set it to your object store endpoint for production.',\n default: 'http://localhost:3900',\n }),\n LOG_STORAGE_S3_REGION: str({\n desc: 'Region passed to the S3 client. Any value works for Garage; set the real region for AWS S3. Defaults to garage for local development.',\n default: 'garage',\n }),\n LOG_STORAGE_S3_BUCKET: str({\n desc: 'Name of the bucket that stores compacted log objects. Defaults to shipfox-logs (created by dev/garage/bootstrap.sh); create the bucket and set this for production.',\n default: 'shipfox-logs',\n }),\n LOG_STORAGE_S3_PREFIX: str({\n desc: 'Key prefix under which compacted log objects are stored in the bucket. Set this to host several modules in one bucket, each under its own prefix. Use a value without a leading or trailing slash. Defaults to logs.',\n default: 'logs',\n }),\n LOG_STORAGE_S3_ACCESS_KEY_ID: str({\n desc: 'Optional access key ID used to authenticate to the object store. Set it together with LOG_STORAGE_S3_SECRET_ACCESS_KEY for an explicit credential pair, or leave both unset to use the standard AWS SDK credential provider chain.',\n default: undefined,\n }),\n LOG_STORAGE_S3_SECRET_ACCESS_KEY: str({\n desc: 'Optional secret access key used to authenticate to the object store. Set it together with LOG_STORAGE_S3_ACCESS_KEY_ID for an explicit credential pair, or leave both unset to use the standard AWS SDK credential provider chain.',\n default: undefined,\n }),\n LOG_STORAGE_S3_FORCE_PATH_STYLE: bool({\n desc: 'Whether to address the bucket as a path (endpoint/bucket) instead of a subdomain. Set it to true for Garage and MinIO; false works for AWS S3.',\n default: true,\n }),\n LOG_BUDGET_BASE_BYTES: num({\n desc: 'Base of the per-job log accrual budget, in stored bytes (normalized NDJSON the server keeps, framing included). A job may always store this much before the time-based rate is added. Sized to hold a few inline agent_session entries (such as base64 images) before the shared per-job cap trips. Defaults to 32 MiB.',\n default: 33_554_432,\n }),\n LOG_BUDGET_RATE_BYTES_PER_MINUTE: num({\n desc: 'Stored bytes added to the per-job log budget for each minute since the first log append. Defaults to 1 MiB per minute.',\n default: 1_048_576,\n }),\n LOG_STREAM_CLOSE_GRACE_SECONDS: num({\n desc: 'How long to wait after a job reaches a terminal state before force-closing any of its log streams the runner never ended itself (it died, was capped, or its log spool failed). The wait lets a last in-flight chunk land before the stream is marked truncated. Defaults to 120 seconds.',\n default: 120,\n }),\n LOG_COMPACTION_RECONCILE_STALE_SECONDS: num({\n desc: 'How long a closed log stream may stay uncompacted before the reconcile cron re-drives it. The event-triggered compaction normally runs within seconds of close; this backstop only re-drives streams whose compaction never started or permanently failed. Defaults to 900 seconds (15 minutes).',\n default: 900,\n }),\n LOG_STREAM_REAP_AFTER_SECONDS: num({\n desc: 'How long a log stream may stay open before the reaper cron force-closes it as abandoned. This is the backstop for a stream a runner started after its job already went terminal (the one-shot close sweep had already run); the reaper marks it truncated so it re-enters the compaction and retention lifecycle. The application validates that this exceeds the Auth job lease lifetime. Defaults to 7200 seconds (2 hours).',\n default: 7200,\n }),\n LOG_APPEND_BODY_LIMIT_BYTES: num({\n desc: 'Maximum size of a single log append request body, in bytes. Output records cap at 16 KiB each, but one agent_session record carries a whole session entry (which can inline base64 images), so the body must hold a full LOG_MAX_SESSION_LINE_BYTES line plus framing. This also bounds how far a job can overshoot its log budget: the one append that crosses the budget is stored in full before the job is capped. Defaults to 8 MiB.',\n default: 8_388_608,\n }),\n LOG_MAX_SESSION_LINE_BYTES: num({\n desc: 'Maximum size of one agent_session log line (a verbatim agent session entry), in bytes. A larger line is rejected with 400, and the runner drops it with a gap marker instead of sending it. Sized generously for entries that inline large content such as base64 images. Must be at most LOG_APPEND_BODY_LIMIT_BYTES, since a line that cannot fit in one request body could never be accepted. Defaults to 4 MiB.',\n default: 4_194_304,\n }),\n LOG_READ_URL_TTL_SECONDS: num({\n desc: 'Lifetime, in seconds, of a presigned GET URL handed out for a compacted (cold) log read. The browser must finish fetching the object before it expires; raise it for slow links or large logs. Must be between 1 and 604800 (7 days, the longest a presigned URL can live). Defaults to 3600 (one hour).',\n default: 3600,\n }),\n LOG_READ_INLINE_MAX_BYTES: num({\n desc: 'Maximum number of stored log bytes returned in a single inline (hot) read response. A long-running step can buffer far more than this in Postgres before compaction, so a read is capped here and the client re-polls (following has_more) to drain the backlog before it tails. Bounds per-request memory. Must be at least 1. Defaults to 1 MiB.',\n default: 1_048_576,\n }),\n LOG_RETENTION_DAYS: num({\n desc: 'How many days a closed log stream is kept before the retention cron hard-deletes its stored object and database row. Our own worker enforces this (not bucket lifecycle rules), so behavior is identical across object stores. Must be a whole number of days, 1 or greater. Defaults to 90 days.',\n default: 90,\n }),\n});\n\nconst hasS3AccessKeyId = Boolean(config.LOG_STORAGE_S3_ACCESS_KEY_ID);\nconst hasS3SecretAccessKey = Boolean(config.LOG_STORAGE_S3_SECRET_ACCESS_KEY);\nif (hasS3AccessKeyId !== hasS3SecretAccessKey) {\n throw new Error(\n 'LOG_STORAGE_S3_ACCESS_KEY_ID and LOG_STORAGE_S3_SECRET_ACCESS_KEY must be set together or both left unset.',\n );\n}\n\n// SigV4 caps a presigned URL's lifetime at 7 days, so a larger TTL would fail at signing\n// time on every cold read. Reject out-of-range values at startup rather than per request.\nconst MAX_PRESIGN_TTL_SECONDS = 604_800;\nif (\n config.LOG_READ_URL_TTL_SECONDS < 1 ||\n config.LOG_READ_URL_TTL_SECONDS > MAX_PRESIGN_TTL_SECONDS\n) {\n throw new Error(\n `LOG_READ_URL_TTL_SECONDS must be between 1 and ${MAX_PRESIGN_TTL_SECONDS}; got ${config.LOG_READ_URL_TTL_SECONDS}`,\n );\n}\nif (config.LOG_READ_INLINE_MAX_BYTES < 1) {\n throw new Error(\n `LOG_READ_INLINE_MAX_BYTES must be at least 1; got ${config.LOG_READ_INLINE_MAX_BYTES}`,\n );\n}\n\n// A whole agent_session line is one record in one body. If the body limit is below the\n// line cap, a legitimate large session line could never fit in one request body — Fastify\n// would 413 it before the per-line validator runs. Fail fast at startup instead.\nif (config.LOG_APPEND_BODY_LIMIT_BYTES < config.LOG_MAX_SESSION_LINE_BYTES) {\n throw new Error(\n `LOG_APPEND_BODY_LIMIT_BYTES (${config.LOG_APPEND_BODY_LIMIT_BYTES}) must be >= LOG_MAX_SESSION_LINE_BYTES (${config.LOG_MAX_SESSION_LINE_BYTES}).`,\n );\n}\n\n// `num` accepts zero, negative, and fractional values; reject them before they reach\n// `make_interval(days => ...)` and hard-delete the wrong streams.\nif (!Number.isInteger(config.LOG_RETENTION_DAYS) || config.LOG_RETENTION_DAYS < 1) {\n throw new Error(\n `LOG_RETENTION_DAYS (${config.LOG_RETENTION_DAYS}) must be a whole number of days >= 1.`,\n );\n}\n\nexport function validateLogStreamReapAfterSeconds(jobLeaseTokenTtlSeconds: number): void {\n if (\n !Number.isFinite(config.LOG_STREAM_REAP_AFTER_SECONDS) ||\n config.LOG_STREAM_REAP_AFTER_SECONDS <= jobLeaseTokenTtlSeconds\n ) {\n throw new Error(\n `LOG_STREAM_REAP_AFTER_SECONDS (${config.LOG_STREAM_REAP_AFTER_SECONDS}) must be greater than the Auth job lease TTL (${jobLeaseTokenTtlSeconds}s); a smaller value would let the reaper force-close a stream a still-valid lease is appending to and silently truncate live logs.`,\n );\n }\n}\n"],"names":["bool","createConfig","num","str","url","config","LOG_STORAGE_S3_ENDPOINT","desc","default","LOG_STORAGE_S3_REGION","LOG_STORAGE_S3_BUCKET","LOG_STORAGE_S3_PREFIX","LOG_STORAGE_S3_ACCESS_KEY_ID","undefined","LOG_STORAGE_S3_SECRET_ACCESS_KEY","LOG_STORAGE_S3_FORCE_PATH_STYLE","LOG_BUDGET_BASE_BYTES","LOG_BUDGET_RATE_BYTES_PER_MINUTE","LOG_STREAM_CLOSE_GRACE_SECONDS","LOG_COMPACTION_RECONCILE_STALE_SECONDS","LOG_STREAM_REAP_AFTER_SECONDS","LOG_APPEND_BODY_LIMIT_BYTES","LOG_MAX_SESSION_LINE_BYTES","LOG_READ_URL_TTL_SECONDS","LOG_READ_INLINE_MAX_BYTES","LOG_RETENTION_DAYS","hasS3AccessKeyId","Boolean","hasS3SecretAccessKey","Error","MAX_PRESIGN_TTL_SECONDS","Number","isInteger","validateLogStreamReapAfterSeconds","jobLeaseTokenTtlSeconds","isFinite"],"mappings":"AAAA,SAAQA,IAAI,EAAEC,YAAY,EAAEC,GAAG,EAAEC,GAAG,EAAEC,GAAG,QAAO,kBAAkB;AAElE,OAAO,MAAMC,SAASJ,aAAa;IACjCK,yBAAyBF,IAAI;QAC3BG,MAAM;QACNC,SAAS;IACX;IACAC,uBAAuBN,IAAI;QACzBI,MAAM;QACNC,SAAS;IACX;IACAE,uBAAuBP,IAAI;QACzBI,MAAM;QACNC,SAAS;IACX;IACAG,uBAAuBR,IAAI;QACzBI,MAAM;QACNC,SAAS;IACX;IACAI,8BAA8BT,IAAI;QAChCI,MAAM;QACNC,SAASK;IACX;IACAC,kCAAkCX,IAAI;QACpCI,MAAM;QACNC,SAASK;IACX;IACAE,iCAAiCf,KAAK;QACpCO,MAAM;QACNC,SAAS;IACX;IACAQ,uBAAuBd,IAAI;QACzBK,MAAM;QACNC,SAAS;IACX;IACAS,kCAAkCf,IAAI;QACpCK,MAAM;QACNC,SAAS;IACX;IACAU,gCAAgChB,IAAI;QAClCK,MAAM;QACNC,SAAS;IACX;IACAW,wCAAwCjB,IAAI;QAC1CK,MAAM;QACNC,SAAS;IACX;IACAY,+BAA+BlB,IAAI;QACjCK,MAAM;QACNC,SAAS;IACX;IACAa,6BAA6BnB,IAAI;QAC/BK,MAAM;QACNC,SAAS;IACX;IACAc,4BAA4BpB,IAAI;QAC9BK,MAAM;QACNC,SAAS;IACX;IACAe,0BAA0BrB,IAAI;QAC5BK,MAAM;QACNC,SAAS;IACX;IACAgB,2BAA2BtB,IAAI;QAC7BK,MAAM;QACNC,SAAS;IACX;IACAiB,oBAAoBvB,IAAI;QACtBK,MAAM;QACNC,SAAS;IACX;AACF,GAAG;AAEH,MAAMkB,mBAAmBC,QAAQtB,OAAOO,4BAA4B;AACpE,MAAMgB,uBAAuBD,QAAQtB,OAAOS,gCAAgC;AAC5E,IAAIY,qBAAqBE,sBAAsB;IAC7C,MAAM,IAAIC,MACR;AAEJ;AAEA,yFAAyF;AACzF,0FAA0F;AAC1F,MAAMC,0BAA0B;AAChC,IACEzB,OAAOkB,wBAAwB,GAAG,KAClClB,OAAOkB,wBAAwB,GAAGO,yBAClC;IACA,MAAM,IAAID,MACR,CAAC,+CAA+C,EAAEC,wBAAwB,MAAM,EAAEzB,OAAOkB,wBAAwB,EAAE;AAEvH;AACA,IAAIlB,OAAOmB,yBAAyB,GAAG,GAAG;IACxC,MAAM,IAAIK,MACR,CAAC,kDAAkD,EAAExB,OAAOmB,yBAAyB,EAAE;AAE3F;AAEA,uFAAuF;AACvF,0FAA0F;AAC1F,iFAAiF;AACjF,IAAInB,OAAOgB,2BAA2B,GAAGhB,OAAOiB,0BAA0B,EAAE;IAC1E,MAAM,IAAIO,MACR,CAAC,6BAA6B,EAAExB,OAAOgB,2BAA2B,CAAC,yCAAyC,EAAEhB,OAAOiB,0BAA0B,CAAC,EAAE,CAAC;AAEvJ;AAEA,qFAAqF;AACrF,kEAAkE;AAClE,IAAI,CAACS,OAAOC,SAAS,CAAC3B,OAAOoB,kBAAkB,KAAKpB,OAAOoB,kBAAkB,GAAG,GAAG;IACjF,MAAM,IAAII,MACR,CAAC,oBAAoB,EAAExB,OAAOoB,kBAAkB,CAAC,sCAAsC,CAAC;AAE5F;AAEA,OAAO,SAASQ,kCAAkCC,uBAA+B;IAC/E,IACE,CAACH,OAAOI,QAAQ,CAAC9B,OAAOe,6BAA6B,KACrDf,OAAOe,6BAA6B,IAAIc,yBACxC;QACA,MAAM,IAAIL,MACR,CAAC,+BAA+B,EAAExB,OAAOe,6BAA6B,CAAC,+CAA+C,EAAEc,wBAAwB,kIAAkI,CAAC;IAEvR;AACF"}
|
|
@@ -1,8 +1,5 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
|
-
type
|
|
3
|
-
config: Record<string, unknown>;
|
|
4
|
-
} | undefined>;
|
|
5
|
-
export declare function setStepLookupForTesting(lookup: StepLookup): () => void;
|
|
2
|
+
import type { WorkflowsModuleClient } from '@shipfox/api-workflows-dto/inter-module';
|
|
6
3
|
export interface AppendLogsParams {
|
|
7
4
|
jobId: string;
|
|
8
5
|
workspaceId: string;
|
|
@@ -23,6 +20,5 @@ export interface AppendLogsResult {
|
|
|
23
20
|
* job contend on its single accounting row, so the path is multi-instance safe
|
|
24
21
|
* but not lock-free.
|
|
25
22
|
*/
|
|
26
|
-
export declare function appendLogs(params: AppendLogsParams): Promise<AppendLogsResult>;
|
|
27
|
-
export {};
|
|
23
|
+
export declare function appendLogs(params: AppendLogsParams, workflows?: WorkflowsModuleClient): Promise<AppendLogsResult>;
|
|
28
24
|
//# sourceMappingURL=append-logs.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"append-logs.d.ts","sourceRoot":"","sources":["../../src/core/append-logs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"append-logs.d.ts","sourceRoot":"","sources":["../../src/core/append-logs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AAQnC,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,yCAAyC,CAAC;AAwBnF,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACjB;AAoND;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,gBAAgB,EACxB,SAAS,CAAC,EAAE,qBAAqB,GAChC,OAAO,CAAC,gBAAgB,CAAC,CAmG3B"}
|
package/dist/core/append-logs.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
|
-
import { DEFAULT_HARNESS
|
|
2
|
+
import { DEFAULT_HARNESS } from '@shipfox/api-agent-dto';
|
|
3
3
|
import { parseLogRecordLine, parseRawLogRecordLine } from '@shipfox/api-logs-dto';
|
|
4
|
-
import { getStepById } from '@shipfox/api-workflows';
|
|
5
4
|
import { logger } from '@shipfox/node-opentelemetry';
|
|
6
5
|
import { config } from '#config.js';
|
|
7
6
|
import { accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped } from '#db/accounting.js';
|
|
@@ -13,14 +12,6 @@ import { allowedBudget } from './budget.js';
|
|
|
13
12
|
import { closeStream, controlTombstone } from './close-stream.js';
|
|
14
13
|
import { MalformedLogChunkError, OffsetGapError } from './errors.js';
|
|
15
14
|
import { parseSessionRecord } from './session/parse-session.js';
|
|
16
|
-
let stepLookup = getStepById;
|
|
17
|
-
export function setStepLookupForTesting(lookup) {
|
|
18
|
-
const previous = stepLookup;
|
|
19
|
-
stepLookup = lookup;
|
|
20
|
-
return ()=>{
|
|
21
|
-
stepLookup = previous;
|
|
22
|
-
};
|
|
23
|
-
}
|
|
24
15
|
/**
|
|
25
16
|
* Pure pre-transaction parse. Requires whole, newline-terminated lines so
|
|
26
17
|
* `committed_length` always lands on a line boundary (one body = one chunk = whole
|
|
@@ -87,10 +78,10 @@ export function setStepLookupForTesting(lookup) {
|
|
|
87
78
|
return undefined;
|
|
88
79
|
}
|
|
89
80
|
}
|
|
90
|
-
async function getSessionHarness(stepId) {
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
81
|
+
async function getSessionHarness(workflows, stepId) {
|
|
82
|
+
return workflows ? (await workflows.getStepLogContext({
|
|
83
|
+
stepId
|
|
84
|
+
})).harness : DEFAULT_HARNESS;
|
|
94
85
|
}
|
|
95
86
|
/**
|
|
96
87
|
* Empty-body heartbeat: report the current committed length without materializing
|
|
@@ -214,7 +205,7 @@ function agentSessionRecord(record) {
|
|
|
214
205
|
* conditional UPDATEs, not an explicit `SELECT ... FOR UPDATE`. Appends for one
|
|
215
206
|
* job contend on its single accounting row, so the path is multi-instance safe
|
|
216
207
|
* but not lock-free.
|
|
217
|
-
*/ export async function appendLogs(params) {
|
|
208
|
+
*/ export async function appendLogs(params, workflows) {
|
|
218
209
|
let parsed;
|
|
219
210
|
try {
|
|
220
211
|
parsed = parseAppendBody(params.body);
|
|
@@ -230,7 +221,7 @@ function agentSessionRecord(record) {
|
|
|
230
221
|
throw error;
|
|
231
222
|
}
|
|
232
223
|
const { declaredTotalBytes } = parsed;
|
|
233
|
-
const sessionHarness = parsed.hasAgentSessionRecord ? await getSessionHarness(params.stepId) : DEFAULT_HARNESS;
|
|
224
|
+
const sessionHarness = parsed.hasAgentSessionRecord ? await getSessionHarness(workflows, params.stepId) : DEFAULT_HARNESS;
|
|
234
225
|
const stored = buildStoredBody(parsed.records, sessionHarness);
|
|
235
226
|
const commitByteLen = params.body.length;
|
|
236
227
|
const metrics = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/core/append-logs.ts"],"sourcesContent":["import {Buffer} from 'node:buffer';\nimport {DEFAULT_HARNESS, type Harness, harnessSchema} from '@shipfox/api-agent-dto';\nimport {\n type LogRecord,\n parseLogRecordLine,\n parseRawLogRecordLine,\n type RawLogRecord,\n} from '@shipfox/api-logs-dto';\nimport {getStepById} from '@shipfox/api-workflows';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {config} from '#config.js';\nimport {accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped} from '#db/accounting.js';\nimport {insertChunk} from '#db/chunks.js';\nimport {db, type Transaction} from '#db/db.js';\nimport {\n casExtendCommittedLength,\n getAttemptStream,\n getOrCreateAttemptStreamWithStatus,\n setDeclaredTotalBytes,\n} from '#db/streams.js';\nimport {\n type LogRecordMetricKind,\n recordAppendedCount,\n streamClosedCount,\n streamOpenedCount,\n} from '#metrics/instance.js';\nimport {allowedBudget} from './budget.js';\nimport {closeStream, controlTombstone} from './close-stream.js';\nimport {MalformedLogChunkError, OffsetGapError} from './errors.js';\nimport {parseSessionRecord} from './session/parse-session.js';\nimport type {AgentSessionRecord} from './session/session-record.js';\n\ntype StepLookup = (stepId: string) => Promise<{config: Record<string, unknown>} | undefined>;\n\nlet stepLookup: StepLookup = getStepById;\n\nexport function setStepLookupForTesting(lookup: StepLookup): () => void {\n const previous = stepLookup;\n stepLookup = lookup;\n return () => {\n stepLookup = previous;\n };\n}\n\nexport interface AppendLogsParams {\n jobId: string;\n workspaceId: string;\n projectId: string;\n workflowRunAttemptId: string;\n stepId: string;\n attempt: number;\n offset: number;\n body: Buffer;\n}\n\nexport interface AppendLogsResult {\n committedLength: number;\n capped: boolean;\n}\n\ninterface ParsedBody {\n declaredTotalBytes?: number;\n records: RawLogRecord[];\n hasAgentSessionRecord: boolean;\n}\n\n/**\n * Pure pre-transaction parse. Requires whole, newline-terminated lines so\n * `committed_length` always lands on a line boundary (one body = one chunk = whole\n * lines; no line ever spans two chunks). Runs before any lock is taken, so a\n * malformed or large body never holds a row. The offset CAS uses the raw append byte length; the\n * budget charges the normalized body built from these parsed records.\n *\n * Each line is validated against the raw record union (a forged server-only\n * `capped`/`runner_lost` fails here); the declared total is pulled from an `end`\n * record. A line that is a valid server-only record under the read union surfaces\n * its type via `forgedType` for the narrowed audit warn.\n */\nfunction parseAppendBody(body: Buffer): ParsedBody {\n if (body.length === 0) return {records: [], hasAgentSessionRecord: false};\n\n const text = body.toString('utf8');\n if (!text.endsWith('\\n')) {\n throw new MalformedLogChunkError('append body must end with a newline (whole records only)');\n }\n const lines = text.split('\\n');\n lines.pop();\n\n let declaredTotalBytes: number | undefined;\n const records: RawLogRecord[] = [];\n let hasAgentSessionRecord = false;\n for (const line of lines) {\n let record: ReturnType<typeof parseRawLogRecordLine>;\n try {\n record = parseRawLogRecordLine(line);\n } catch {\n throw new MalformedLogChunkError(\n 'append body contains an invalid NDJSON record',\n detectForgedType(line),\n );\n }\n records.push(record);\n if (record.type === 'end') {\n declaredTotalBytes = record.total_bytes;\n }\n // An agent_session line is one whole entry in one record; bound its size here (the DTO\n // schema leaves `data` uncapped because it cannot read this runtime config). An over-cap\n // line is rejected so a single entry can never blow the request body or the spool window.\n if (\n record.type === 'agent_session' &&\n Buffer.byteLength(line, 'utf8') > config.LOG_MAX_SESSION_LINE_BYTES\n ) {\n throw new MalformedLogChunkError(\n `agent_session line exceeds ${config.LOG_MAX_SESSION_LINE_BYTES} bytes`,\n );\n }\n if (record.type === 'agent_session') {\n hasAgentSessionRecord = true;\n }\n }\n\n return declaredTotalBytes === undefined\n ? {records, hasAgentSessionRecord}\n : {declaredTotalBytes, records, hasAgentSessionRecord};\n}\n\n/**\n * A line that fails the raw write union but is a valid record under the read union\n * can only be a server-only `capped`/`runner_lost` tombstone — i.e. a forgery\n * attempt. Returns its type for the audit warn, or undefined for plain garbage.\n */\nfunction detectForgedType(line: string): string | undefined {\n try {\n return parseLogRecordLine(line).type;\n } catch {\n return undefined;\n }\n}\n\nasync function getSessionHarness(stepId: string): Promise<Harness> {\n const step = await stepLookup(stepId);\n const parsed = harnessSchema.safeParse(step?.config.harness);\n return parsed.success ? parsed.data : DEFAULT_HARNESS;\n}\n\n/**\n * Empty-body heartbeat: report the current committed length without materializing\n * a stream, so a runner cannot mint unbounded rows with empty appends.\n */\nasync function readHeartbeat(tx: Transaction, params: AppendLogsParams): Promise<AppendLogsResult> {\n const existing = await getAttemptStream(tx, {\n jobId: params.jobId,\n stepId: params.stepId,\n attempt: params.attempt,\n });\n return {\n committedLength: existing?.committedLength ?? 0,\n capped: await isJobCapped(tx, params.jobId),\n };\n}\n\ninterface StoreChunkParams {\n params: AppendLogsParams;\n streamId: string;\n body: Buffer;\n committedLength: number;\n declaredTotalBytes: number | undefined;\n}\n\ninterface StoreChunkResult extends AppendLogsResult {\n /**\n * Whether the runner chunk was persisted. False only when the job was already\n * capped and the body (including any `end` record) was dropped, so the stream is\n * not whole and must not be declared-closed.\n */\n stored: boolean;\n recordCounts: Partial<Record<LogRecord['type'], number>>;\n}\n\n/**\n * Accrues the stored bytes, persists the chunk, and trips the per-job cap when\n * this append crosses the budget. Runs only after the offset-CAS extended\n * `committed_length`, so the committed length already reflects the accepted bytes.\n */\nasync function storeChunk(\n tx: Transaction,\n {params, streamId, body, committedLength, declaredTotalBytes}: StoreChunkParams,\n): Promise<StoreChunkResult> {\n await ensureJobAccounting(tx, {jobId: params.jobId, workspaceId: params.workspaceId});\n const storedByteLen = body.length;\n const accrued = await accrueStoredBytes(tx, {jobId: params.jobId, delta: storedByteLen});\n\n // Already capped: accept-and-drop. committed_length has advanced so the runner\n // drains its spool cleanly instead of retry-looping; nothing is stored.\n if (!accrued) return {committedLength, capped: true, stored: false, recordCounts: {}};\n\n await insertChunk(tx, {\n streamId,\n streamOffset: params.offset,\n byteLen: storedByteLen,\n data: body,\n origin: 'runner',\n });\n if (declaredTotalBytes !== undefined) {\n await setDeclaredTotalBytes(tx, {streamId, declaredTotalBytes});\n }\n\n const allowed = allowedBudget({\n baseBytes: config.LOG_BUDGET_BASE_BYTES,\n ratePerMinuteBytes: config.LOG_BUDGET_RATE_BYTES_PER_MINUTE,\n elapsedMs: Date.now() - accrued.startedAt.getTime(),\n });\n if (accrued.used <= allowed) {\n return {committedLength, capped: false, stored: true, recordCounts: {}};\n }\n\n // Over budget. No hard ceiling: this crossing append is stored in full (overshoot\n // bounded by one body). Claim the cap once and inject an in-band `capped` tombstone\n // for the winner.\n const won = await claimCap(tx, params.jobId);\n if (won) {\n const tombstone = controlTombstone('capped');\n await insertChunk(tx, {\n streamId,\n streamOffset: committedLength,\n byteLen: tombstone.length,\n data: tombstone,\n origin: 'control',\n });\n }\n return {\n committedLength,\n capped: true,\n stored: true,\n recordCounts: won ? {capped: 1} : {},\n };\n}\n\nfunction buildStoredBody(\n records: readonly RawLogRecord[],\n harness: Harness,\n): {body: Buffer; recordCounts: Partial<Record<LogRecord['type'], number>>} {\n const storedRecords: LogRecord[] = [];\n for (const record of records) {\n if (record.type !== 'agent_session') {\n storedRecords.push(record);\n continue;\n }\n\n for (const row of parseSessionRecord(agentSessionRecord(record), harness)) {\n storedRecords.push({v: 1, ts: row.timestamp, type: 'agent_session', row});\n }\n }\n\n const body = Buffer.from(storedRecords.map((record) => `${JSON.stringify(record)}\\n`).join(''));\n const recordCounts: Partial<Record<LogRecord['type'], number>> = {};\n for (const record of storedRecords) {\n recordCounts[record.type] = (recordCounts[record.type] ?? 0) + 1;\n }\n\n return {body, recordCounts};\n}\n\nfunction agentSessionRecord(\n record: Extract<RawLogRecord, {type: 'agent_session'}>,\n): AgentSessionRecord {\n return {data: record.data, ts: record.ts};\n}\n\n/**\n * Concurrency is serialized through Postgres row locks taken implicitly by the\n * conditional UPDATEs, not an explicit `SELECT ... FOR UPDATE`. Appends for one\n * job contend on its single accounting row, so the path is multi-instance safe\n * but not lock-free.\n */\nexport async function appendLogs(params: AppendLogsParams): Promise<AppendLogsResult> {\n let parsed: ParsedBody;\n try {\n parsed = parseAppendBody(params.body);\n } catch (error) {\n // Narrowed audit: only the detectable forgery case, never the payload or a token.\n if (error instanceof MalformedLogChunkError && error.forgedType !== undefined) {\n logger().warn(\n {\n jobId: params.jobId,\n stepId: params.stepId,\n offendingType: error.forgedType,\n },\n 'Rejected forged server-only log record on append',\n );\n }\n throw error;\n }\n const {declaredTotalBytes} = parsed;\n const sessionHarness = parsed.hasAgentSessionRecord\n ? await getSessionHarness(params.stepId)\n : DEFAULT_HARNESS;\n const stored = buildStoredBody(parsed.records, sessionHarness);\n const commitByteLen = params.body.length;\n const metrics = {\n recordCounts: {} as Partial<Record<LogRecordMetricKind, number>>,\n streamClosedReason: undefined as 'declared' | undefined,\n streamOpened: false,\n };\n\n const result = await db().transaction(async (tx) => {\n if (commitByteLen === 0) return readHeartbeat(tx, params);\n\n const {created, stream} = await getOrCreateAttemptStreamWithStatus(tx, {\n jobId: params.jobId,\n stepId: params.stepId,\n attempt: params.attempt,\n workspaceId: params.workspaceId,\n projectId: params.projectId,\n workflowRunAttemptId: params.workflowRunAttemptId,\n });\n metrics.streamOpened = created;\n\n // Closed stream (the runner's end already landed, or the job-terminated sweep ran):\n // accept-and-drop so a late chunk can never race compaction. committed_length is\n // frozen at close, so this reports the final offset and the runner stops cleanly.\n if (stream.state === 'closed') {\n return {committedLength: stream.committedLength, capped: await isJobCapped(tx, params.jobId)};\n }\n\n const cas = await casExtendCommittedLength(tx, {\n streamId: stream.id,\n offset: params.offset,\n byteLen: commitByteLen,\n });\n if (cas.outcome === 'gap') throw new OffsetGapError(cas.committedLength);\n if (cas.outcome === 'retry') {\n return {committedLength: cas.committedLength, capped: await isJobCapped(tx, params.jobId)};\n }\n\n const {\n recordCounts,\n stored: chunkStored,\n ...result\n } = await storeChunk(tx, {\n params,\n streamId: stream.id,\n body: stored.body,\n committedLength: cas.committedLength,\n declaredTotalBytes,\n });\n if (chunkStored) {\n addRecordCounts(metrics.recordCounts, stored.recordCounts);\n }\n addRecordCounts(metrics.recordCounts, recordCounts);\n\n // The runner's end record was committed in this append (offset-CAS guarantees\n // everything before it is already committed), so the stream is whole. Declared-close\n // it in-band so compaction starts at once instead of waiting for the timeout sweep.\n // Only when the chunk was actually stored: an end body dropped because the job was\n // already capped persists nothing, so the stream is not whole and stays open for the\n // timeout sweep to close it as truncated.\n if (declaredTotalBytes !== undefined && chunkStored) {\n const closed = await closeStream(tx, {streamId: stream.id, reason: 'declared'});\n if (closed) metrics.streamClosedReason = 'declared';\n }\n\n return result;\n });\n\n if (metrics.streamOpened) streamOpenedCount.add(1);\n for (const [kind, count] of Object.entries(metrics.recordCounts)) {\n if (count > 0) recordAppendedCount.add(count, {kind: kind as LogRecordMetricKind});\n }\n if (metrics.streamClosedReason) {\n streamClosedCount.add(1, {reason: metrics.streamClosedReason});\n }\n\n return result;\n}\n\nfunction addRecordCounts(\n target: Partial<Record<LogRecordMetricKind, number>>,\n source: Partial<Record<LogRecordMetricKind, number>>,\n): void {\n for (const [kind, count] of Object.entries(source)) {\n target[kind as LogRecordMetricKind] = (target[kind as LogRecordMetricKind] ?? 0) + (count ?? 0);\n }\n}\n"],"names":["Buffer","DEFAULT_HARNESS","harnessSchema","parseLogRecordLine","parseRawLogRecordLine","getStepById","logger","config","accrueStoredBytes","claimCap","ensureJobAccounting","isJobCapped","insertChunk","db","casExtendCommittedLength","getAttemptStream","getOrCreateAttemptStreamWithStatus","setDeclaredTotalBytes","recordAppendedCount","streamClosedCount","streamOpenedCount","allowedBudget","closeStream","controlTombstone","MalformedLogChunkError","OffsetGapError","parseSessionRecord","stepLookup","setStepLookupForTesting","lookup","previous","parseAppendBody","body","length","records","hasAgentSessionRecord","text","toString","endsWith","lines","split","pop","declaredTotalBytes","line","record","detectForgedType","push","type","total_bytes","byteLength","LOG_MAX_SESSION_LINE_BYTES","undefined","getSessionHarness","stepId","step","parsed","safeParse","harness","success","data","readHeartbeat","tx","params","existing","jobId","attempt","committedLength","capped","storeChunk","streamId","workspaceId","storedByteLen","accrued","delta","stored","recordCounts","streamOffset","offset","byteLen","origin","allowed","baseBytes","LOG_BUDGET_BASE_BYTES","ratePerMinuteBytes","LOG_BUDGET_RATE_BYTES_PER_MINUTE","elapsedMs","Date","now","startedAt","getTime","used","won","tombstone","buildStoredBody","storedRecords","row","agentSessionRecord","v","ts","timestamp","from","map","JSON","stringify","join","appendLogs","error","forgedType","warn","offendingType","sessionHarness","commitByteLen","metrics","streamClosedReason","streamOpened","result","transaction","created","stream","projectId","workflowRunAttemptId","state","cas","id","outcome","chunkStored","addRecordCounts","closed","reason","add","kind","count","Object","entries","target","source"],"mappings":"AAAA,SAAQA,MAAM,QAAO,cAAc;AACnC,SAAQC,eAAe,EAAgBC,aAAa,QAAO,yBAAyB;AACpF,SAEEC,kBAAkB,EAClBC,qBAAqB,QAEhB,wBAAwB;AAC/B,SAAQC,WAAW,QAAO,yBAAyB;AACnD,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAQC,iBAAiB,EAAEC,QAAQ,EAAEC,mBAAmB,EAAEC,WAAW,QAAO,oBAAoB;AAChG,SAAQC,WAAW,QAAO,gBAAgB;AAC1C,SAAQC,EAAE,QAAyB,YAAY;AAC/C,SACEC,wBAAwB,EACxBC,gBAAgB,EAChBC,kCAAkC,EAClCC,qBAAqB,QAChB,iBAAiB;AACxB,SAEEC,mBAAmB,EACnBC,iBAAiB,EACjBC,iBAAiB,QACZ,uBAAuB;AAC9B,SAAQC,aAAa,QAAO,cAAc;AAC1C,SAAQC,WAAW,EAAEC,gBAAgB,QAAO,oBAAoB;AAChE,SAAQC,sBAAsB,EAAEC,cAAc,QAAO,cAAc;AACnE,SAAQC,kBAAkB,QAAO,6BAA6B;AAK9D,IAAIC,aAAyBtB;AAE7B,OAAO,SAASuB,wBAAwBC,MAAkB;IACxD,MAAMC,WAAWH;IACjBA,aAAaE;IACb,OAAO;QACLF,aAAaG;IACf;AACF;AAwBA;;;;;;;;;;;CAWC,GACD,SAASC,gBAAgBC,IAAY;IACnC,IAAIA,KAAKC,MAAM,KAAK,GAAG,OAAO;QAACC,SAAS,EAAE;QAAEC,uBAAuB;IAAK;IAExE,MAAMC,OAAOJ,KAAKK,QAAQ,CAAC;IAC3B,IAAI,CAACD,KAAKE,QAAQ,CAAC,OAAO;QACxB,MAAM,IAAId,uBAAuB;IACnC;IACA,MAAMe,QAAQH,KAAKI,KAAK,CAAC;IACzBD,MAAME,GAAG;IAET,IAAIC;IACJ,MAAMR,UAA0B,EAAE;IAClC,IAAIC,wBAAwB;IAC5B,KAAK,MAAMQ,QAAQJ,MAAO;QACxB,IAAIK;QACJ,IAAI;YACFA,SAASxC,sBAAsBuC;QACjC,EAAE,OAAM;YACN,MAAM,IAAInB,uBACR,iDACAqB,iBAAiBF;QAErB;QACAT,QAAQY,IAAI,CAACF;QACb,IAAIA,OAAOG,IAAI,KAAK,OAAO;YACzBL,qBAAqBE,OAAOI,WAAW;QACzC;QACA,uFAAuF;QACvF,yFAAyF;QACzF,0FAA0F;QAC1F,IACEJ,OAAOG,IAAI,KAAK,mBAChB/C,OAAOiD,UAAU,CAACN,MAAM,UAAUpC,OAAO2C,0BAA0B,EACnE;YACA,MAAM,IAAI1B,uBACR,CAAC,2BAA2B,EAAEjB,OAAO2C,0BAA0B,CAAC,MAAM,CAAC;QAE3E;QACA,IAAIN,OAAOG,IAAI,KAAK,iBAAiB;YACnCZ,wBAAwB;QAC1B;IACF;IAEA,OAAOO,uBAAuBS,YAC1B;QAACjB;QAASC;IAAqB,IAC/B;QAACO;QAAoBR;QAASC;IAAqB;AACzD;AAEA;;;;CAIC,GACD,SAASU,iBAAiBF,IAAY;IACpC,IAAI;QACF,OAAOxC,mBAAmBwC,MAAMI,IAAI;IACtC,EAAE,OAAM;QACN,OAAOI;IACT;AACF;AAEA,eAAeC,kBAAkBC,MAAc;IAC7C,MAAMC,OAAO,MAAM3B,WAAW0B;IAC9B,MAAME,SAASrD,cAAcsD,SAAS,CAACF,MAAM/C,OAAOkD;IACpD,OAAOF,OAAOG,OAAO,GAAGH,OAAOI,IAAI,GAAG1D;AACxC;AAEA;;;CAGC,GACD,eAAe2D,cAAcC,EAAe,EAAEC,MAAwB;IACpE,MAAMC,WAAW,MAAMhD,iBAAiB8C,IAAI;QAC1CG,OAAOF,OAAOE,KAAK;QACnBX,QAAQS,OAAOT,MAAM;QACrBY,SAASH,OAAOG,OAAO;IACzB;IACA,OAAO;QACLC,iBAAiBH,UAAUG,mBAAmB;QAC9CC,QAAQ,MAAMxD,YAAYkD,IAAIC,OAAOE,KAAK;IAC5C;AACF;AAoBA;;;;CAIC,GACD,eAAeI,WACbP,EAAe,EACf,EAACC,MAAM,EAAEO,QAAQ,EAAErC,IAAI,EAAEkC,eAAe,EAAExB,kBAAkB,EAAmB;IAE/E,MAAMhC,oBAAoBmD,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAEM,aAAaR,OAAOQ,WAAW;IAAA;IACnF,MAAMC,gBAAgBvC,KAAKC,MAAM;IACjC,MAAMuC,UAAU,MAAMhE,kBAAkBqD,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAES,OAAOF;IAAa;IAEtF,+EAA+E;IAC/E,wEAAwE;IACxE,IAAI,CAACC,SAAS,OAAO;QAACN;QAAiBC,QAAQ;QAAMO,QAAQ;QAAOC,cAAc,CAAC;IAAC;IAEpF,MAAM/D,YAAYiD,IAAI;QACpBQ;QACAO,cAAcd,OAAOe,MAAM;QAC3BC,SAASP;QACTZ,MAAM3B;QACN+C,QAAQ;IACV;IACA,IAAIrC,uBAAuBS,WAAW;QACpC,MAAMlC,sBAAsB4C,IAAI;YAACQ;YAAU3B;QAAkB;IAC/D;IAEA,MAAMsC,UAAU3D,cAAc;QAC5B4D,WAAW1E,OAAO2E,qBAAqB;QACvCC,oBAAoB5E,OAAO6E,gCAAgC;QAC3DC,WAAWC,KAAKC,GAAG,KAAKf,QAAQgB,SAAS,CAACC,OAAO;IACnD;IACA,IAAIjB,QAAQkB,IAAI,IAAIV,SAAS;QAC3B,OAAO;YAACd;YAAiBC,QAAQ;YAAOO,QAAQ;YAAMC,cAAc,CAAC;QAAC;IACxE;IAEA,kFAAkF;IAClF,oFAAoF;IACpF,kBAAkB;IAClB,MAAMgB,MAAM,MAAMlF,SAASoD,IAAIC,OAAOE,KAAK;IAC3C,IAAI2B,KAAK;QACP,MAAMC,YAAYrE,iBAAiB;QACnC,MAAMX,YAAYiD,IAAI;YACpBQ;YACAO,cAAcV;YACdY,SAASc,UAAU3D,MAAM;YACzB0B,MAAMiC;YACNb,QAAQ;QACV;IACF;IACA,OAAO;QACLb;QACAC,QAAQ;QACRO,QAAQ;QACRC,cAAcgB,MAAM;YAACxB,QAAQ;QAAC,IAAI,CAAC;IACrC;AACF;AAEA,SAAS0B,gBACP3D,OAAgC,EAChCuB,OAAgB;IAEhB,MAAMqC,gBAA6B,EAAE;IACrC,KAAK,MAAMlD,UAAUV,QAAS;QAC5B,IAAIU,OAAOG,IAAI,KAAK,iBAAiB;YACnC+C,cAAchD,IAAI,CAACF;YACnB;QACF;QAEA,KAAK,MAAMmD,OAAOrE,mBAAmBsE,mBAAmBpD,SAASa,SAAU;YACzEqC,cAAchD,IAAI,CAAC;gBAACmD,GAAG;gBAAGC,IAAIH,IAAII,SAAS;gBAAEpD,MAAM;gBAAiBgD;YAAG;QACzE;IACF;IAEA,MAAM/D,OAAOhC,OAAOoG,IAAI,CAACN,cAAcO,GAAG,CAAC,CAACzD,SAAW,GAAG0D,KAAKC,SAAS,CAAC3D,QAAQ,EAAE,CAAC,EAAE4D,IAAI,CAAC;IAC3F,MAAM7B,eAA2D,CAAC;IAClE,KAAK,MAAM/B,UAAUkD,cAAe;QAClCnB,YAAY,CAAC/B,OAAOG,IAAI,CAAC,GAAG,AAAC4B,CAAAA,YAAY,CAAC/B,OAAOG,IAAI,CAAC,IAAI,CAAA,IAAK;IACjE;IAEA,OAAO;QAACf;QAAM2C;IAAY;AAC5B;AAEA,SAASqB,mBACPpD,MAAsD;IAEtD,OAAO;QAACe,MAAMf,OAAOe,IAAI;QAAEuC,IAAItD,OAAOsD,EAAE;IAAA;AAC1C;AAEA;;;;;CAKC,GACD,OAAO,eAAeO,WAAW3C,MAAwB;IACvD,IAAIP;IACJ,IAAI;QACFA,SAASxB,gBAAgB+B,OAAO9B,IAAI;IACtC,EAAE,OAAO0E,OAAO;QACd,kFAAkF;QAClF,IAAIA,iBAAiBlF,0BAA0BkF,MAAMC,UAAU,KAAKxD,WAAW;YAC7E7C,SAASsG,IAAI,CACX;gBACE5C,OAAOF,OAAOE,KAAK;gBACnBX,QAAQS,OAAOT,MAAM;gBACrBwD,eAAeH,MAAMC,UAAU;YACjC,GACA;QAEJ;QACA,MAAMD;IACR;IACA,MAAM,EAAChE,kBAAkB,EAAC,GAAGa;IAC7B,MAAMuD,iBAAiBvD,OAAOpB,qBAAqB,GAC/C,MAAMiB,kBAAkBU,OAAOT,MAAM,IACrCpD;IACJ,MAAMyE,SAASmB,gBAAgBtC,OAAOrB,OAAO,EAAE4E;IAC/C,MAAMC,gBAAgBjD,OAAO9B,IAAI,CAACC,MAAM;IACxC,MAAM+E,UAAU;QACdrC,cAAc,CAAC;QACfsC,oBAAoB9D;QACpB+D,cAAc;IAChB;IAEA,MAAMC,SAAS,MAAMtG,KAAKuG,WAAW,CAAC,OAAOvD;QAC3C,IAAIkD,kBAAkB,GAAG,OAAOnD,cAAcC,IAAIC;QAElD,MAAM,EAACuD,OAAO,EAAEC,MAAM,EAAC,GAAG,MAAMtG,mCAAmC6C,IAAI;YACrEG,OAAOF,OAAOE,KAAK;YACnBX,QAAQS,OAAOT,MAAM;YACrBY,SAASH,OAAOG,OAAO;YACvBK,aAAaR,OAAOQ,WAAW;YAC/BiD,WAAWzD,OAAOyD,SAAS;YAC3BC,sBAAsB1D,OAAO0D,oBAAoB;QACnD;QACAR,QAAQE,YAAY,GAAGG;QAEvB,oFAAoF;QACpF,iFAAiF;QACjF,kFAAkF;QAClF,IAAIC,OAAOG,KAAK,KAAK,UAAU;YAC7B,OAAO;gBAACvD,iBAAiBoD,OAAOpD,eAAe;gBAAEC,QAAQ,MAAMxD,YAAYkD,IAAIC,OAAOE,KAAK;YAAC;QAC9F;QAEA,MAAM0D,MAAM,MAAM5G,yBAAyB+C,IAAI;YAC7CQ,UAAUiD,OAAOK,EAAE;YACnB9C,QAAQf,OAAOe,MAAM;YACrBC,SAASiC;QACX;QACA,IAAIW,IAAIE,OAAO,KAAK,OAAO,MAAM,IAAInG,eAAeiG,IAAIxD,eAAe;QACvE,IAAIwD,IAAIE,OAAO,KAAK,SAAS;YAC3B,OAAO;gBAAC1D,iBAAiBwD,IAAIxD,eAAe;gBAAEC,QAAQ,MAAMxD,YAAYkD,IAAIC,OAAOE,KAAK;YAAC;QAC3F;QAEA,MAAM,EACJW,YAAY,EACZD,QAAQmD,WAAW,EACnB,GAAGV,QACJ,GAAG,MAAM/C,WAAWP,IAAI;YACvBC;YACAO,UAAUiD,OAAOK,EAAE;YACnB3F,MAAM0C,OAAO1C,IAAI;YACjBkC,iBAAiBwD,IAAIxD,eAAe;YACpCxB;QACF;QACA,IAAImF,aAAa;YACfC,gBAAgBd,QAAQrC,YAAY,EAAED,OAAOC,YAAY;QAC3D;QACAmD,gBAAgBd,QAAQrC,YAAY,EAAEA;QAEtC,8EAA8E;QAC9E,qFAAqF;QACrF,oFAAoF;QACpF,mFAAmF;QACnF,qFAAqF;QACrF,0CAA0C;QAC1C,IAAIjC,uBAAuBS,aAAa0E,aAAa;YACnD,MAAME,SAAS,MAAMzG,YAAYuC,IAAI;gBAACQ,UAAUiD,OAAOK,EAAE;gBAAEK,QAAQ;YAAU;YAC7E,IAAID,QAAQf,QAAQC,kBAAkB,GAAG;QAC3C;QAEA,OAAOE;IACT;IAEA,IAAIH,QAAQE,YAAY,EAAE9F,kBAAkB6G,GAAG,CAAC;IAChD,KAAK,MAAM,CAACC,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACrB,QAAQrC,YAAY,EAAG;QAChE,IAAIwD,QAAQ,GAAGjH,oBAAoB+G,GAAG,CAACE,OAAO;YAACD,MAAMA;QAA2B;IAClF;IACA,IAAIlB,QAAQC,kBAAkB,EAAE;QAC9B9F,kBAAkB8G,GAAG,CAAC,GAAG;YAACD,QAAQhB,QAAQC,kBAAkB;QAAA;IAC9D;IAEA,OAAOE;AACT;AAEA,SAASW,gBACPQ,MAAoD,EACpDC,MAAoD;IAEpD,KAAK,MAAM,CAACL,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACE,QAAS;QAClDD,MAAM,CAACJ,KAA4B,GAAG,AAACI,CAAAA,MAAM,CAACJ,KAA4B,IAAI,CAAA,IAAMC,CAAAA,SAAS,CAAA;IAC/F;AACF"}
|
|
1
|
+
{"version":3,"sources":["../../src/core/append-logs.ts"],"sourcesContent":["import {Buffer} from 'node:buffer';\nimport {DEFAULT_HARNESS, type Harness} from '@shipfox/api-agent-dto';\nimport {\n type LogRecord,\n parseLogRecordLine,\n parseRawLogRecordLine,\n type RawLogRecord,\n} from '@shipfox/api-logs-dto';\nimport type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {config} from '#config.js';\nimport {accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped} from '#db/accounting.js';\nimport {insertChunk} from '#db/chunks.js';\nimport {db, type Transaction} from '#db/db.js';\nimport {\n casExtendCommittedLength,\n getAttemptStream,\n getOrCreateAttemptStreamWithStatus,\n setDeclaredTotalBytes,\n} from '#db/streams.js';\nimport {\n type LogRecordMetricKind,\n recordAppendedCount,\n streamClosedCount,\n streamOpenedCount,\n} from '#metrics/instance.js';\nimport {allowedBudget} from './budget.js';\nimport {closeStream, controlTombstone} from './close-stream.js';\nimport {MalformedLogChunkError, OffsetGapError} from './errors.js';\nimport {parseSessionRecord} from './session/parse-session.js';\nimport type {AgentSessionRecord} from './session/session-record.js';\n\nexport interface AppendLogsParams {\n jobId: string;\n workspaceId: string;\n projectId: string;\n workflowRunAttemptId: string;\n stepId: string;\n attempt: number;\n offset: number;\n body: Buffer;\n}\n\nexport interface AppendLogsResult {\n committedLength: number;\n capped: boolean;\n}\n\ninterface ParsedBody {\n declaredTotalBytes?: number;\n records: RawLogRecord[];\n hasAgentSessionRecord: boolean;\n}\n\n/**\n * Pure pre-transaction parse. Requires whole, newline-terminated lines so\n * `committed_length` always lands on a line boundary (one body = one chunk = whole\n * lines; no line ever spans two chunks). Runs before any lock is taken, so a\n * malformed or large body never holds a row. The offset CAS uses the raw append byte length; the\n * budget charges the normalized body built from these parsed records.\n *\n * Each line is validated against the raw record union (a forged server-only\n * `capped`/`runner_lost` fails here); the declared total is pulled from an `end`\n * record. A line that is a valid server-only record under the read union surfaces\n * its type via `forgedType` for the narrowed audit warn.\n */\nfunction parseAppendBody(body: Buffer): ParsedBody {\n if (body.length === 0) return {records: [], hasAgentSessionRecord: false};\n\n const text = body.toString('utf8');\n if (!text.endsWith('\\n')) {\n throw new MalformedLogChunkError('append body must end with a newline (whole records only)');\n }\n const lines = text.split('\\n');\n lines.pop();\n\n let declaredTotalBytes: number | undefined;\n const records: RawLogRecord[] = [];\n let hasAgentSessionRecord = false;\n for (const line of lines) {\n let record: ReturnType<typeof parseRawLogRecordLine>;\n try {\n record = parseRawLogRecordLine(line);\n } catch {\n throw new MalformedLogChunkError(\n 'append body contains an invalid NDJSON record',\n detectForgedType(line),\n );\n }\n records.push(record);\n if (record.type === 'end') {\n declaredTotalBytes = record.total_bytes;\n }\n // An agent_session line is one whole entry in one record; bound its size here (the DTO\n // schema leaves `data` uncapped because it cannot read this runtime config). An over-cap\n // line is rejected so a single entry can never blow the request body or the spool window.\n if (\n record.type === 'agent_session' &&\n Buffer.byteLength(line, 'utf8') > config.LOG_MAX_SESSION_LINE_BYTES\n ) {\n throw new MalformedLogChunkError(\n `agent_session line exceeds ${config.LOG_MAX_SESSION_LINE_BYTES} bytes`,\n );\n }\n if (record.type === 'agent_session') {\n hasAgentSessionRecord = true;\n }\n }\n\n return declaredTotalBytes === undefined\n ? {records, hasAgentSessionRecord}\n : {declaredTotalBytes, records, hasAgentSessionRecord};\n}\n\n/**\n * A line that fails the raw write union but is a valid record under the read union\n * can only be a server-only `capped`/`runner_lost` tombstone — i.e. a forgery\n * attempt. Returns its type for the audit warn, or undefined for plain garbage.\n */\nfunction detectForgedType(line: string): string | undefined {\n try {\n return parseLogRecordLine(line).type;\n } catch {\n return undefined;\n }\n}\n\nasync function getSessionHarness(\n workflows: WorkflowsModuleClient | undefined,\n stepId: string,\n): Promise<Harness> {\n return workflows ? (await workflows.getStepLogContext({stepId})).harness : DEFAULT_HARNESS;\n}\n\n/**\n * Empty-body heartbeat: report the current committed length without materializing\n * a stream, so a runner cannot mint unbounded rows with empty appends.\n */\nasync function readHeartbeat(tx: Transaction, params: AppendLogsParams): Promise<AppendLogsResult> {\n const existing = await getAttemptStream(tx, {\n jobId: params.jobId,\n stepId: params.stepId,\n attempt: params.attempt,\n });\n return {\n committedLength: existing?.committedLength ?? 0,\n capped: await isJobCapped(tx, params.jobId),\n };\n}\n\ninterface StoreChunkParams {\n params: AppendLogsParams;\n streamId: string;\n body: Buffer;\n committedLength: number;\n declaredTotalBytes: number | undefined;\n}\n\ninterface StoreChunkResult extends AppendLogsResult {\n /**\n * Whether the runner chunk was persisted. False only when the job was already\n * capped and the body (including any `end` record) was dropped, so the stream is\n * not whole and must not be declared-closed.\n */\n stored: boolean;\n recordCounts: Partial<Record<LogRecord['type'], number>>;\n}\n\n/**\n * Accrues the stored bytes, persists the chunk, and trips the per-job cap when\n * this append crosses the budget. Runs only after the offset-CAS extended\n * `committed_length`, so the committed length already reflects the accepted bytes.\n */\nasync function storeChunk(\n tx: Transaction,\n {params, streamId, body, committedLength, declaredTotalBytes}: StoreChunkParams,\n): Promise<StoreChunkResult> {\n await ensureJobAccounting(tx, {jobId: params.jobId, workspaceId: params.workspaceId});\n const storedByteLen = body.length;\n const accrued = await accrueStoredBytes(tx, {jobId: params.jobId, delta: storedByteLen});\n\n // Already capped: accept-and-drop. committed_length has advanced so the runner\n // drains its spool cleanly instead of retry-looping; nothing is stored.\n if (!accrued) return {committedLength, capped: true, stored: false, recordCounts: {}};\n\n await insertChunk(tx, {\n streamId,\n streamOffset: params.offset,\n byteLen: storedByteLen,\n data: body,\n origin: 'runner',\n });\n if (declaredTotalBytes !== undefined) {\n await setDeclaredTotalBytes(tx, {streamId, declaredTotalBytes});\n }\n\n const allowed = allowedBudget({\n baseBytes: config.LOG_BUDGET_BASE_BYTES,\n ratePerMinuteBytes: config.LOG_BUDGET_RATE_BYTES_PER_MINUTE,\n elapsedMs: Date.now() - accrued.startedAt.getTime(),\n });\n if (accrued.used <= allowed) {\n return {committedLength, capped: false, stored: true, recordCounts: {}};\n }\n\n // Over budget. No hard ceiling: this crossing append is stored in full (overshoot\n // bounded by one body). Claim the cap once and inject an in-band `capped` tombstone\n // for the winner.\n const won = await claimCap(tx, params.jobId);\n if (won) {\n const tombstone = controlTombstone('capped');\n await insertChunk(tx, {\n streamId,\n streamOffset: committedLength,\n byteLen: tombstone.length,\n data: tombstone,\n origin: 'control',\n });\n }\n return {\n committedLength,\n capped: true,\n stored: true,\n recordCounts: won ? {capped: 1} : {},\n };\n}\n\nfunction buildStoredBody(\n records: readonly RawLogRecord[],\n harness: Harness,\n): {body: Buffer; recordCounts: Partial<Record<LogRecord['type'], number>>} {\n const storedRecords: LogRecord[] = [];\n for (const record of records) {\n if (record.type !== 'agent_session') {\n storedRecords.push(record);\n continue;\n }\n\n for (const row of parseSessionRecord(agentSessionRecord(record), harness)) {\n storedRecords.push({v: 1, ts: row.timestamp, type: 'agent_session', row});\n }\n }\n\n const body = Buffer.from(storedRecords.map((record) => `${JSON.stringify(record)}\\n`).join(''));\n const recordCounts: Partial<Record<LogRecord['type'], number>> = {};\n for (const record of storedRecords) {\n recordCounts[record.type] = (recordCounts[record.type] ?? 0) + 1;\n }\n\n return {body, recordCounts};\n}\n\nfunction agentSessionRecord(\n record: Extract<RawLogRecord, {type: 'agent_session'}>,\n): AgentSessionRecord {\n return {data: record.data, ts: record.ts};\n}\n\n/**\n * Concurrency is serialized through Postgres row locks taken implicitly by the\n * conditional UPDATEs, not an explicit `SELECT ... FOR UPDATE`. Appends for one\n * job contend on its single accounting row, so the path is multi-instance safe\n * but not lock-free.\n */\nexport async function appendLogs(\n params: AppendLogsParams,\n workflows?: WorkflowsModuleClient,\n): Promise<AppendLogsResult> {\n let parsed: ParsedBody;\n try {\n parsed = parseAppendBody(params.body);\n } catch (error) {\n // Narrowed audit: only the detectable forgery case, never the payload or a token.\n if (error instanceof MalformedLogChunkError && error.forgedType !== undefined) {\n logger().warn(\n {\n jobId: params.jobId,\n stepId: params.stepId,\n offendingType: error.forgedType,\n },\n 'Rejected forged server-only log record on append',\n );\n }\n throw error;\n }\n const {declaredTotalBytes} = parsed;\n const sessionHarness = parsed.hasAgentSessionRecord\n ? await getSessionHarness(workflows, params.stepId)\n : DEFAULT_HARNESS;\n const stored = buildStoredBody(parsed.records, sessionHarness);\n const commitByteLen = params.body.length;\n const metrics = {\n recordCounts: {} as Partial<Record<LogRecordMetricKind, number>>,\n streamClosedReason: undefined as 'declared' | undefined,\n streamOpened: false,\n };\n\n const result = await db().transaction(async (tx) => {\n if (commitByteLen === 0) return readHeartbeat(tx, params);\n\n const {created, stream} = await getOrCreateAttemptStreamWithStatus(tx, {\n jobId: params.jobId,\n stepId: params.stepId,\n attempt: params.attempt,\n workspaceId: params.workspaceId,\n projectId: params.projectId,\n workflowRunAttemptId: params.workflowRunAttemptId,\n });\n metrics.streamOpened = created;\n\n // Closed stream (the runner's end already landed, or the job-terminated sweep ran):\n // accept-and-drop so a late chunk can never race compaction. committed_length is\n // frozen at close, so this reports the final offset and the runner stops cleanly.\n if (stream.state === 'closed') {\n return {committedLength: stream.committedLength, capped: await isJobCapped(tx, params.jobId)};\n }\n\n const cas = await casExtendCommittedLength(tx, {\n streamId: stream.id,\n offset: params.offset,\n byteLen: commitByteLen,\n });\n if (cas.outcome === 'gap') throw new OffsetGapError(cas.committedLength);\n if (cas.outcome === 'retry') {\n return {committedLength: cas.committedLength, capped: await isJobCapped(tx, params.jobId)};\n }\n\n const {\n recordCounts,\n stored: chunkStored,\n ...result\n } = await storeChunk(tx, {\n params,\n streamId: stream.id,\n body: stored.body,\n committedLength: cas.committedLength,\n declaredTotalBytes,\n });\n if (chunkStored) {\n addRecordCounts(metrics.recordCounts, stored.recordCounts);\n }\n addRecordCounts(metrics.recordCounts, recordCounts);\n\n // The runner's end record was committed in this append (offset-CAS guarantees\n // everything before it is already committed), so the stream is whole. Declared-close\n // it in-band so compaction starts at once instead of waiting for the timeout sweep.\n // Only when the chunk was actually stored: an end body dropped because the job was\n // already capped persists nothing, so the stream is not whole and stays open for the\n // timeout sweep to close it as truncated.\n if (declaredTotalBytes !== undefined && chunkStored) {\n const closed = await closeStream(tx, {streamId: stream.id, reason: 'declared'});\n if (closed) metrics.streamClosedReason = 'declared';\n }\n\n return result;\n });\n\n if (metrics.streamOpened) streamOpenedCount.add(1);\n for (const [kind, count] of Object.entries(metrics.recordCounts)) {\n if (count > 0) recordAppendedCount.add(count, {kind: kind as LogRecordMetricKind});\n }\n if (metrics.streamClosedReason) {\n streamClosedCount.add(1, {reason: metrics.streamClosedReason});\n }\n\n return result;\n}\n\nfunction addRecordCounts(\n target: Partial<Record<LogRecordMetricKind, number>>,\n source: Partial<Record<LogRecordMetricKind, number>>,\n): void {\n for (const [kind, count] of Object.entries(source)) {\n target[kind as LogRecordMetricKind] = (target[kind as LogRecordMetricKind] ?? 0) + (count ?? 0);\n }\n}\n"],"names":["Buffer","DEFAULT_HARNESS","parseLogRecordLine","parseRawLogRecordLine","logger","config","accrueStoredBytes","claimCap","ensureJobAccounting","isJobCapped","insertChunk","db","casExtendCommittedLength","getAttemptStream","getOrCreateAttemptStreamWithStatus","setDeclaredTotalBytes","recordAppendedCount","streamClosedCount","streamOpenedCount","allowedBudget","closeStream","controlTombstone","MalformedLogChunkError","OffsetGapError","parseSessionRecord","parseAppendBody","body","length","records","hasAgentSessionRecord","text","toString","endsWith","lines","split","pop","declaredTotalBytes","line","record","detectForgedType","push","type","total_bytes","byteLength","LOG_MAX_SESSION_LINE_BYTES","undefined","getSessionHarness","workflows","stepId","getStepLogContext","harness","readHeartbeat","tx","params","existing","jobId","attempt","committedLength","capped","storeChunk","streamId","workspaceId","storedByteLen","accrued","delta","stored","recordCounts","streamOffset","offset","byteLen","data","origin","allowed","baseBytes","LOG_BUDGET_BASE_BYTES","ratePerMinuteBytes","LOG_BUDGET_RATE_BYTES_PER_MINUTE","elapsedMs","Date","now","startedAt","getTime","used","won","tombstone","buildStoredBody","storedRecords","row","agentSessionRecord","v","ts","timestamp","from","map","JSON","stringify","join","appendLogs","parsed","error","forgedType","warn","offendingType","sessionHarness","commitByteLen","metrics","streamClosedReason","streamOpened","result","transaction","created","stream","projectId","workflowRunAttemptId","state","cas","id","outcome","chunkStored","addRecordCounts","closed","reason","add","kind","count","Object","entries","target","source"],"mappings":"AAAA,SAAQA,MAAM,QAAO,cAAc;AACnC,SAAQC,eAAe,QAAqB,yBAAyB;AACrE,SAEEC,kBAAkB,EAClBC,qBAAqB,QAEhB,wBAAwB;AAE/B,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAQC,iBAAiB,EAAEC,QAAQ,EAAEC,mBAAmB,EAAEC,WAAW,QAAO,oBAAoB;AAChG,SAAQC,WAAW,QAAO,gBAAgB;AAC1C,SAAQC,EAAE,QAAyB,YAAY;AAC/C,SACEC,wBAAwB,EACxBC,gBAAgB,EAChBC,kCAAkC,EAClCC,qBAAqB,QAChB,iBAAiB;AACxB,SAEEC,mBAAmB,EACnBC,iBAAiB,EACjBC,iBAAiB,QACZ,uBAAuB;AAC9B,SAAQC,aAAa,QAAO,cAAc;AAC1C,SAAQC,WAAW,EAAEC,gBAAgB,QAAO,oBAAoB;AAChE,SAAQC,sBAAsB,EAAEC,cAAc,QAAO,cAAc;AACnE,SAAQC,kBAAkB,QAAO,6BAA6B;AAyB9D;;;;;;;;;;;CAWC,GACD,SAASC,gBAAgBC,IAAY;IACnC,IAAIA,KAAKC,MAAM,KAAK,GAAG,OAAO;QAACC,SAAS,EAAE;QAAEC,uBAAuB;IAAK;IAExE,MAAMC,OAAOJ,KAAKK,QAAQ,CAAC;IAC3B,IAAI,CAACD,KAAKE,QAAQ,CAAC,OAAO;QACxB,MAAM,IAAIV,uBAAuB;IACnC;IACA,MAAMW,QAAQH,KAAKI,KAAK,CAAC;IACzBD,MAAME,GAAG;IAET,IAAIC;IACJ,MAAMR,UAA0B,EAAE;IAClC,IAAIC,wBAAwB;IAC5B,KAAK,MAAMQ,QAAQJ,MAAO;QACxB,IAAIK;QACJ,IAAI;YACFA,SAASnC,sBAAsBkC;QACjC,EAAE,OAAM;YACN,MAAM,IAAIf,uBACR,iDACAiB,iBAAiBF;QAErB;QACAT,QAAQY,IAAI,CAACF;QACb,IAAIA,OAAOG,IAAI,KAAK,OAAO;YACzBL,qBAAqBE,OAAOI,WAAW;QACzC;QACA,uFAAuF;QACvF,yFAAyF;QACzF,0FAA0F;QAC1F,IACEJ,OAAOG,IAAI,KAAK,mBAChBzC,OAAO2C,UAAU,CAACN,MAAM,UAAUhC,OAAOuC,0BAA0B,EACnE;YACA,MAAM,IAAItB,uBACR,CAAC,2BAA2B,EAAEjB,OAAOuC,0BAA0B,CAAC,MAAM,CAAC;QAE3E;QACA,IAAIN,OAAOG,IAAI,KAAK,iBAAiB;YACnCZ,wBAAwB;QAC1B;IACF;IAEA,OAAOO,uBAAuBS,YAC1B;QAACjB;QAASC;IAAqB,IAC/B;QAACO;QAAoBR;QAASC;IAAqB;AACzD;AAEA;;;;CAIC,GACD,SAASU,iBAAiBF,IAAY;IACpC,IAAI;QACF,OAAOnC,mBAAmBmC,MAAMI,IAAI;IACtC,EAAE,OAAM;QACN,OAAOI;IACT;AACF;AAEA,eAAeC,kBACbC,SAA4C,EAC5CC,MAAc;IAEd,OAAOD,YAAY,AAAC,CAAA,MAAMA,UAAUE,iBAAiB,CAAC;QAACD;IAAM,EAAC,EAAGE,OAAO,GAAGjD;AAC7E;AAEA;;;CAGC,GACD,eAAekD,cAAcC,EAAe,EAAEC,MAAwB;IACpE,MAAMC,WAAW,MAAMzC,iBAAiBuC,IAAI;QAC1CG,OAAOF,OAAOE,KAAK;QACnBP,QAAQK,OAAOL,MAAM;QACrBQ,SAASH,OAAOG,OAAO;IACzB;IACA,OAAO;QACLC,iBAAiBH,UAAUG,mBAAmB;QAC9CC,QAAQ,MAAMjD,YAAY2C,IAAIC,OAAOE,KAAK;IAC5C;AACF;AAoBA;;;;CAIC,GACD,eAAeI,WACbP,EAAe,EACf,EAACC,MAAM,EAAEO,QAAQ,EAAElC,IAAI,EAAE+B,eAAe,EAAErB,kBAAkB,EAAmB;IAE/E,MAAM5B,oBAAoB4C,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAEM,aAAaR,OAAOQ,WAAW;IAAA;IACnF,MAAMC,gBAAgBpC,KAAKC,MAAM;IACjC,MAAMoC,UAAU,MAAMzD,kBAAkB8C,IAAI;QAACG,OAAOF,OAAOE,KAAK;QAAES,OAAOF;IAAa;IAEtF,+EAA+E;IAC/E,wEAAwE;IACxE,IAAI,CAACC,SAAS,OAAO;QAACN;QAAiBC,QAAQ;QAAMO,QAAQ;QAAOC,cAAc,CAAC;IAAC;IAEpF,MAAMxD,YAAY0C,IAAI;QACpBQ;QACAO,cAAcd,OAAOe,MAAM;QAC3BC,SAASP;QACTQ,MAAM5C;QACN6C,QAAQ;IACV;IACA,IAAInC,uBAAuBS,WAAW;QACpC,MAAM9B,sBAAsBqC,IAAI;YAACQ;YAAUxB;QAAkB;IAC/D;IAEA,MAAMoC,UAAUrD,cAAc;QAC5BsD,WAAWpE,OAAOqE,qBAAqB;QACvCC,oBAAoBtE,OAAOuE,gCAAgC;QAC3DC,WAAWC,KAAKC,GAAG,KAAKhB,QAAQiB,SAAS,CAACC,OAAO;IACnD;IACA,IAAIlB,QAAQmB,IAAI,IAAIV,SAAS;QAC3B,OAAO;YAACf;YAAiBC,QAAQ;YAAOO,QAAQ;YAAMC,cAAc,CAAC;QAAC;IACxE;IAEA,kFAAkF;IAClF,oFAAoF;IACpF,kBAAkB;IAClB,MAAMiB,MAAM,MAAM5E,SAAS6C,IAAIC,OAAOE,KAAK;IAC3C,IAAI4B,KAAK;QACP,MAAMC,YAAY/D,iBAAiB;QACnC,MAAMX,YAAY0C,IAAI;YACpBQ;YACAO,cAAcV;YACdY,SAASe,UAAUzD,MAAM;YACzB2C,MAAMc;YACNb,QAAQ;QACV;IACF;IACA,OAAO;QACLd;QACAC,QAAQ;QACRO,QAAQ;QACRC,cAAciB,MAAM;YAACzB,QAAQ;QAAC,IAAI,CAAC;IACrC;AACF;AAEA,SAAS2B,gBACPzD,OAAgC,EAChCsB,OAAgB;IAEhB,MAAMoC,gBAA6B,EAAE;IACrC,KAAK,MAAMhD,UAAUV,QAAS;QAC5B,IAAIU,OAAOG,IAAI,KAAK,iBAAiB;YACnC6C,cAAc9C,IAAI,CAACF;YACnB;QACF;QAEA,KAAK,MAAMiD,OAAO/D,mBAAmBgE,mBAAmBlD,SAASY,SAAU;YACzEoC,cAAc9C,IAAI,CAAC;gBAACiD,GAAG;gBAAGC,IAAIH,IAAII,SAAS;gBAAElD,MAAM;gBAAiB8C;YAAG;QACzE;IACF;IAEA,MAAM7D,OAAO1B,OAAO4F,IAAI,CAACN,cAAcO,GAAG,CAAC,CAACvD,SAAW,GAAGwD,KAAKC,SAAS,CAACzD,QAAQ,EAAE,CAAC,EAAE0D,IAAI,CAAC;IAC3F,MAAM9B,eAA2D,CAAC;IAClE,KAAK,MAAM5B,UAAUgD,cAAe;QAClCpB,YAAY,CAAC5B,OAAOG,IAAI,CAAC,GAAG,AAACyB,CAAAA,YAAY,CAAC5B,OAAOG,IAAI,CAAC,IAAI,CAAA,IAAK;IACjE;IAEA,OAAO;QAACf;QAAMwC;IAAY;AAC5B;AAEA,SAASsB,mBACPlD,MAAsD;IAEtD,OAAO;QAACgC,MAAMhC,OAAOgC,IAAI;QAAEoB,IAAIpD,OAAOoD,EAAE;IAAA;AAC1C;AAEA;;;;;CAKC,GACD,OAAO,eAAeO,WACpB5C,MAAwB,EACxBN,SAAiC;IAEjC,IAAImD;IACJ,IAAI;QACFA,SAASzE,gBAAgB4B,OAAO3B,IAAI;IACtC,EAAE,OAAOyE,OAAO;QACd,kFAAkF;QAClF,IAAIA,iBAAiB7E,0BAA0B6E,MAAMC,UAAU,KAAKvD,WAAW;YAC7EzC,SAASiG,IAAI,CACX;gBACE9C,OAAOF,OAAOE,KAAK;gBACnBP,QAAQK,OAAOL,MAAM;gBACrBsD,eAAeH,MAAMC,UAAU;YACjC,GACA;QAEJ;QACA,MAAMD;IACR;IACA,MAAM,EAAC/D,kBAAkB,EAAC,GAAG8D;IAC7B,MAAMK,iBAAiBL,OAAOrE,qBAAqB,GAC/C,MAAMiB,kBAAkBC,WAAWM,OAAOL,MAAM,IAChD/C;IACJ,MAAMgE,SAASoB,gBAAgBa,OAAOtE,OAAO,EAAE2E;IAC/C,MAAMC,gBAAgBnD,OAAO3B,IAAI,CAACC,MAAM;IACxC,MAAM8E,UAAU;QACdvC,cAAc,CAAC;QACfwC,oBAAoB7D;QACpB8D,cAAc;IAChB;IAEA,MAAMC,SAAS,MAAMjG,KAAKkG,WAAW,CAAC,OAAOzD;QAC3C,IAAIoD,kBAAkB,GAAG,OAAOrD,cAAcC,IAAIC;QAElD,MAAM,EAACyD,OAAO,EAAEC,MAAM,EAAC,GAAG,MAAMjG,mCAAmCsC,IAAI;YACrEG,OAAOF,OAAOE,KAAK;YACnBP,QAAQK,OAAOL,MAAM;YACrBQ,SAASH,OAAOG,OAAO;YACvBK,aAAaR,OAAOQ,WAAW;YAC/BmD,WAAW3D,OAAO2D,SAAS;YAC3BC,sBAAsB5D,OAAO4D,oBAAoB;QACnD;QACAR,QAAQE,YAAY,GAAGG;QAEvB,oFAAoF;QACpF,iFAAiF;QACjF,kFAAkF;QAClF,IAAIC,OAAOG,KAAK,KAAK,UAAU;YAC7B,OAAO;gBAACzD,iBAAiBsD,OAAOtD,eAAe;gBAAEC,QAAQ,MAAMjD,YAAY2C,IAAIC,OAAOE,KAAK;YAAC;QAC9F;QAEA,MAAM4D,MAAM,MAAMvG,yBAAyBwC,IAAI;YAC7CQ,UAAUmD,OAAOK,EAAE;YACnBhD,QAAQf,OAAOe,MAAM;YACrBC,SAASmC;QACX;QACA,IAAIW,IAAIE,OAAO,KAAK,OAAO,MAAM,IAAI9F,eAAe4F,IAAI1D,eAAe;QACvE,IAAI0D,IAAIE,OAAO,KAAK,SAAS;YAC3B,OAAO;gBAAC5D,iBAAiB0D,IAAI1D,eAAe;gBAAEC,QAAQ,MAAMjD,YAAY2C,IAAIC,OAAOE,KAAK;YAAC;QAC3F;QAEA,MAAM,EACJW,YAAY,EACZD,QAAQqD,WAAW,EACnB,GAAGV,QACJ,GAAG,MAAMjD,WAAWP,IAAI;YACvBC;YACAO,UAAUmD,OAAOK,EAAE;YACnB1F,MAAMuC,OAAOvC,IAAI;YACjB+B,iBAAiB0D,IAAI1D,eAAe;YACpCrB;QACF;QACA,IAAIkF,aAAa;YACfC,gBAAgBd,QAAQvC,YAAY,EAAED,OAAOC,YAAY;QAC3D;QACAqD,gBAAgBd,QAAQvC,YAAY,EAAEA;QAEtC,8EAA8E;QAC9E,qFAAqF;QACrF,oFAAoF;QACpF,mFAAmF;QACnF,qFAAqF;QACrF,0CAA0C;QAC1C,IAAI9B,uBAAuBS,aAAayE,aAAa;YACnD,MAAME,SAAS,MAAMpG,YAAYgC,IAAI;gBAACQ,UAAUmD,OAAOK,EAAE;gBAAEK,QAAQ;YAAU;YAC7E,IAAID,QAAQf,QAAQC,kBAAkB,GAAG;QAC3C;QAEA,OAAOE;IACT;IAEA,IAAIH,QAAQE,YAAY,EAAEzF,kBAAkBwG,GAAG,CAAC;IAChD,KAAK,MAAM,CAACC,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACrB,QAAQvC,YAAY,EAAG;QAChE,IAAI0D,QAAQ,GAAG5G,oBAAoB0G,GAAG,CAACE,OAAO;YAACD,MAAMA;QAA2B;IAClF;IACA,IAAIlB,QAAQC,kBAAkB,EAAE;QAC9BzF,kBAAkByG,GAAG,CAAC,GAAG;YAACD,QAAQhB,QAAQC,kBAAkB;QAAA;IAC9D;IAEA,OAAOE;AACT;AAEA,SAASW,gBACPQ,MAAoD,EACpDC,MAAoD;IAEpD,KAAK,MAAM,CAACL,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACE,QAAS;QAClDD,MAAM,CAACJ,KAA4B,GAAG,AAACI,CAAAA,MAAM,CAACJ,KAA4B,IAAI,CAAA,IAAMC,CAAAA,SAAS,CAAA;IAC/F;AACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
|
+
import type { WorkflowsModuleClient } from '@shipfox/api-workflows-dto/inter-module';
|
|
1
2
|
import { type ShipfoxModule } from '@shipfox/node-module';
|
|
2
3
|
export { checkBucketReachable } from '#api/object-storage.js';
|
|
3
|
-
export declare
|
|
4
|
+
export declare function createLogsModule({ workflows, jobLeaseTokenTtlSeconds, }: {
|
|
5
|
+
workflows: WorkflowsModuleClient;
|
|
6
|
+
jobLeaseTokenTtlSeconds: number;
|
|
7
|
+
}): ShipfoxModule;
|
|
4
8
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,OAAO,EAAC,KAAK,aAAa,EAAoB,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,yCAAyC,CAAC;AACnF,OAAO,EAAC,KAAK,aAAa,EAAoB,MAAM,sBAAsB,CAAC;AAa3E,OAAO,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AAO5D,wBAAgB,gBAAgB,CAAC,EAC/B,SAAS,EACT,uBAAuB,GACxB,EAAE;IACD,SAAS,EAAE,qBAAqB,CAAC;IACjC,uBAAuB,EAAE,MAAM,CAAC;CACjC,GAAG,aAAa,CAqDhB"}
|
package/dist/index.js
CHANGED
|
@@ -3,11 +3,12 @@ import { fileURLToPath } from 'node:url';
|
|
|
3
3
|
import { LOG_STREAM_CLOSED, logsEventSchemas } from '@shipfox/api-logs-dto';
|
|
4
4
|
import { WORKFLOWS_JOB_TERMINATED, WORKFLOWS_STEP_ATTEMPT_TERMINATED } from '@shipfox/api-workflows-dto';
|
|
5
5
|
import { subscriberFactory } from '@shipfox/node-module';
|
|
6
|
+
import { validateLogStreamReapAfterSeconds } from '#config.js';
|
|
6
7
|
import { db } from '#db/db.js';
|
|
7
8
|
import { migrationsPath } from '#db/index.js';
|
|
8
9
|
import { logsOutbox } from '#db/schema/outbox.js';
|
|
9
10
|
import { registerLogsServiceMetrics } from '#metrics/service.js';
|
|
10
|
-
import {
|
|
11
|
+
import { createLogsRoutes } from '#presentation/routes/index.js';
|
|
11
12
|
import { onJobTerminated } from '#presentation/subscribers/on-job-terminated.js';
|
|
12
13
|
import { onLogStreamClosed } from '#presentation/subscribers/on-log-stream-closed.js';
|
|
13
14
|
import { onStepAttemptTerminated } from '#presentation/subscribers/on-step-attempt-terminated.js';
|
|
@@ -17,65 +18,68 @@ export { checkBucketReachable } from '#api/object-storage.js';
|
|
|
17
18
|
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
18
19
|
const workflowsPath = resolve(packageRoot, 'dist/temporal/workflows/index.js');
|
|
19
20
|
const subscriber = subscriberFactory();
|
|
20
|
-
export
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
},
|
|
26
|
-
routes: logsRoutes,
|
|
27
|
-
metrics: registerLogsServiceMetrics,
|
|
28
|
-
// `logs.stream.closed` is written by the close paths: the job-terminated subscriber
|
|
29
|
-
// force-closes streams the runner never ended, and the closed event drives compaction.
|
|
30
|
-
publishers: [
|
|
31
|
-
{
|
|
32
|
-
name: 'logs',
|
|
33
|
-
table: logsOutbox,
|
|
21
|
+
export function createLogsModule({ workflows, jobLeaseTokenTtlSeconds }) {
|
|
22
|
+
validateLogStreamReapAfterSeconds(jobLeaseTokenTtlSeconds);
|
|
23
|
+
return {
|
|
24
|
+
name: 'logs',
|
|
25
|
+
database: {
|
|
34
26
|
db,
|
|
35
|
-
|
|
36
|
-
}
|
|
37
|
-
],
|
|
38
|
-
subscribers: [
|
|
39
|
-
subscriber(WORKFLOWS_STEP_ATTEMPT_TERMINATED, onStepAttemptTerminated),
|
|
40
|
-
subscriber(WORKFLOWS_JOB_TERMINATED, onJobTerminated),
|
|
41
|
-
subscriber(LOG_STREAM_CLOSED, onLogStreamClosed)
|
|
42
|
-
],
|
|
43
|
-
workers: [
|
|
44
|
-
{
|
|
45
|
-
taskQueue: LOGS_LIFECYCLE_TASK_QUEUE,
|
|
46
|
-
workflowsPath,
|
|
47
|
-
activities: createLogsActivities,
|
|
48
|
-
// Temporal cron schedules are fixed after creation, so making cadence configurable would
|
|
49
|
-
// look adjustable while keeping the old schedule. Tune LOG_RETENTION_DAYS for the horizon.
|
|
50
|
-
workflows: [
|
|
51
|
-
{
|
|
52
|
-
name: 'retentionSweepCron',
|
|
53
|
-
id: 'logs-retention-sweep',
|
|
54
|
-
cronSchedule: '0 * * * *'
|
|
55
|
-
},
|
|
56
|
-
// Offset from retention's top-of-hour sweep; stale-stream age is governed by
|
|
57
|
-
// LOG_STREAM_REAP_AFTER_SECONDS.
|
|
58
|
-
{
|
|
59
|
-
name: 'reapStaleOpenStreamsCron',
|
|
60
|
-
id: 'logs-reap-stale-open-streams',
|
|
61
|
-
cronSchedule: '5,15,25,35,45,55 * * * *'
|
|
62
|
-
}
|
|
63
|
-
]
|
|
27
|
+
migrationsPath
|
|
64
28
|
},
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
29
|
+
routes: createLogsRoutes(workflows),
|
|
30
|
+
metrics: registerLogsServiceMetrics,
|
|
31
|
+
// `logs.stream.closed` is written by the close paths: the job-terminated subscriber
|
|
32
|
+
// force-closes streams the runner never ended, and the closed event drives compaction.
|
|
33
|
+
publishers: [
|
|
34
|
+
{
|
|
35
|
+
name: 'logs',
|
|
36
|
+
table: logsOutbox,
|
|
37
|
+
db,
|
|
38
|
+
eventSchemas: logsEventSchemas
|
|
39
|
+
}
|
|
40
|
+
],
|
|
41
|
+
subscribers: [
|
|
42
|
+
subscriber(WORKFLOWS_STEP_ATTEMPT_TERMINATED, onStepAttemptTerminated),
|
|
43
|
+
subscriber(WORKFLOWS_JOB_TERMINATED, onJobTerminated),
|
|
44
|
+
subscriber(LOG_STREAM_CLOSED, onLogStreamClosed)
|
|
45
|
+
],
|
|
46
|
+
workers: [
|
|
47
|
+
{
|
|
48
|
+
taskQueue: LOGS_LIFECYCLE_TASK_QUEUE,
|
|
49
|
+
workflowsPath,
|
|
50
|
+
activities: createLogsActivities,
|
|
51
|
+
// Temporal cron schedules are fixed after creation, so making cadence configurable would
|
|
52
|
+
// look adjustable while keeping the old schedule. Tune LOG_RETENTION_DAYS for the horizon.
|
|
53
|
+
workflows: [
|
|
54
|
+
{
|
|
55
|
+
name: 'retentionSweepCron',
|
|
56
|
+
id: 'logs-retention-sweep',
|
|
57
|
+
cronSchedule: '0 * * * *'
|
|
58
|
+
},
|
|
59
|
+
// Offset from retention's top-of-hour sweep; stale-stream age is governed by
|
|
60
|
+
// LOG_STREAM_REAP_AFTER_SECONDS.
|
|
61
|
+
{
|
|
62
|
+
name: 'reapStaleOpenStreamsCron',
|
|
63
|
+
id: 'logs-reap-stale-open-streams',
|
|
64
|
+
cronSchedule: '5,15,25,35,45,55 * * * *'
|
|
65
|
+
}
|
|
66
|
+
]
|
|
67
|
+
},
|
|
68
|
+
// Compaction runs on its own queue so long uploads cannot starve the lifecycle sweep.
|
|
69
|
+
{
|
|
70
|
+
taskQueue: LOGS_COMPACTION_TASK_QUEUE,
|
|
71
|
+
workflowsPath,
|
|
72
|
+
activities: createLogsActivities,
|
|
73
|
+
workflows: [
|
|
74
|
+
{
|
|
75
|
+
name: 'compactionReconcileCron',
|
|
76
|
+
id: 'logs-compaction-reconcile',
|
|
77
|
+
cronSchedule: '*/10 * * * *'
|
|
78
|
+
}
|
|
79
|
+
]
|
|
80
|
+
}
|
|
81
|
+
]
|
|
82
|
+
};
|
|
83
|
+
}
|
|
80
84
|
|
|
81
85
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import {dirname, resolve} from 'node:path';\nimport {fileURLToPath} from 'node:url';\nimport {LOG_STREAM_CLOSED, type LogsEventMap, logsEventSchemas} from '@shipfox/api-logs-dto';\nimport {\n WORKFLOWS_JOB_TERMINATED,\n WORKFLOWS_STEP_ATTEMPT_TERMINATED,\n type WorkflowsEventMapDto,\n} from '@shipfox/api-workflows-dto';\nimport {type ShipfoxModule, subscriberFactory} from '@shipfox/node-module';\nimport {db} from '#db/db.js';\nimport {migrationsPath} from '#db/index.js';\nimport {logsOutbox} from '#db/schema/outbox.js';\nimport {registerLogsServiceMetrics} from '#metrics/service.js';\nimport {
|
|
1
|
+
{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import {dirname, resolve} from 'node:path';\nimport {fileURLToPath} from 'node:url';\nimport {LOG_STREAM_CLOSED, type LogsEventMap, logsEventSchemas} from '@shipfox/api-logs-dto';\nimport {\n WORKFLOWS_JOB_TERMINATED,\n WORKFLOWS_STEP_ATTEMPT_TERMINATED,\n type WorkflowsEventMapDto,\n} from '@shipfox/api-workflows-dto';\nimport type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';\nimport {type ShipfoxModule, subscriberFactory} from '@shipfox/node-module';\nimport {validateLogStreamReapAfterSeconds} from '#config.js';\nimport {db} from '#db/db.js';\nimport {migrationsPath} from '#db/index.js';\nimport {logsOutbox} from '#db/schema/outbox.js';\nimport {registerLogsServiceMetrics} from '#metrics/service.js';\nimport {createLogsRoutes} from '#presentation/routes/index.js';\nimport {onJobTerminated} from '#presentation/subscribers/on-job-terminated.js';\nimport {onLogStreamClosed} from '#presentation/subscribers/on-log-stream-closed.js';\nimport {onStepAttemptTerminated} from '#presentation/subscribers/on-step-attempt-terminated.js';\nimport {createLogsActivities} from '#temporal/activities/index.js';\nimport {LOGS_COMPACTION_TASK_QUEUE, LOGS_LIFECYCLE_TASK_QUEUE} from '#temporal/constants.js';\n\nexport {checkBucketReachable} from '#api/object-storage.js';\n\nconst packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');\nconst workflowsPath = resolve(packageRoot, 'dist/temporal/workflows/index.js');\n\nconst subscriber = subscriberFactory<WorkflowsEventMapDto & LogsEventMap>();\n\nexport function createLogsModule({\n workflows,\n jobLeaseTokenTtlSeconds,\n}: {\n workflows: WorkflowsModuleClient;\n jobLeaseTokenTtlSeconds: number;\n}): ShipfoxModule {\n validateLogStreamReapAfterSeconds(jobLeaseTokenTtlSeconds);\n\n return {\n name: 'logs',\n database: {db, migrationsPath},\n routes: createLogsRoutes(workflows),\n metrics: registerLogsServiceMetrics,\n // `logs.stream.closed` is written by the close paths: the job-terminated subscriber\n // force-closes streams the runner never ended, and the closed event drives compaction.\n publishers: [{name: 'logs', table: logsOutbox, db, eventSchemas: logsEventSchemas}],\n subscribers: [\n subscriber(WORKFLOWS_STEP_ATTEMPT_TERMINATED, onStepAttemptTerminated),\n subscriber(WORKFLOWS_JOB_TERMINATED, onJobTerminated),\n subscriber(LOG_STREAM_CLOSED, onLogStreamClosed),\n ],\n workers: [\n {\n taskQueue: LOGS_LIFECYCLE_TASK_QUEUE,\n workflowsPath,\n activities: createLogsActivities,\n // Temporal cron schedules are fixed after creation, so making cadence configurable would\n // look adjustable while keeping the old schedule. Tune LOG_RETENTION_DAYS for the horizon.\n workflows: [\n {\n name: 'retentionSweepCron',\n id: 'logs-retention-sweep',\n cronSchedule: '0 * * * *',\n },\n // Offset from retention's top-of-hour sweep; stale-stream age is governed by\n // LOG_STREAM_REAP_AFTER_SECONDS.\n {\n name: 'reapStaleOpenStreamsCron',\n id: 'logs-reap-stale-open-streams',\n cronSchedule: '5,15,25,35,45,55 * * * *',\n },\n ],\n },\n // Compaction runs on its own queue so long uploads cannot starve the lifecycle sweep.\n {\n taskQueue: LOGS_COMPACTION_TASK_QUEUE,\n workflowsPath,\n activities: createLogsActivities,\n workflows: [\n {\n name: 'compactionReconcileCron',\n id: 'logs-compaction-reconcile',\n cronSchedule: '*/10 * * * *',\n },\n ],\n },\n ],\n };\n}\n"],"names":["dirname","resolve","fileURLToPath","LOG_STREAM_CLOSED","logsEventSchemas","WORKFLOWS_JOB_TERMINATED","WORKFLOWS_STEP_ATTEMPT_TERMINATED","subscriberFactory","validateLogStreamReapAfterSeconds","db","migrationsPath","logsOutbox","registerLogsServiceMetrics","createLogsRoutes","onJobTerminated","onLogStreamClosed","onStepAttemptTerminated","createLogsActivities","LOGS_COMPACTION_TASK_QUEUE","LOGS_LIFECYCLE_TASK_QUEUE","checkBucketReachable","packageRoot","url","workflowsPath","subscriber","createLogsModule","workflows","jobLeaseTokenTtlSeconds","name","database","routes","metrics","publishers","table","eventSchemas","subscribers","workers","taskQueue","activities","id","cronSchedule"],"mappings":"AAAA,SAAQA,OAAO,EAAEC,OAAO,QAAO,YAAY;AAC3C,SAAQC,aAAa,QAAO,WAAW;AACvC,SAAQC,iBAAiB,EAAqBC,gBAAgB,QAAO,wBAAwB;AAC7F,SACEC,wBAAwB,EACxBC,iCAAiC,QAE5B,6BAA6B;AAEpC,SAA4BC,iBAAiB,QAAO,uBAAuB;AAC3E,SAAQC,iCAAiC,QAAO,aAAa;AAC7D,SAAQC,EAAE,QAAO,YAAY;AAC7B,SAAQC,cAAc,QAAO,eAAe;AAC5C,SAAQC,UAAU,QAAO,uBAAuB;AAChD,SAAQC,0BAA0B,QAAO,sBAAsB;AAC/D,SAAQC,gBAAgB,QAAO,gCAAgC;AAC/D,SAAQC,eAAe,QAAO,iDAAiD;AAC/E,SAAQC,iBAAiB,QAAO,oDAAoD;AACpF,SAAQC,uBAAuB,QAAO,0DAA0D;AAChG,SAAQC,oBAAoB,QAAO,gCAAgC;AACnE,SAAQC,0BAA0B,EAAEC,yBAAyB,QAAO,yBAAyB;AAE7F,SAAQC,oBAAoB,QAAO,yBAAyB;AAE5D,MAAMC,cAAcpB,QAAQD,QAAQE,cAAc,YAAYoB,GAAG,IAAI;AACrE,MAAMC,gBAAgBtB,QAAQoB,aAAa;AAE3C,MAAMG,aAAajB;AAEnB,OAAO,SAASkB,iBAAiB,EAC/BC,SAAS,EACTC,uBAAuB,EAIxB;IACCnB,kCAAkCmB;IAElC,OAAO;QACLC,MAAM;QACNC,UAAU;YAACpB;YAAIC;QAAc;QAC7BoB,QAAQjB,iBAAiBa;QACzBK,SAASnB;QACT,oFAAoF;QACpF,uFAAuF;QACvFoB,YAAY;YAAC;gBAACJ,MAAM;gBAAQK,OAAOtB;gBAAYF;gBAAIyB,cAAc9B;YAAgB;SAAE;QACnF+B,aAAa;YACXX,WAAWlB,mCAAmCU;YAC9CQ,WAAWnB,0BAA0BS;YACrCU,WAAWrB,mBAAmBY;SAC/B;QACDqB,SAAS;YACP;gBACEC,WAAWlB;gBACXI;gBACAe,YAAYrB;gBACZ,yFAAyF;gBACzF,2FAA2F;gBAC3FS,WAAW;oBACT;wBACEE,MAAM;wBACNW,IAAI;wBACJC,cAAc;oBAChB;oBACA,6EAA6E;oBAC7E,iCAAiC;oBACjC;wBACEZ,MAAM;wBACNW,IAAI;wBACJC,cAAc;oBAChB;iBACD;YACH;YACA,sFAAsF;YACtF;gBACEH,WAAWnB;gBACXK;gBACAe,YAAYrB;gBACZS,WAAW;oBACT;wBACEE,MAAM;wBACNW,IAAI;wBACJC,cAAc;oBAChB;iBACD;YACH;SACD;IACH;AACF"}
|
|
@@ -1,2 +1,3 @@
|
|
|
1
|
-
|
|
1
|
+
import type { WorkflowsModuleClient } from '@shipfox/api-workflows-dto/inter-module';
|
|
2
|
+
export declare function createAppendLogsRoute(workflows: WorkflowsModuleClient): import("@shipfox/node-fastify").RouteDefinition;
|
|
2
3
|
//# sourceMappingURL=append-logs.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"append-logs.d.ts","sourceRoot":"","sources":["../../../src/presentation/routes/append-logs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"append-logs.d.ts","sourceRoot":"","sources":["../../../src/presentation/routes/append-logs.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,yCAAyC,CAAC;AAMnF,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,qBAAqB,mDAwDrE"}
|