@shipfox/api-logs 5.0.0 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/.turbo/turbo-build.log +4 -4
  2. package/CHANGELOG.md +32 -0
  3. package/dist/config.d.ts +1 -0
  4. package/dist/config.d.ts.map +1 -1
  5. package/dist/config.js +5 -10
  6. package/dist/config.js.map +1 -1
  7. package/dist/core/append-logs.d.ts +2 -6
  8. package/dist/core/append-logs.d.ts.map +1 -1
  9. package/dist/core/append-logs.js +7 -16
  10. package/dist/core/append-logs.js.map +1 -1
  11. package/dist/index.d.ts +5 -1
  12. package/dist/index.d.ts.map +1 -1
  13. package/dist/index.js +63 -59
  14. package/dist/index.js.map +1 -1
  15. package/dist/presentation/routes/append-logs.d.ts +2 -1
  16. package/dist/presentation/routes/append-logs.d.ts.map +1 -1
  17. package/dist/presentation/routes/append-logs.js +61 -61
  18. package/dist/presentation/routes/append-logs.js.map +1 -1
  19. package/dist/presentation/routes/index.d.ts +2 -1
  20. package/dist/presentation/routes/index.d.ts.map +1 -1
  21. package/dist/presentation/routes/index.js +25 -23
  22. package/dist/presentation/routes/index.js.map +1 -1
  23. package/dist/tsconfig.test.tsbuildinfo +1 -1
  24. package/package.json +16 -13
  25. package/src/config.test.ts +8 -32
  26. package/src/config.ts +10 -15
  27. package/src/core/append-logs.test.ts +13 -19
  28. package/src/core/append-logs.ts +12 -20
  29. package/src/index.ts +63 -51
  30. package/src/presentation/routes/append-logs.test.ts +3 -2
  31. package/src/presentation/routes/append-logs.ts +54 -50
  32. package/src/presentation/routes/index.ts +22 -19
  33. package/src/presentation/routes/read-logs.test.ts +3 -2
  34. package/test/env.ts +1 -3
  35. package/test/fixtures/lease-token.ts +2 -1
  36. package/test/fixtures/workflows-client.ts +17 -0
  37. package/tsconfig.build.tsbuildinfo +1 -1
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@shipfox/api-logs",
3
3
  "license": "MIT",
4
- "version": "5.0.0",
4
+ "version": "6.0.0",
5
5
  "repository": {
6
6
  "type": "git",
7
7
  "url": "git+https://github.com/ShipfoxHQ/shipfox.git",
@@ -29,19 +29,18 @@
29
29
  "@temporalio/workflow": "1.18.1",
30
30
  "drizzle-orm": "^0.45.2",
31
31
  "zod": "^4.4.3",
32
- "@shipfox/api-auth-context": "5.0.0",
33
- "@shipfox/api-agent-dto": "5.0.0",
32
+ "@shipfox/api-auth-context": "6.0.0",
33
+ "@shipfox/api-agent-dto": "6.0.0",
34
34
  "@shipfox/api-logs-dto": "5.0.0",
35
- "@shipfox/api-workflows": "5.0.0",
36
- "@shipfox/api-workflows-dto": "5.0.0",
35
+ "@shipfox/api-workflows-dto": "6.0.0",
37
36
  "@shipfox/config": "1.2.2",
38
- "@shipfox/node-drizzle": "0.3.1",
39
- "@shipfox/node-fastify": "0.2.3",
40
- "@shipfox/node-module": "0.3.2",
37
+ "@shipfox/node-drizzle": "0.3.2",
38
+ "@shipfox/node-fastify": "0.2.4",
39
+ "@shipfox/node-module": "0.4.0",
41
40
  "@shipfox/node-opentelemetry": "0.5.2",
42
- "@shipfox/node-outbox": "0.2.3",
41
+ "@shipfox/node-outbox": "0.2.4",
43
42
  "@shipfox/node-postgres": "0.4.2",
44
- "@shipfox/node-temporal": "0.3.1"
43
+ "@shipfox/node-temporal": "0.3.2"
45
44
  },
46
45
  "devDependencies": {
47
46
  "@temporalio/client": "1.18.1",
@@ -53,10 +52,13 @@
53
52
  "fastify": "^5.3.3",
54
53
  "fastify-type-provider-zod": "^6.0.0",
55
54
  "fishery": "^2.4.0",
56
- "@shipfox/api-auth": "5.0.0",
57
- "@shipfox/api-auth-dto": "5.0.0",
55
+ "@shipfox/node-auth-root-key": "0.2.0",
56
+ "@shipfox/api-auth": "6.0.0",
57
+ "@shipfox/api-auth-dto": "6.0.0",
58
+ "@shipfox/inter-module": "0.2.0",
58
59
  "@shipfox/biome": "1.8.2",
59
- "@shipfox/node-jwt": "0.2.1",
60
+ "@shipfox/depcruise": "1.0.2",
61
+ "@shipfox/node-jwt": "0.3.0",
60
62
  "@shipfox/swc": "1.2.6",
61
63
  "@shipfox/ts-config": "1.3.8",
62
64
  "@shipfox/typescript": "1.1.7",
@@ -66,6 +68,7 @@
66
68
  "build": "shipfox-swc && shipfox-temporal-bundle dist/temporal/workflows/index.js",
67
69
  "check": "shipfox-biome-check",
68
70
  "check:fix": "shipfox-biome-check --write",
71
+ "depcruise": "shipfox-depcruise",
69
72
  "test": "shipfox-vitest-run",
70
73
  "test:watch": "shipfox-vitest-watch",
71
74
  "type": "shipfox-tsc-check",
@@ -47,48 +47,24 @@ describe('LOG_STREAM_REAP_AFTER_SECONDS validation', () => {
47
47
  vi.resetModules();
48
48
  });
49
49
 
50
- // Reaping a stream a still-valid lease is appending to silently truncates live logs, so a value
51
- // at or below the job lease TTL (default 90m = 5400s) must fail startup.
52
50
  it.each([
53
- '0',
54
- '-100',
55
- '5400',
56
- ])('fails startup when LOG_STREAM_REAP_AFTER_SECONDS is %s (at or below the default lease TTL)', async (value) => {
57
- vi.stubEnv('LOG_STREAM_REAP_AFTER_SECONDS', value);
51
+ 0, -100, 5400,
52
+ ])('rejects a reaper window of %d seconds for a 90-minute Auth lease', async (value) => {
53
+ vi.stubEnv('LOG_STREAM_REAP_AFTER_SECONDS', String(value));
58
54
  vi.resetModules();
59
55
 
60
- await expect(import('#config.js')).rejects.toThrow('LOG_STREAM_REAP_AFTER_SECONDS');
61
- });
62
-
63
- it('accepts a value above the lease TTL', async () => {
64
- vi.stubEnv('LOG_STREAM_REAP_AFTER_SECONDS', '9000');
65
- vi.resetModules();
66
-
67
- const {config} = await import('#config.js');
56
+ const {validateLogStreamReapAfterSeconds} = await import('#config.js');
68
57
 
69
- expect(config.LOG_STREAM_REAP_AFTER_SECONDS).toBe(9000);
58
+ expect(() => validateLogStreamReapAfterSeconds(5400)).toThrow('LOG_STREAM_REAP_AFTER_SECONDS');
70
59
  });
71
60
 
72
- // The check reads the actual configured lease TTL, not a hardcoded floor: raising the lease past
73
- // the reaper window must fail startup even though the value clears the default lease.
74
- it('fails when AUTH_JOB_LEASE_TOKEN_EXPIRES_IN is raised above the reaper window', async () => {
75
- vi.stubEnv('AUTH_JOB_LEASE_TOKEN_EXPIRES_IN', '180m');
76
- vi.stubEnv('LOG_STREAM_REAP_AFTER_SECONDS', '9000');
77
- vi.resetModules();
78
-
79
- await expect(import('#config.js')).rejects.toThrow('LOG_STREAM_REAP_AFTER_SECONDS');
80
- });
81
-
82
- // The converse: a small reaper window is fine when the lease is correspondingly small, proving
83
- // the bound tracks the lease rather than a fixed 5400s floor.
84
- it('accepts a small reaper window when the lease TTL is correspondingly small', async () => {
85
- vi.stubEnv('AUTH_JOB_LEASE_TOKEN_EXPIRES_IN', '5m');
61
+ it('accepts a reaper window above the Auth lease lifetime', async () => {
86
62
  vi.stubEnv('LOG_STREAM_REAP_AFTER_SECONDS', '600');
87
63
  vi.resetModules();
88
64
 
89
- const {config} = await import('#config.js');
65
+ const {validateLogStreamReapAfterSeconds} = await import('#config.js');
90
66
 
91
- expect(config.LOG_STREAM_REAP_AFTER_SECONDS).toBe(600);
67
+ expect(() => validateLogStreamReapAfterSeconds(300)).not.toThrow();
92
68
  });
93
69
  });
94
70
 
package/src/config.ts CHANGED
@@ -1,6 +1,4 @@
1
- import {config as authConfig} from '@shipfox/api-auth/config';
2
1
  import {bool, createConfig, num, str, url} from '@shipfox/config';
3
- import {durationToSeconds} from '@shipfox/node-jwt';
4
2
 
5
3
  export const config = createConfig({
6
4
  LOG_STORAGE_S3_ENDPOINT: url({
@@ -48,7 +46,7 @@ export const config = createConfig({
48
46
  default: 900,
49
47
  }),
50
48
  LOG_STREAM_REAP_AFTER_SECONDS: num({
51
- desc: 'How long a log stream may stay open before the reaper cron force-closes it as abandoned. This is the backstop for a stream a runner started after its job already went terminal (the one-shot close sweep had already run); the reaper marks it truncated so it re-enters the compaction and retention lifecycle. Must be greater than AUTH_JOB_LEASE_TOKEN_EXPIRES_IN (the job lease lifetime, default 90 minutes) so a still-valid lease can never be force-closed mid-append, which would silently truncate live logs. Defaults to 7200 seconds (2 hours).',
49
+ desc: 'How long a log stream may stay open before the reaper cron force-closes it as abandoned. This is the backstop for a stream a runner started after its job already went terminal (the one-shot close sweep had already run); the reaper marks it truncated so it re-enters the compaction and retention lifecycle. The application validates that this exceeds the Auth job lease lifetime. Defaults to 7200 seconds (2 hours).',
52
50
  default: 7200,
53
51
  }),
54
52
  LOG_APPEND_BODY_LIMIT_BYTES: num({
@@ -115,16 +113,13 @@ if (!Number.isInteger(config.LOG_RETENTION_DAYS) || config.LOG_RETENTION_DAYS <
115
113
  );
116
114
  }
117
115
 
118
- // A stream still accepts appends until its job lease expires, so reaping it any sooner would
119
- // truncate live logs. Validate against the real lease TTL read from auth's config (not a
120
- // hardcoded floor), so raising AUTH_JOB_LEASE_TOKEN_EXPIRES_IN can never silently outrun the
121
- // reaper window.
122
- const jobLeaseTtlSeconds = durationToSeconds(authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN);
123
- if (
124
- !Number.isFinite(config.LOG_STREAM_REAP_AFTER_SECONDS) ||
125
- config.LOG_STREAM_REAP_AFTER_SECONDS <= jobLeaseTtlSeconds
126
- ) {
127
- throw new Error(
128
- `LOG_STREAM_REAP_AFTER_SECONDS (${config.LOG_STREAM_REAP_AFTER_SECONDS}) must be greater than the job lease TTL (${jobLeaseTtlSeconds}s, from AUTH_JOB_LEASE_TOKEN_EXPIRES_IN=${authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN}); a smaller value would let the reaper force-close a stream a still-valid lease is appending to and silently truncate live logs.`,
129
- );
116
+ export function validateLogStreamReapAfterSeconds(jobLeaseTokenTtlSeconds: number): void {
117
+ if (
118
+ !Number.isFinite(config.LOG_STREAM_REAP_AFTER_SECONDS) ||
119
+ config.LOG_STREAM_REAP_AFTER_SECONDS <= jobLeaseTokenTtlSeconds
120
+ ) {
121
+ throw new Error(
122
+ `LOG_STREAM_REAP_AFTER_SECONDS (${config.LOG_STREAM_REAP_AFTER_SECONDS}) must be greater than the Auth job lease TTL (${jobLeaseTokenTtlSeconds}s); a smaller value would let the reaper force-close a stream a still-valid lease is appending to and silently truncate live logs.`,
123
+ );
124
+ }
130
125
  }
@@ -1,5 +1,8 @@
1
1
  import {type LogRecord, parseLogRecordLine} from '@shipfox/api-logs-dto';
2
- import {appendLogs, setStepLookupForTesting} from '#core/append-logs.js';
2
+ import {workflowsInterModuleContract} from '@shipfox/api-workflows-dto/inter-module';
3
+ import {defineInterModulePresentation} from '@shipfox/inter-module';
4
+ import {createFakeInterModuleClients} from '@shipfox/node-module/inter-module/testing';
5
+ import {appendLogs} from '#core/append-logs.js';
3
6
  import {LeaseStreamMismatchError, MalformedLogChunkError, OffsetGapError} from '#core/errors.js';
4
7
  import {jobAccountingFactory} from '#test/factories/job-accounting.js';
5
8
  import {
@@ -13,8 +16,6 @@ import {
13
16
  } from '#test/fixtures/ndjson.js';
14
17
  import {findAccounting, findStream, listChunks, listStreamClosedEvents} from '#test/queries.js';
15
18
 
16
- const mockSteps = new Map<string, {config: Record<string, unknown>}>();
17
-
18
19
  interface Ctx {
19
20
  jobId: string;
20
21
  stepId: string;
@@ -48,20 +49,6 @@ function recordsFromChunks(chunks: Awaited<ReturnType<typeof listChunks>>): LogR
48
49
  }
49
50
 
50
51
  describe('appendLogs', () => {
51
- let restoreStepLookup: () => void;
52
-
53
- beforeAll(() => {
54
- restoreStepLookup = setStepLookupForTesting((stepId) => Promise.resolve(mockSteps.get(stepId)));
55
- });
56
-
57
- afterAll(() => {
58
- restoreStepLookup();
59
- });
60
-
61
- beforeEach(() => {
62
- mockSteps.clear();
63
- });
64
-
65
52
  describe('offset-CAS', () => {
66
53
  it('extends committed_length and stores one runner chunk on an in-order append', async () => {
67
54
  const ctx = newCtx();
@@ -203,7 +190,14 @@ describe('appendLogs', () => {
203
190
  it('selects the Claude parser from the step harness', async () => {
204
191
  const ctx = newCtx();
205
192
  await allowLargeLogBudget(ctx);
206
- mockSteps.set(ctx.stepId, {config: {harness: 'claude'}});
193
+ const workflows = createFakeInterModuleClients({
194
+ workflows: defineInterModulePresentation(workflowsInterModuleContract, {
195
+ startRunFromTrigger: vi.fn(),
196
+ deliverEventToJobListener: vi.fn(),
197
+ getStepLogContext: () => ({harness: 'claude' as const}),
198
+ getLeasedAgentToolContext: vi.fn(),
199
+ }),
200
+ }).workflows;
207
201
  const body = ndjsonBody(
208
202
  sessionLine(
209
203
  JSON.stringify({
@@ -216,7 +210,7 @@ describe('appendLogs', () => {
216
210
  ),
217
211
  );
218
212
 
219
- await appendLogs({...ctx, attempt: 1, offset: 0, body});
213
+ await appendLogs({...ctx, attempt: 1, offset: 0, body}, workflows);
220
214
 
221
215
  const stream = await findStream({...ctx, attempt: 1});
222
216
  const rows = recordsFromChunks(await listChunks(stream?.id as string)).map((record) => {
@@ -1,12 +1,12 @@
1
1
  import {Buffer} from 'node:buffer';
2
- import {DEFAULT_HARNESS, type Harness, harnessSchema} from '@shipfox/api-agent-dto';
2
+ import {DEFAULT_HARNESS, type Harness} from '@shipfox/api-agent-dto';
3
3
  import {
4
4
  type LogRecord,
5
5
  parseLogRecordLine,
6
6
  parseRawLogRecordLine,
7
7
  type RawLogRecord,
8
8
  } from '@shipfox/api-logs-dto';
9
- import {getStepById} from '@shipfox/api-workflows';
9
+ import type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';
10
10
  import {logger} from '@shipfox/node-opentelemetry';
11
11
  import {config} from '#config.js';
12
12
  import {accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped} from '#db/accounting.js';
@@ -30,18 +30,6 @@ import {MalformedLogChunkError, OffsetGapError} from './errors.js';
30
30
  import {parseSessionRecord} from './session/parse-session.js';
31
31
  import type {AgentSessionRecord} from './session/session-record.js';
32
32
 
33
- type StepLookup = (stepId: string) => Promise<{config: Record<string, unknown>} | undefined>;
34
-
35
- let stepLookup: StepLookup = getStepById;
36
-
37
- export function setStepLookupForTesting(lookup: StepLookup): () => void {
38
- const previous = stepLookup;
39
- stepLookup = lookup;
40
- return () => {
41
- stepLookup = previous;
42
- };
43
- }
44
-
45
33
  export interface AppendLogsParams {
46
34
  jobId: string;
47
35
  workspaceId: string;
@@ -137,10 +125,11 @@ function detectForgedType(line: string): string | undefined {
137
125
  }
138
126
  }
139
127
 
140
- async function getSessionHarness(stepId: string): Promise<Harness> {
141
- const step = await stepLookup(stepId);
142
- const parsed = harnessSchema.safeParse(step?.config.harness);
143
- return parsed.success ? parsed.data : DEFAULT_HARNESS;
128
+ async function getSessionHarness(
129
+ workflows: WorkflowsModuleClient | undefined,
130
+ stepId: string,
131
+ ): Promise<Harness> {
132
+ return workflows ? (await workflows.getStepLogContext({stepId})).harness : DEFAULT_HARNESS;
144
133
  }
145
134
 
146
135
  /**
@@ -273,7 +262,10 @@ function agentSessionRecord(
273
262
  * job contend on its single accounting row, so the path is multi-instance safe
274
263
  * but not lock-free.
275
264
  */
276
- export async function appendLogs(params: AppendLogsParams): Promise<AppendLogsResult> {
265
+ export async function appendLogs(
266
+ params: AppendLogsParams,
267
+ workflows?: WorkflowsModuleClient,
268
+ ): Promise<AppendLogsResult> {
277
269
  let parsed: ParsedBody;
278
270
  try {
279
271
  parsed = parseAppendBody(params.body);
@@ -293,7 +285,7 @@ export async function appendLogs(params: AppendLogsParams): Promise<AppendLogsRe
293
285
  }
294
286
  const {declaredTotalBytes} = parsed;
295
287
  const sessionHarness = parsed.hasAgentSessionRecord
296
- ? await getSessionHarness(params.stepId)
288
+ ? await getSessionHarness(workflows, params.stepId)
297
289
  : DEFAULT_HARNESS;
298
290
  const stored = buildStoredBody(parsed.records, sessionHarness);
299
291
  const commitByteLen = params.body.length;
package/src/index.ts CHANGED
@@ -6,12 +6,14 @@ import {
6
6
  WORKFLOWS_STEP_ATTEMPT_TERMINATED,
7
7
  type WorkflowsEventMapDto,
8
8
  } from '@shipfox/api-workflows-dto';
9
+ import type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';
9
10
  import {type ShipfoxModule, subscriberFactory} from '@shipfox/node-module';
11
+ import {validateLogStreamReapAfterSeconds} from '#config.js';
10
12
  import {db} from '#db/db.js';
11
13
  import {migrationsPath} from '#db/index.js';
12
14
  import {logsOutbox} from '#db/schema/outbox.js';
13
15
  import {registerLogsServiceMetrics} from '#metrics/service.js';
14
- import {logsRoutes} from '#presentation/routes/index.js';
16
+ import {createLogsRoutes} from '#presentation/routes/index.js';
15
17
  import {onJobTerminated} from '#presentation/subscribers/on-job-terminated.js';
16
18
  import {onLogStreamClosed} from '#presentation/subscribers/on-log-stream-closed.js';
17
19
  import {onStepAttemptTerminated} from '#presentation/subscribers/on-step-attempt-terminated.js';
@@ -25,53 +27,63 @@ const workflowsPath = resolve(packageRoot, 'dist/temporal/workflows/index.js');
25
27
 
26
28
  const subscriber = subscriberFactory<WorkflowsEventMapDto & LogsEventMap>();
27
29
 
28
- export const logsModule: ShipfoxModule = {
29
- name: 'logs',
30
- database: {db, migrationsPath},
31
- routes: logsRoutes,
32
- metrics: registerLogsServiceMetrics,
33
- // `logs.stream.closed` is written by the close paths: the job-terminated subscriber
34
- // force-closes streams the runner never ended, and the closed event drives compaction.
35
- publishers: [{name: 'logs', table: logsOutbox, db, eventSchemas: logsEventSchemas}],
36
- subscribers: [
37
- subscriber(WORKFLOWS_STEP_ATTEMPT_TERMINATED, onStepAttemptTerminated),
38
- subscriber(WORKFLOWS_JOB_TERMINATED, onJobTerminated),
39
- subscriber(LOG_STREAM_CLOSED, onLogStreamClosed),
40
- ],
41
- workers: [
42
- {
43
- taskQueue: LOGS_LIFECYCLE_TASK_QUEUE,
44
- workflowsPath,
45
- activities: createLogsActivities,
46
- // Temporal cron schedules are fixed after creation, so making cadence configurable would
47
- // look adjustable while keeping the old schedule. Tune LOG_RETENTION_DAYS for the horizon.
48
- workflows: [
49
- {
50
- name: 'retentionSweepCron',
51
- id: 'logs-retention-sweep',
52
- cronSchedule: '0 * * * *',
53
- },
54
- // Offset from retention's top-of-hour sweep; stale-stream age is governed by
55
- // LOG_STREAM_REAP_AFTER_SECONDS.
56
- {
57
- name: 'reapStaleOpenStreamsCron',
58
- id: 'logs-reap-stale-open-streams',
59
- cronSchedule: '5,15,25,35,45,55 * * * *',
60
- },
61
- ],
62
- },
63
- // Compaction runs on its own queue so long uploads cannot starve the lifecycle sweep.
64
- {
65
- taskQueue: LOGS_COMPACTION_TASK_QUEUE,
66
- workflowsPath,
67
- activities: createLogsActivities,
68
- workflows: [
69
- {
70
- name: 'compactionReconcileCron',
71
- id: 'logs-compaction-reconcile',
72
- cronSchedule: '*/10 * * * *',
73
- },
74
- ],
75
- },
76
- ],
77
- };
30
+ export function createLogsModule({
31
+ workflows,
32
+ jobLeaseTokenTtlSeconds,
33
+ }: {
34
+ workflows: WorkflowsModuleClient;
35
+ jobLeaseTokenTtlSeconds: number;
36
+ }): ShipfoxModule {
37
+ validateLogStreamReapAfterSeconds(jobLeaseTokenTtlSeconds);
38
+
39
+ return {
40
+ name: 'logs',
41
+ database: {db, migrationsPath},
42
+ routes: createLogsRoutes(workflows),
43
+ metrics: registerLogsServiceMetrics,
44
+ // `logs.stream.closed` is written by the close paths: the job-terminated subscriber
45
+ // force-closes streams the runner never ended, and the closed event drives compaction.
46
+ publishers: [{name: 'logs', table: logsOutbox, db, eventSchemas: logsEventSchemas}],
47
+ subscribers: [
48
+ subscriber(WORKFLOWS_STEP_ATTEMPT_TERMINATED, onStepAttemptTerminated),
49
+ subscriber(WORKFLOWS_JOB_TERMINATED, onJobTerminated),
50
+ subscriber(LOG_STREAM_CLOSED, onLogStreamClosed),
51
+ ],
52
+ workers: [
53
+ {
54
+ taskQueue: LOGS_LIFECYCLE_TASK_QUEUE,
55
+ workflowsPath,
56
+ activities: createLogsActivities,
57
+ // Temporal cron schedules are fixed after creation, so making cadence configurable would
58
+ // look adjustable while keeping the old schedule. Tune LOG_RETENTION_DAYS for the horizon.
59
+ workflows: [
60
+ {
61
+ name: 'retentionSweepCron',
62
+ id: 'logs-retention-sweep',
63
+ cronSchedule: '0 * * * *',
64
+ },
65
+ // Offset from retention's top-of-hour sweep; stale-stream age is governed by
66
+ // LOG_STREAM_REAP_AFTER_SECONDS.
67
+ {
68
+ name: 'reapStaleOpenStreamsCron',
69
+ id: 'logs-reap-stale-open-streams',
70
+ cronSchedule: '5,15,25,35,45,55 * * * *',
71
+ },
72
+ ],
73
+ },
74
+ // Compaction runs on its own queue so long uploads cannot starve the lifecycle sweep.
75
+ {
76
+ taskQueue: LOGS_COMPACTION_TASK_QUEUE,
77
+ workflowsPath,
78
+ activities: createLogsActivities,
79
+ workflows: [
80
+ {
81
+ name: 'compactionReconcileCron',
82
+ id: 'logs-compaction-reconcile',
83
+ cronSchedule: '*/10 * * * *',
84
+ },
85
+ ],
86
+ },
87
+ ],
88
+ };
89
+ }
@@ -4,7 +4,8 @@ import {AUTH_USER} from '@shipfox/api-auth-context';
4
4
  import {type AuthMethod, closeApp, createApp, type FastifyInstance} from '@shipfox/node-fastify';
5
5
  import {mintLeaseToken} from '#test/fixtures/lease-token.js';
6
6
  import {endLine, ndjsonBody, outputLine, recordLine} from '#test/fixtures/ndjson.js';
7
- import {logsRoutes} from './index.js';
7
+ import {createTestWorkflowsClient} from '#test/fixtures/workflows-client.js';
8
+ import {createLogsRoutes} from './index.js';
8
9
 
9
10
  const NDJSON = 'application/x-ndjson';
10
11
  const JOB_EXECUTION_ID = '00000000-0000-4000-8000-0000000000ee';
@@ -36,7 +37,7 @@ describe('POST /runs/jobs/current/steps/:stepId/logs', () => {
36
37
  beforeAll(async () => {
37
38
  app = await createApp({
38
39
  auth: [createLeaseTokenAuthMethod(), stubUserAuth],
39
- routes: logsRoutes,
40
+ routes: createLogsRoutes(createTestWorkflowsClient()),
40
41
  swagger: false,
41
42
  });
42
43
  await app.ready();
@@ -5,62 +5,66 @@ import {
5
5
  appendLogsResponseSchema,
6
6
  offsetGapResponseSchema,
7
7
  } from '@shipfox/api-logs-dto';
8
+ import type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';
8
9
  import {ClientError, defineRoute} from '@shipfox/node-fastify';
9
10
  import {z} from 'zod';
10
11
  import {appendLogs} from '#core/append-logs.js';
11
12
  import {LeaseStreamMismatchError, MalformedLogChunkError, OffsetGapError} from '#core/errors.js';
12
13
 
13
- export const appendLogsRoute = defineRoute({
14
- method: 'POST',
15
- path: '/steps/:stepId/logs',
16
- description: 'Append a chunk of logs for a step attempt of the leased job.',
17
- schema: {
18
- params: z.object({stepId: z.string().uuid()}),
19
- querystring: appendLogsQuerySchema,
20
- response: {
21
- 200: appendLogsResponseSchema,
22
- 409: offsetGapResponseSchema,
14
+ export function createAppendLogsRoute(workflows: WorkflowsModuleClient) {
15
+ return defineRoute({
16
+ method: 'POST',
17
+ path: '/steps/:stepId/logs',
18
+ description: 'Append a chunk of logs for a step attempt of the leased job.',
19
+ schema: {
20
+ params: z.object({stepId: z.string().uuid()}),
21
+ querystring: appendLogsQuerySchema,
22
+ response: {
23
+ 200: appendLogsResponseSchema,
24
+ 409: offsetGapResponseSchema,
25
+ },
23
26
  },
24
- },
25
- errorHandler: (error) => {
26
- if (error instanceof OffsetGapError) {
27
- throw new ClientError('Append offset is ahead of the committed length', 'offset-gap', {
28
- status: 409,
29
- details: {committed_length: error.committedLength},
30
- });
31
- }
32
- if (error instanceof MalformedLogChunkError) {
33
- throw new ClientError(error.message, 'malformed-log-chunk', {status: 400});
34
- }
35
- if (error instanceof LeaseStreamMismatchError) {
36
- throw new ClientError(error.message, 'lease-stream-mismatch', {status: 403});
37
- }
38
- throw error;
39
- },
40
- handler: async (request) => {
41
- const leasedJob = requireLeasedJobContext(request);
42
- const {stepId} = request.params;
43
- const {attempt, offset} = request.query;
27
+ errorHandler: (error) => {
28
+ if (error instanceof OffsetGapError) {
29
+ throw new ClientError('Append offset is ahead of the committed length', 'offset-gap', {
30
+ status: 409,
31
+ details: {committed_length: error.committedLength},
32
+ });
33
+ }
34
+ if (error instanceof MalformedLogChunkError) {
35
+ throw new ClientError(error.message, 'malformed-log-chunk', {status: 400});
36
+ }
37
+ if (error instanceof LeaseStreamMismatchError) {
38
+ throw new ClientError(error.message, 'lease-stream-mismatch', {status: 403});
39
+ }
40
+ throw error;
41
+ },
42
+ handler: async (request) => {
43
+ const leasedJob = requireLeasedJobContext(request);
44
+ const {stepId} = request.params;
45
+ const {attempt, offset} = request.query;
44
46
 
45
- // The scoped lease proves membership in the dispatched step attempt; active-step
46
- // completion remains governed by workflow/report state, not the log append route.
47
- if (leasedJob.currentStepId !== stepId || leasedJob.currentStepAttempt !== attempt) {
48
- throw new ClientError('Step not found for leased job execution', 'step-not-found', {
49
- status: 404,
50
- });
51
- }
47
+ if (leasedJob.currentStepId !== stepId || leasedJob.currentStepAttempt !== attempt) {
48
+ throw new ClientError('Step not found for leased job execution', 'step-not-found', {
49
+ status: 404,
50
+ });
51
+ }
52
52
 
53
- const result = await appendLogs({
54
- jobId: leasedJob.jobId,
55
- workspaceId: leasedJob.workspaceId,
56
- projectId: leasedJob.projectId,
57
- workflowRunAttemptId: leasedJob.workflowRunAttemptId,
58
- stepId,
59
- attempt,
60
- offset,
61
- body: (request.body as Buffer | undefined) ?? Buffer.alloc(0),
62
- });
53
+ const result = await appendLogs(
54
+ {
55
+ jobId: leasedJob.jobId,
56
+ workspaceId: leasedJob.workspaceId,
57
+ projectId: leasedJob.projectId,
58
+ workflowRunAttemptId: leasedJob.workflowRunAttemptId,
59
+ stepId,
60
+ attempt,
61
+ offset,
62
+ body: (request.body as Buffer | undefined) ?? Buffer.alloc(0),
63
+ },
64
+ workflows,
65
+ );
63
66
 
64
- return {committed_length: result.committedLength, capped: result.capped};
65
- },
66
- });
67
+ return {committed_length: result.committedLength, capped: result.capped};
68
+ },
69
+ });
70
+ }
@@ -1,27 +1,30 @@
1
1
  import {AUTH_LEASED_JOB, AUTH_USER} from '@shipfox/api-auth-context';
2
+ import type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';
2
3
  import {createRawBodyPlugin, type RouteGroup} from '@shipfox/node-fastify';
3
4
  import {config} from '#config.js';
4
- import {appendLogsRoute} from './append-logs.js';
5
+ import {createAppendLogsRoute} from './append-logs.js';
5
6
  import {readLogsRoute} from './read-logs.js';
6
7
 
7
8
  // Keep the lease-authed append in its own Fastify scope so the raw NDJSON parser does not
8
9
  // disturb the JSON read route (or workflow routes). The body limit also bounds one-append
9
10
  // budget overshoot. The read route is session-authed and workspace-scoped via the row.
10
- export const logsRoutes: RouteGroup[] = [
11
- {
12
- prefix: '/runs/jobs/current',
13
- auth: AUTH_LEASED_JOB,
14
- plugins: [
15
- createRawBodyPlugin({
16
- contentType: 'application/x-ndjson',
17
- bodyLimit: config.LOG_APPEND_BODY_LIMIT_BYTES,
18
- }),
19
- ],
20
- routes: [appendLogsRoute],
21
- },
22
- {
23
- prefix: '/steps',
24
- auth: AUTH_USER,
25
- routes: [readLogsRoute],
26
- },
27
- ];
11
+ export function createLogsRoutes(workflows: WorkflowsModuleClient): RouteGroup[] {
12
+ return [
13
+ {
14
+ prefix: '/runs/jobs/current',
15
+ auth: AUTH_LEASED_JOB,
16
+ plugins: [
17
+ createRawBodyPlugin({
18
+ contentType: 'application/x-ndjson',
19
+ bodyLimit: config.LOG_APPEND_BODY_LIMIT_BYTES,
20
+ }),
21
+ ],
22
+ routes: [createAppendLogsRoute(workflows)],
23
+ },
24
+ {
25
+ prefix: '/steps',
26
+ auth: AUTH_USER,
27
+ routes: [readLogsRoute],
28
+ },
29
+ ];
30
+ }
@@ -35,9 +35,10 @@ import {
35
35
  compactStreamActivity,
36
36
  } from '#temporal/activities/compact-stream.js';
37
37
  import {ndjsonBody, outputLine, recordLine} from '#test/fixtures/ndjson.js';
38
+ import {createTestWorkflowsClient} from '#test/fixtures/workflows-client.js';
38
39
  import {findStream} from '#test/queries.js';
39
40
  import {onStepAttemptTerminated} from '../subscribers/on-step-attempt-terminated.js';
40
- import {logsRoutes} from './index.js';
41
+ import {createLogsRoutes} from './index.js';
41
42
 
42
43
  // AUTH_USER stub: a `Bearer user` request is a member of whatever workspace it names in the
43
44
  // `x-test-workspace` header, so each test grants or withholds access against the arranged
@@ -159,7 +160,7 @@ describe('GET /steps/:stepId/attempts/:attempt/logs', () => {
159
160
  beforeAll(async () => {
160
161
  app = await createApp({
161
162
  auth: [fakeUserAuth, stubLeaseAuth],
162
- routes: logsRoutes,
163
+ routes: createLogsRoutes(createTestWorkflowsClient()),
163
164
  swagger: false,
164
165
  });
165
166
  await app.ready();