@shipfox/api-logs 5.0.0 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +14 -14
- package/CHANGELOG.md +46 -0
- package/dist/api/object-storage.d.ts.map +1 -1
- package/dist/api/object-storage.js +28 -2
- package/dist/api/object-storage.js.map +1 -1
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +5 -10
- package/dist/config.js.map +1 -1
- package/dist/core/append-logs.d.ts +2 -6
- package/dist/core/append-logs.d.ts.map +1 -1
- package/dist/core/append-logs.js +7 -16
- package/dist/core/append-logs.js.map +1 -1
- package/dist/core/reap-stale-open-streams.d.ts.map +1 -1
- package/dist/core/reap-stale-open-streams.js +7 -0
- package/dist/core/reap-stale-open-streams.js.map +1 -1
- package/dist/core/retention.d.ts.map +1 -1
- package/dist/core/retention.js +25 -0
- package/dist/core/retention.js.map +1 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +63 -59
- package/dist/index.js.map +1 -1
- package/dist/presentation/routes/append-logs.d.ts +2 -1
- package/dist/presentation/routes/append-logs.d.ts.map +1 -1
- package/dist/presentation/routes/append-logs.js +61 -61
- package/dist/presentation/routes/append-logs.js.map +1 -1
- package/dist/presentation/routes/index.d.ts +2 -1
- package/dist/presentation/routes/index.d.ts.map +1 -1
- package/dist/presentation/routes/index.js +25 -23
- package/dist/presentation/routes/index.js.map +1 -1
- package/dist/temporal/activities/compact-stream.d.ts.map +1 -1
- package/dist/temporal/activities/compact-stream.js +17 -1
- package/dist/temporal/activities/compact-stream.js.map +1 -1
- package/dist/temporal/activities/compaction-reconcile.d.ts.map +1 -1
- package/dist/temporal/activities/compaction-reconcile.js +7 -0
- package/dist/temporal/activities/compaction-reconcile.js.map +1 -1
- package/dist/temporal/workflows/index.bundle.js +17164 -3306
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +12 -30
- package/src/api/object-storage.ts +21 -2
- package/src/config.test.ts +8 -32
- package/src/config.ts +10 -15
- package/src/core/append-logs.test.ts +13 -19
- package/src/core/append-logs.ts +12 -20
- package/src/core/reap-stale-open-streams.ts +2 -0
- package/src/core/retention.ts +5 -0
- package/src/index.ts +63 -51
- package/src/presentation/routes/append-logs.test.ts +3 -2
- package/src/presentation/routes/append-logs.ts +54 -50
- package/src/presentation/routes/index.ts +22 -19
- package/src/presentation/routes/read-logs.test.ts +3 -2
- package/src/temporal/activities/compact-stream.ts +13 -1
- package/src/temporal/activities/compaction-reconcile.ts +2 -0
- package/test/env.ts +1 -3
- package/test/fixtures/lease-token.ts +2 -1
- package/test/fixtures/workflows-client.ts +17 -0
- package/tsconfig.build.tsbuildinfo +1 -1
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@shipfox/api-logs",
|
|
3
3
|
"license": "MIT",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "7.1.0",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
7
7
|
"url": "git+https://github.com/ShipfoxHQ/shipfox.git",
|
|
@@ -29,43 +29,25 @@
|
|
|
29
29
|
"@temporalio/workflow": "1.18.1",
|
|
30
30
|
"drizzle-orm": "^0.45.2",
|
|
31
31
|
"zod": "^4.4.3",
|
|
32
|
-
"@shipfox/api-auth-context": "
|
|
33
|
-
"@shipfox/api-agent-dto": "
|
|
32
|
+
"@shipfox/api-auth-context": "7.1.0",
|
|
33
|
+
"@shipfox/api-agent-dto": "6.0.0",
|
|
34
34
|
"@shipfox/api-logs-dto": "5.0.0",
|
|
35
|
-
"@shipfox/api-workflows": "
|
|
36
|
-
"@shipfox/api-workflows-dto": "5.0.0",
|
|
35
|
+
"@shipfox/api-workflows-dto": "6.0.0",
|
|
37
36
|
"@shipfox/config": "1.2.2",
|
|
38
|
-
"@shipfox/node-drizzle": "0.3.
|
|
39
|
-
"@shipfox/node-
|
|
40
|
-
"@shipfox/node-
|
|
41
|
-
"@shipfox/node-
|
|
42
|
-
"@shipfox/node-
|
|
37
|
+
"@shipfox/node-drizzle": "0.3.2",
|
|
38
|
+
"@shipfox/node-error-monitoring": "0.2.0",
|
|
39
|
+
"@shipfox/node-fastify": "0.3.0",
|
|
40
|
+
"@shipfox/node-module": "0.5.0",
|
|
41
|
+
"@shipfox/node-opentelemetry": "0.6.0",
|
|
42
|
+
"@shipfox/node-outbox": "0.2.4",
|
|
43
43
|
"@shipfox/node-postgres": "0.4.2",
|
|
44
|
-
"@shipfox/node-temporal": "0.
|
|
45
|
-
},
|
|
46
|
-
"devDependencies": {
|
|
47
|
-
"@temporalio/client": "1.18.1",
|
|
48
|
-
"@temporalio/common": "1.18.1",
|
|
49
|
-
"@temporalio/testing": "1.18.1",
|
|
50
|
-
"@temporalio/worker": "1.18.1",
|
|
51
|
-
"@types/pg": "^8.15.5",
|
|
52
|
-
"drizzle-kit": "^0.31.10",
|
|
53
|
-
"fastify": "^5.3.3",
|
|
54
|
-
"fastify-type-provider-zod": "^6.0.0",
|
|
55
|
-
"fishery": "^2.4.0",
|
|
56
|
-
"@shipfox/api-auth": "5.0.0",
|
|
57
|
-
"@shipfox/api-auth-dto": "5.0.0",
|
|
58
|
-
"@shipfox/biome": "1.8.2",
|
|
59
|
-
"@shipfox/node-jwt": "0.2.1",
|
|
60
|
-
"@shipfox/swc": "1.2.6",
|
|
61
|
-
"@shipfox/ts-config": "1.3.8",
|
|
62
|
-
"@shipfox/typescript": "1.1.7",
|
|
63
|
-
"@shipfox/vitest": "1.2.3"
|
|
44
|
+
"@shipfox/node-temporal": "0.4.0"
|
|
64
45
|
},
|
|
65
46
|
"scripts": {
|
|
66
47
|
"build": "shipfox-swc && shipfox-temporal-bundle dist/temporal/workflows/index.js",
|
|
67
48
|
"check": "shipfox-biome-check",
|
|
68
49
|
"check:fix": "shipfox-biome-check --write",
|
|
50
|
+
"depcruise": "shipfox-depcruise",
|
|
69
51
|
"test": "shipfox-vitest-run",
|
|
70
52
|
"test:watch": "shipfox-vitest-watch",
|
|
71
53
|
"type": "shipfox-tsc-check",
|
|
@@ -9,6 +9,8 @@ import {
|
|
|
9
9
|
} from '@aws-sdk/client-s3';
|
|
10
10
|
import {Upload} from '@aws-sdk/lib-storage';
|
|
11
11
|
import {getSignedUrl} from '@aws-sdk/s3-request-presigner';
|
|
12
|
+
import {reportError} from '@shipfox/node-error-monitoring';
|
|
13
|
+
import {logger} from '@shipfox/node-opentelemetry';
|
|
12
14
|
import {config} from '#config.js';
|
|
13
15
|
import {type LogObjectKeyParams, logObjectKey} from '#core/entities/log-object.js';
|
|
14
16
|
|
|
@@ -140,12 +142,29 @@ export async function putCompactedObject(params: PutCompactedObjectParams): Prom
|
|
|
140
142
|
params.signal.addEventListener(
|
|
141
143
|
'abort',
|
|
142
144
|
() => {
|
|
143
|
-
void upload.abort().catch(() =>
|
|
145
|
+
void upload.abort().catch((error) => {
|
|
146
|
+
logger().error(
|
|
147
|
+
{err: error, objectKey: params.key},
|
|
148
|
+
'Failed to abort multipart log upload',
|
|
149
|
+
);
|
|
150
|
+
reportError(error, {
|
|
151
|
+
boundary: 'logs.cleanup',
|
|
152
|
+
operation: 'abort-multipart-upload',
|
|
153
|
+
extra: {objectKey: params.key},
|
|
154
|
+
});
|
|
155
|
+
});
|
|
144
156
|
},
|
|
145
157
|
{once: true},
|
|
146
158
|
);
|
|
147
159
|
if (params.signal.aborted) {
|
|
148
|
-
await upload.abort().catch(() =>
|
|
160
|
+
await upload.abort().catch((error) => {
|
|
161
|
+
logger().error({err: error, objectKey: params.key}, 'Failed to abort multipart log upload');
|
|
162
|
+
reportError(error, {
|
|
163
|
+
boundary: 'logs.cleanup',
|
|
164
|
+
operation: 'abort-multipart-upload',
|
|
165
|
+
extra: {objectKey: params.key},
|
|
166
|
+
});
|
|
167
|
+
});
|
|
149
168
|
throw new Error('Compaction upload aborted before it started');
|
|
150
169
|
}
|
|
151
170
|
}
|
package/src/config.test.ts
CHANGED
|
@@ -47,48 +47,24 @@ describe('LOG_STREAM_REAP_AFTER_SECONDS validation', () => {
|
|
|
47
47
|
vi.resetModules();
|
|
48
48
|
});
|
|
49
49
|
|
|
50
|
-
// Reaping a stream a still-valid lease is appending to silently truncates live logs, so a value
|
|
51
|
-
// at or below the job lease TTL (default 90m = 5400s) must fail startup.
|
|
52
50
|
it.each([
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
'
|
|
56
|
-
])('fails startup when LOG_STREAM_REAP_AFTER_SECONDS is %s (at or below the default lease TTL)', async (value) => {
|
|
57
|
-
vi.stubEnv('LOG_STREAM_REAP_AFTER_SECONDS', value);
|
|
51
|
+
0, -100, 5400,
|
|
52
|
+
])('rejects a reaper window of %d seconds for a 90-minute Auth lease', async (value) => {
|
|
53
|
+
vi.stubEnv('LOG_STREAM_REAP_AFTER_SECONDS', String(value));
|
|
58
54
|
vi.resetModules();
|
|
59
55
|
|
|
60
|
-
await
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
it('accepts a value above the lease TTL', async () => {
|
|
64
|
-
vi.stubEnv('LOG_STREAM_REAP_AFTER_SECONDS', '9000');
|
|
65
|
-
vi.resetModules();
|
|
66
|
-
|
|
67
|
-
const {config} = await import('#config.js');
|
|
56
|
+
const {validateLogStreamReapAfterSeconds} = await import('#config.js');
|
|
68
57
|
|
|
69
|
-
expect(
|
|
58
|
+
expect(() => validateLogStreamReapAfterSeconds(5400)).toThrow('LOG_STREAM_REAP_AFTER_SECONDS');
|
|
70
59
|
});
|
|
71
60
|
|
|
72
|
-
|
|
73
|
-
// the reaper window must fail startup even though the value clears the default lease.
|
|
74
|
-
it('fails when AUTH_JOB_LEASE_TOKEN_EXPIRES_IN is raised above the reaper window', async () => {
|
|
75
|
-
vi.stubEnv('AUTH_JOB_LEASE_TOKEN_EXPIRES_IN', '180m');
|
|
76
|
-
vi.stubEnv('LOG_STREAM_REAP_AFTER_SECONDS', '9000');
|
|
77
|
-
vi.resetModules();
|
|
78
|
-
|
|
79
|
-
await expect(import('#config.js')).rejects.toThrow('LOG_STREAM_REAP_AFTER_SECONDS');
|
|
80
|
-
});
|
|
81
|
-
|
|
82
|
-
// The converse: a small reaper window is fine when the lease is correspondingly small, proving
|
|
83
|
-
// the bound tracks the lease rather than a fixed 5400s floor.
|
|
84
|
-
it('accepts a small reaper window when the lease TTL is correspondingly small', async () => {
|
|
85
|
-
vi.stubEnv('AUTH_JOB_LEASE_TOKEN_EXPIRES_IN', '5m');
|
|
61
|
+
it('accepts a reaper window above the Auth lease lifetime', async () => {
|
|
86
62
|
vi.stubEnv('LOG_STREAM_REAP_AFTER_SECONDS', '600');
|
|
87
63
|
vi.resetModules();
|
|
88
64
|
|
|
89
|
-
const {
|
|
65
|
+
const {validateLogStreamReapAfterSeconds} = await import('#config.js');
|
|
90
66
|
|
|
91
|
-
expect(
|
|
67
|
+
expect(() => validateLogStreamReapAfterSeconds(300)).not.toThrow();
|
|
92
68
|
});
|
|
93
69
|
});
|
|
94
70
|
|
package/src/config.ts
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import {config as authConfig} from '@shipfox/api-auth/config';
|
|
2
1
|
import {bool, createConfig, num, str, url} from '@shipfox/config';
|
|
3
|
-
import {durationToSeconds} from '@shipfox/node-jwt';
|
|
4
2
|
|
|
5
3
|
export const config = createConfig({
|
|
6
4
|
LOG_STORAGE_S3_ENDPOINT: url({
|
|
@@ -48,7 +46,7 @@ export const config = createConfig({
|
|
|
48
46
|
default: 900,
|
|
49
47
|
}),
|
|
50
48
|
LOG_STREAM_REAP_AFTER_SECONDS: num({
|
|
51
|
-
desc: 'How long a log stream may stay open before the reaper cron force-closes it as abandoned. This is the backstop for a stream a runner started after its job already went terminal (the one-shot close sweep had already run); the reaper marks it truncated so it re-enters the compaction and retention lifecycle.
|
|
49
|
+
desc: 'How long a log stream may stay open before the reaper cron force-closes it as abandoned. This is the backstop for a stream a runner started after its job already went terminal (the one-shot close sweep had already run); the reaper marks it truncated so it re-enters the compaction and retention lifecycle. The application validates that this exceeds the Auth job lease lifetime. Defaults to 7200 seconds (2 hours).',
|
|
52
50
|
default: 7200,
|
|
53
51
|
}),
|
|
54
52
|
LOG_APPEND_BODY_LIMIT_BYTES: num({
|
|
@@ -115,16 +113,13 @@ if (!Number.isInteger(config.LOG_RETENTION_DAYS) || config.LOG_RETENTION_DAYS <
|
|
|
115
113
|
);
|
|
116
114
|
}
|
|
117
115
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
throw new Error(
|
|
128
|
-
`LOG_STREAM_REAP_AFTER_SECONDS (${config.LOG_STREAM_REAP_AFTER_SECONDS}) must be greater than the job lease TTL (${jobLeaseTtlSeconds}s, from AUTH_JOB_LEASE_TOKEN_EXPIRES_IN=${authConfig.AUTH_JOB_LEASE_TOKEN_EXPIRES_IN}); a smaller value would let the reaper force-close a stream a still-valid lease is appending to and silently truncate live logs.`,
|
|
129
|
-
);
|
|
116
|
+
export function validateLogStreamReapAfterSeconds(jobLeaseTokenTtlSeconds: number): void {
|
|
117
|
+
if (
|
|
118
|
+
!Number.isFinite(config.LOG_STREAM_REAP_AFTER_SECONDS) ||
|
|
119
|
+
config.LOG_STREAM_REAP_AFTER_SECONDS <= jobLeaseTokenTtlSeconds
|
|
120
|
+
) {
|
|
121
|
+
throw new Error(
|
|
122
|
+
`LOG_STREAM_REAP_AFTER_SECONDS (${config.LOG_STREAM_REAP_AFTER_SECONDS}) must be greater than the Auth job lease TTL (${jobLeaseTokenTtlSeconds}s); a smaller value would let the reaper force-close a stream a still-valid lease is appending to and silently truncate live logs.`,
|
|
123
|
+
);
|
|
124
|
+
}
|
|
130
125
|
}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import {type LogRecord, parseLogRecordLine} from '@shipfox/api-logs-dto';
|
|
2
|
-
import {
|
|
2
|
+
import {workflowsInterModuleContract} from '@shipfox/api-workflows-dto/inter-module';
|
|
3
|
+
import {defineInterModulePresentation} from '@shipfox/inter-module';
|
|
4
|
+
import {createFakeInterModuleClients} from '@shipfox/node-module/inter-module/testing';
|
|
5
|
+
import {appendLogs} from '#core/append-logs.js';
|
|
3
6
|
import {LeaseStreamMismatchError, MalformedLogChunkError, OffsetGapError} from '#core/errors.js';
|
|
4
7
|
import {jobAccountingFactory} from '#test/factories/job-accounting.js';
|
|
5
8
|
import {
|
|
@@ -13,8 +16,6 @@ import {
|
|
|
13
16
|
} from '#test/fixtures/ndjson.js';
|
|
14
17
|
import {findAccounting, findStream, listChunks, listStreamClosedEvents} from '#test/queries.js';
|
|
15
18
|
|
|
16
|
-
const mockSteps = new Map<string, {config: Record<string, unknown>}>();
|
|
17
|
-
|
|
18
19
|
interface Ctx {
|
|
19
20
|
jobId: string;
|
|
20
21
|
stepId: string;
|
|
@@ -48,20 +49,6 @@ function recordsFromChunks(chunks: Awaited<ReturnType<typeof listChunks>>): LogR
|
|
|
48
49
|
}
|
|
49
50
|
|
|
50
51
|
describe('appendLogs', () => {
|
|
51
|
-
let restoreStepLookup: () => void;
|
|
52
|
-
|
|
53
|
-
beforeAll(() => {
|
|
54
|
-
restoreStepLookup = setStepLookupForTesting((stepId) => Promise.resolve(mockSteps.get(stepId)));
|
|
55
|
-
});
|
|
56
|
-
|
|
57
|
-
afterAll(() => {
|
|
58
|
-
restoreStepLookup();
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
beforeEach(() => {
|
|
62
|
-
mockSteps.clear();
|
|
63
|
-
});
|
|
64
|
-
|
|
65
52
|
describe('offset-CAS', () => {
|
|
66
53
|
it('extends committed_length and stores one runner chunk on an in-order append', async () => {
|
|
67
54
|
const ctx = newCtx();
|
|
@@ -203,7 +190,14 @@ describe('appendLogs', () => {
|
|
|
203
190
|
it('selects the Claude parser from the step harness', async () => {
|
|
204
191
|
const ctx = newCtx();
|
|
205
192
|
await allowLargeLogBudget(ctx);
|
|
206
|
-
|
|
193
|
+
const workflows = createFakeInterModuleClients({
|
|
194
|
+
workflows: defineInterModulePresentation(workflowsInterModuleContract, {
|
|
195
|
+
startRunFromTrigger: vi.fn(),
|
|
196
|
+
deliverEventToJobListener: vi.fn(),
|
|
197
|
+
getStepLogContext: () => ({harness: 'claude' as const}),
|
|
198
|
+
getLeasedAgentToolContext: vi.fn(),
|
|
199
|
+
}),
|
|
200
|
+
}).workflows;
|
|
207
201
|
const body = ndjsonBody(
|
|
208
202
|
sessionLine(
|
|
209
203
|
JSON.stringify({
|
|
@@ -216,7 +210,7 @@ describe('appendLogs', () => {
|
|
|
216
210
|
),
|
|
217
211
|
);
|
|
218
212
|
|
|
219
|
-
await appendLogs({...ctx, attempt: 1, offset: 0, body});
|
|
213
|
+
await appendLogs({...ctx, attempt: 1, offset: 0, body}, workflows);
|
|
220
214
|
|
|
221
215
|
const stream = await findStream({...ctx, attempt: 1});
|
|
222
216
|
const rows = recordsFromChunks(await listChunks(stream?.id as string)).map((record) => {
|
package/src/core/append-logs.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import {Buffer} from 'node:buffer';
|
|
2
|
-
import {DEFAULT_HARNESS, type Harness
|
|
2
|
+
import {DEFAULT_HARNESS, type Harness} from '@shipfox/api-agent-dto';
|
|
3
3
|
import {
|
|
4
4
|
type LogRecord,
|
|
5
5
|
parseLogRecordLine,
|
|
6
6
|
parseRawLogRecordLine,
|
|
7
7
|
type RawLogRecord,
|
|
8
8
|
} from '@shipfox/api-logs-dto';
|
|
9
|
-
import {
|
|
9
|
+
import type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';
|
|
10
10
|
import {logger} from '@shipfox/node-opentelemetry';
|
|
11
11
|
import {config} from '#config.js';
|
|
12
12
|
import {accrueStoredBytes, claimCap, ensureJobAccounting, isJobCapped} from '#db/accounting.js';
|
|
@@ -30,18 +30,6 @@ import {MalformedLogChunkError, OffsetGapError} from './errors.js';
|
|
|
30
30
|
import {parseSessionRecord} from './session/parse-session.js';
|
|
31
31
|
import type {AgentSessionRecord} from './session/session-record.js';
|
|
32
32
|
|
|
33
|
-
type StepLookup = (stepId: string) => Promise<{config: Record<string, unknown>} | undefined>;
|
|
34
|
-
|
|
35
|
-
let stepLookup: StepLookup = getStepById;
|
|
36
|
-
|
|
37
|
-
export function setStepLookupForTesting(lookup: StepLookup): () => void {
|
|
38
|
-
const previous = stepLookup;
|
|
39
|
-
stepLookup = lookup;
|
|
40
|
-
return () => {
|
|
41
|
-
stepLookup = previous;
|
|
42
|
-
};
|
|
43
|
-
}
|
|
44
|
-
|
|
45
33
|
export interface AppendLogsParams {
|
|
46
34
|
jobId: string;
|
|
47
35
|
workspaceId: string;
|
|
@@ -137,10 +125,11 @@ function detectForgedType(line: string): string | undefined {
|
|
|
137
125
|
}
|
|
138
126
|
}
|
|
139
127
|
|
|
140
|
-
async function getSessionHarness(
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
128
|
+
async function getSessionHarness(
|
|
129
|
+
workflows: WorkflowsModuleClient | undefined,
|
|
130
|
+
stepId: string,
|
|
131
|
+
): Promise<Harness> {
|
|
132
|
+
return workflows ? (await workflows.getStepLogContext({stepId})).harness : DEFAULT_HARNESS;
|
|
144
133
|
}
|
|
145
134
|
|
|
146
135
|
/**
|
|
@@ -273,7 +262,10 @@ function agentSessionRecord(
|
|
|
273
262
|
* job contend on its single accounting row, so the path is multi-instance safe
|
|
274
263
|
* but not lock-free.
|
|
275
264
|
*/
|
|
276
|
-
export async function appendLogs(
|
|
265
|
+
export async function appendLogs(
|
|
266
|
+
params: AppendLogsParams,
|
|
267
|
+
workflows?: WorkflowsModuleClient,
|
|
268
|
+
): Promise<AppendLogsResult> {
|
|
277
269
|
let parsed: ParsedBody;
|
|
278
270
|
try {
|
|
279
271
|
parsed = parseAppendBody(params.body);
|
|
@@ -293,7 +285,7 @@ export async function appendLogs(params: AppendLogsParams): Promise<AppendLogsRe
|
|
|
293
285
|
}
|
|
294
286
|
const {declaredTotalBytes} = parsed;
|
|
295
287
|
const sessionHarness = parsed.hasAgentSessionRecord
|
|
296
|
-
? await getSessionHarness(params.stepId)
|
|
288
|
+
? await getSessionHarness(workflows, params.stepId)
|
|
297
289
|
: DEFAULT_HARNESS;
|
|
298
290
|
const stored = buildStoredBody(parsed.records, sessionHarness);
|
|
299
291
|
const commitByteLen = params.body.length;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import {reportError} from '@shipfox/node-error-monitoring';
|
|
1
2
|
import {logger} from '@shipfox/node-opentelemetry';
|
|
2
3
|
import {db} from '#db/db.js';
|
|
3
4
|
import {listStaleOpenStreams} from '#db/streams.js';
|
|
@@ -53,6 +54,7 @@ export async function reapStaleOpenStreams(
|
|
|
53
54
|
} catch (error) {
|
|
54
55
|
result.failed += 1;
|
|
55
56
|
logger().error({err: error, streamId: stream.id}, 'Failed to reap stale open log stream');
|
|
57
|
+
reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});
|
|
56
58
|
}
|
|
57
59
|
}
|
|
58
60
|
|
package/src/core/retention.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import {reportError} from '@shipfox/node-error-monitoring';
|
|
1
2
|
import {logger} from '@shipfox/node-opentelemetry';
|
|
2
3
|
import {deleteObjectsByPrefix} from '#api/object-storage.js';
|
|
3
4
|
import {config} from '#config.js';
|
|
@@ -116,6 +117,7 @@ export async function runRetentionSweep(
|
|
|
116
117
|
{err: error, streamId: stream.id},
|
|
117
118
|
'Failed to delete expired log stream objects',
|
|
118
119
|
);
|
|
120
|
+
reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});
|
|
119
121
|
continue;
|
|
120
122
|
}
|
|
121
123
|
|
|
@@ -129,6 +131,7 @@ export async function runRetentionSweep(
|
|
|
129
131
|
{err: error, streamId: stream.id},
|
|
130
132
|
'Failed to delete expired log stream row',
|
|
131
133
|
);
|
|
134
|
+
reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});
|
|
132
135
|
continue;
|
|
133
136
|
}
|
|
134
137
|
|
|
@@ -143,6 +146,7 @@ export async function runRetentionSweep(
|
|
|
143
146
|
{err: error, streamId: stream.id},
|
|
144
147
|
'Failed to reload raced expired log stream',
|
|
145
148
|
);
|
|
149
|
+
reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});
|
|
146
150
|
continue;
|
|
147
151
|
}
|
|
148
152
|
if (current) {
|
|
@@ -159,6 +163,7 @@ export async function runRetentionSweep(
|
|
|
159
163
|
{err: error, streamId: stream.id},
|
|
160
164
|
'Failed to delete raced expired log stream',
|
|
161
165
|
);
|
|
166
|
+
reportError(error, {boundary: 'logs.maintenance', extra: {streamId: stream.id}});
|
|
162
167
|
continue;
|
|
163
168
|
}
|
|
164
169
|
}
|
package/src/index.ts
CHANGED
|
@@ -6,12 +6,14 @@ import {
|
|
|
6
6
|
WORKFLOWS_STEP_ATTEMPT_TERMINATED,
|
|
7
7
|
type WorkflowsEventMapDto,
|
|
8
8
|
} from '@shipfox/api-workflows-dto';
|
|
9
|
+
import type {WorkflowsModuleClient} from '@shipfox/api-workflows-dto/inter-module';
|
|
9
10
|
import {type ShipfoxModule, subscriberFactory} from '@shipfox/node-module';
|
|
11
|
+
import {validateLogStreamReapAfterSeconds} from '#config.js';
|
|
10
12
|
import {db} from '#db/db.js';
|
|
11
13
|
import {migrationsPath} from '#db/index.js';
|
|
12
14
|
import {logsOutbox} from '#db/schema/outbox.js';
|
|
13
15
|
import {registerLogsServiceMetrics} from '#metrics/service.js';
|
|
14
|
-
import {
|
|
16
|
+
import {createLogsRoutes} from '#presentation/routes/index.js';
|
|
15
17
|
import {onJobTerminated} from '#presentation/subscribers/on-job-terminated.js';
|
|
16
18
|
import {onLogStreamClosed} from '#presentation/subscribers/on-log-stream-closed.js';
|
|
17
19
|
import {onStepAttemptTerminated} from '#presentation/subscribers/on-step-attempt-terminated.js';
|
|
@@ -25,53 +27,63 @@ const workflowsPath = resolve(packageRoot, 'dist/temporal/workflows/index.js');
|
|
|
25
27
|
|
|
26
28
|
const subscriber = subscriberFactory<WorkflowsEventMapDto & LogsEventMap>();
|
|
27
29
|
|
|
28
|
-
export
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
//
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
30
|
+
export function createLogsModule({
|
|
31
|
+
workflows,
|
|
32
|
+
jobLeaseTokenTtlSeconds,
|
|
33
|
+
}: {
|
|
34
|
+
workflows: WorkflowsModuleClient;
|
|
35
|
+
jobLeaseTokenTtlSeconds: number;
|
|
36
|
+
}): ShipfoxModule {
|
|
37
|
+
validateLogStreamReapAfterSeconds(jobLeaseTokenTtlSeconds);
|
|
38
|
+
|
|
39
|
+
return {
|
|
40
|
+
name: 'logs',
|
|
41
|
+
database: {db, migrationsPath},
|
|
42
|
+
routes: createLogsRoutes(workflows),
|
|
43
|
+
metrics: registerLogsServiceMetrics,
|
|
44
|
+
// `logs.stream.closed` is written by the close paths: the job-terminated subscriber
|
|
45
|
+
// force-closes streams the runner never ended, and the closed event drives compaction.
|
|
46
|
+
publishers: [{name: 'logs', table: logsOutbox, db, eventSchemas: logsEventSchemas}],
|
|
47
|
+
subscribers: [
|
|
48
|
+
subscriber(WORKFLOWS_STEP_ATTEMPT_TERMINATED, onStepAttemptTerminated),
|
|
49
|
+
subscriber(WORKFLOWS_JOB_TERMINATED, onJobTerminated),
|
|
50
|
+
subscriber(LOG_STREAM_CLOSED, onLogStreamClosed),
|
|
51
|
+
],
|
|
52
|
+
workers: [
|
|
53
|
+
{
|
|
54
|
+
taskQueue: LOGS_LIFECYCLE_TASK_QUEUE,
|
|
55
|
+
workflowsPath,
|
|
56
|
+
activities: createLogsActivities,
|
|
57
|
+
// Temporal cron schedules are fixed after creation, so making cadence configurable would
|
|
58
|
+
// look adjustable while keeping the old schedule. Tune LOG_RETENTION_DAYS for the horizon.
|
|
59
|
+
workflows: [
|
|
60
|
+
{
|
|
61
|
+
name: 'retentionSweepCron',
|
|
62
|
+
id: 'logs-retention-sweep',
|
|
63
|
+
cronSchedule: '0 * * * *',
|
|
64
|
+
},
|
|
65
|
+
// Offset from retention's top-of-hour sweep; stale-stream age is governed by
|
|
66
|
+
// LOG_STREAM_REAP_AFTER_SECONDS.
|
|
67
|
+
{
|
|
68
|
+
name: 'reapStaleOpenStreamsCron',
|
|
69
|
+
id: 'logs-reap-stale-open-streams',
|
|
70
|
+
cronSchedule: '5,15,25,35,45,55 * * * *',
|
|
71
|
+
},
|
|
72
|
+
],
|
|
73
|
+
},
|
|
74
|
+
// Compaction runs on its own queue so long uploads cannot starve the lifecycle sweep.
|
|
75
|
+
{
|
|
76
|
+
taskQueue: LOGS_COMPACTION_TASK_QUEUE,
|
|
77
|
+
workflowsPath,
|
|
78
|
+
activities: createLogsActivities,
|
|
79
|
+
workflows: [
|
|
80
|
+
{
|
|
81
|
+
name: 'compactionReconcileCron',
|
|
82
|
+
id: 'logs-compaction-reconcile',
|
|
83
|
+
cronSchedule: '*/10 * * * *',
|
|
84
|
+
},
|
|
85
|
+
],
|
|
86
|
+
},
|
|
87
|
+
],
|
|
88
|
+
};
|
|
89
|
+
}
|
|
@@ -4,7 +4,8 @@ import {AUTH_USER} from '@shipfox/api-auth-context';
|
|
|
4
4
|
import {type AuthMethod, closeApp, createApp, type FastifyInstance} from '@shipfox/node-fastify';
|
|
5
5
|
import {mintLeaseToken} from '#test/fixtures/lease-token.js';
|
|
6
6
|
import {endLine, ndjsonBody, outputLine, recordLine} from '#test/fixtures/ndjson.js';
|
|
7
|
-
import {
|
|
7
|
+
import {createTestWorkflowsClient} from '#test/fixtures/workflows-client.js';
|
|
8
|
+
import {createLogsRoutes} from './index.js';
|
|
8
9
|
|
|
9
10
|
const NDJSON = 'application/x-ndjson';
|
|
10
11
|
const JOB_EXECUTION_ID = '00000000-0000-4000-8000-0000000000ee';
|
|
@@ -36,7 +37,7 @@ describe('POST /runs/jobs/current/steps/:stepId/logs', () => {
|
|
|
36
37
|
beforeAll(async () => {
|
|
37
38
|
app = await createApp({
|
|
38
39
|
auth: [createLeaseTokenAuthMethod(), stubUserAuth],
|
|
39
|
-
routes:
|
|
40
|
+
routes: createLogsRoutes(createTestWorkflowsClient()),
|
|
40
41
|
swagger: false,
|
|
41
42
|
});
|
|
42
43
|
await app.ready();
|