@workflow/world-local 5.0.0-beta.10 → 5.0.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.ts +17 -5
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +18 -13
- package/dist/config.js.map +1 -1
- package/dist/fs.d.ts +102 -2
- package/dist/fs.d.ts.map +1 -1
- package/dist/fs.js +291 -24
- package/dist/fs.js.map +1 -1
- package/dist/index.d.ts +15 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +82 -5
- package/dist/index.js.map +1 -1
- package/dist/init.d.ts +91 -0
- package/dist/init.d.ts.map +1 -0
- package/dist/init.js +268 -0
- package/dist/init.js.map +1 -0
- package/dist/instrumentObject.d.ts +8 -0
- package/dist/instrumentObject.d.ts.map +1 -0
- package/dist/instrumentObject.js +66 -0
- package/dist/instrumentObject.js.map +1 -0
- package/dist/queue.d.ts +8 -1
- package/dist/queue.d.ts.map +1 -1
- package/dist/queue.js +164 -53
- package/dist/queue.js.map +1 -1
- package/dist/storage/events-storage.d.ts +7 -0
- package/dist/storage/events-storage.d.ts.map +1 -0
- package/dist/storage/events-storage.js +760 -0
- package/dist/storage/events-storage.js.map +1 -0
- package/dist/storage/filters.d.ts +22 -0
- package/dist/storage/filters.d.ts.map +1 -0
- package/dist/storage/filters.js +33 -0
- package/dist/storage/filters.js.map +1 -0
- package/dist/storage/helpers.d.ts +18 -0
- package/dist/storage/helpers.d.ts.map +1 -0
- package/dist/storage/helpers.js +44 -0
- package/dist/storage/helpers.js.map +1 -0
- package/dist/storage/hooks-storage.d.ts +12 -0
- package/dist/storage/hooks-storage.d.ts.map +1 -0
- package/dist/storage/hooks-storage.js +93 -0
- package/dist/storage/hooks-storage.js.map +1 -0
- package/dist/storage/index.d.ts +12 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +32 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/legacy.d.ts +13 -0
- package/dist/storage/legacy.d.ts.map +1 -0
- package/dist/storage/legacy.js +77 -0
- package/dist/storage/legacy.js.map +1 -0
- package/dist/storage/runs-storage.d.ts +7 -0
- package/dist/storage/runs-storage.d.ts.map +1 -0
- package/dist/storage/runs-storage.js +59 -0
- package/dist/storage/runs-storage.js.map +1 -0
- package/dist/storage/steps-storage.d.ts +7 -0
- package/dist/storage/steps-storage.d.ts.map +1 -0
- package/dist/storage/steps-storage.js +52 -0
- package/dist/storage/steps-storage.js.map +1 -0
- package/dist/storage.d.ts +9 -2
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +8 -455
- package/dist/storage.js.map +1 -1
- package/dist/streamer.d.ts +4 -2
- package/dist/streamer.d.ts.map +1 -1
- package/dist/streamer.js +456 -104
- package/dist/streamer.js.map +1 -1
- package/dist/telemetry.d.ts +28 -0
- package/dist/telemetry.d.ts.map +1 -0
- package/dist/telemetry.js +71 -0
- package/dist/telemetry.js.map +1 -0
- package/dist/test-helpers.d.ts +54 -0
- package/dist/test-helpers.d.ts.map +1 -0
- package/dist/test-helpers.js +118 -0
- package/dist/test-helpers.js.map +1 -0
- package/dist/util.js.map +1 -1
- package/package.json +12 -11
package/dist/config.d.ts
CHANGED
|
@@ -3,17 +3,29 @@ export type Config = {
|
|
|
3
3
|
dataDir: string;
|
|
4
4
|
port?: number;
|
|
5
5
|
baseUrl?: string;
|
|
6
|
+
/**
|
|
7
|
+
* Optional tag to scope filesystem operations.
|
|
8
|
+
* When set, files are written as `{id}.{tag}.json` and `clear()` only deletes
|
|
9
|
+
* files matching this tag. Used by vitest to isolate test data in the shared
|
|
10
|
+
* `.workflow-data` directory.
|
|
11
|
+
*/
|
|
12
|
+
tag?: string;
|
|
13
|
+
/**
|
|
14
|
+
* Override the flush interval (in ms) for buffered stream writes.
|
|
15
|
+
* Default is 10ms. Set to 0 for immediate flushing.
|
|
16
|
+
*/
|
|
17
|
+
streamFlushIntervalMs?: number;
|
|
6
18
|
};
|
|
7
19
|
export declare const config: {
|
|
8
20
|
readonly value: Config;
|
|
9
21
|
};
|
|
10
22
|
/**
|
|
11
23
|
* Resolves the base URL for queue requests following the priority order:
|
|
12
|
-
* 1. config.baseUrl (highest priority - full override from args
|
|
13
|
-
* 2.
|
|
14
|
-
* 3.
|
|
15
|
-
* 4. PORT env var (
|
|
16
|
-
* 5.
|
|
24
|
+
* 1. config.baseUrl (highest priority - full override from args)
|
|
25
|
+
* 2. WORKFLOW_LOCAL_BASE_URL env var (checked directly to handle late env var setting)
|
|
26
|
+
* 3. config.port (explicit port override from args)
|
|
27
|
+
* 4. PORT env var (explicit configuration)
|
|
28
|
+
* 5. Auto-detected port via getPort (detect actual listening port)
|
|
17
29
|
*/
|
|
18
30
|
export declare function resolveBaseUrl(config: Partial<Config>): Promise<string>;
|
|
19
31
|
//# sourceMappingURL=config.d.ts.map
|
package/dist/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAOA,eAAO,MAAM,2BAA2B,QAAQ,CAAC;AAMjD,MAAM,MAAM,MAAM,GAAG;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAOA,eAAO,MAAM,2BAA2B,QAAQ,CAAC;AAMjD,MAAM,MAAM,MAAM,GAAG;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;OAKG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,eAAO,MAAM,MAAM;;CAKjB,CAAC;AAEH;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAyB7E"}
|
package/dist/config.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { getWorkflowPort } from '@workflow/utils/get-port';
|
|
2
2
|
import { once } from './util.js';
|
|
3
3
|
const getDataDirFromEnv = () => {
|
|
4
|
-
return process.env.
|
|
4
|
+
return process.env.WORKFLOW_LOCAL_DATA_DIR || '.workflow-data';
|
|
5
5
|
};
|
|
6
6
|
export const DEFAULT_RESOLVE_DATA_OPTION = 'all';
|
|
7
7
|
const getBaseUrlFromEnv = () => {
|
|
8
|
-
return process.env.
|
|
8
|
+
return process.env.WORKFLOW_LOCAL_BASE_URL;
|
|
9
9
|
};
|
|
10
10
|
export const config = once(() => {
|
|
11
11
|
const dataDir = getDataDirFromEnv();
|
|
@@ -14,26 +14,31 @@ export const config = once(() => {
|
|
|
14
14
|
});
|
|
15
15
|
/**
|
|
16
16
|
* Resolves the base URL for queue requests following the priority order:
|
|
17
|
-
* 1. config.baseUrl (highest priority - full override from args
|
|
18
|
-
* 2.
|
|
19
|
-
* 3.
|
|
20
|
-
* 4. PORT env var (
|
|
21
|
-
* 5.
|
|
17
|
+
* 1. config.baseUrl (highest priority - full override from args)
|
|
18
|
+
* 2. WORKFLOW_LOCAL_BASE_URL env var (checked directly to handle late env var setting)
|
|
19
|
+
* 3. config.port (explicit port override from args)
|
|
20
|
+
* 4. PORT env var (explicit configuration)
|
|
21
|
+
* 5. Auto-detected port via getPort (detect actual listening port)
|
|
22
22
|
*/
|
|
23
23
|
export async function resolveBaseUrl(config) {
|
|
24
24
|
if (config.baseUrl) {
|
|
25
25
|
return config.baseUrl;
|
|
26
26
|
}
|
|
27
|
+
// Check env var directly in case it was set after the config was cached
|
|
28
|
+
// This is important for CLI tools that set the env var after module import
|
|
29
|
+
if (process.env.WORKFLOW_LOCAL_BASE_URL) {
|
|
30
|
+
return process.env.WORKFLOW_LOCAL_BASE_URL;
|
|
31
|
+
}
|
|
27
32
|
if (typeof config.port === 'number') {
|
|
28
33
|
return `http://localhost:${config.port}`;
|
|
29
34
|
}
|
|
30
|
-
const detectedPort = await getPort();
|
|
31
|
-
if (detectedPort) {
|
|
32
|
-
return `http://localhost:${detectedPort}`;
|
|
33
|
-
}
|
|
34
35
|
if (process.env.PORT) {
|
|
35
36
|
return `http://localhost:${process.env.PORT}`;
|
|
36
37
|
}
|
|
37
|
-
|
|
38
|
+
const detectedPort = await getWorkflowPort();
|
|
39
|
+
if (detectedPort) {
|
|
40
|
+
return `http://localhost:${detectedPort}`;
|
|
41
|
+
}
|
|
42
|
+
throw new Error('Unable to resolve base URL for workflow queue.');
|
|
38
43
|
}
|
|
39
44
|
//# sourceMappingURL=config.js.map
|
package/dist/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,iBAAiB,GAAG,GAAG,EAAE;IAC7B,OAAO,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,gBAAgB,CAAC;AACjE,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,2BAA2B,GAAG,KAAK,CAAC;AAEjD,MAAM,iBAAiB,GAAG,GAAG,EAAE;IAC7B,OAAO,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;AAC7C,CAAC,CAAC;AAoBF,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,CAAS,GAAG,EAAE;IACtC,MAAM,OAAO,GAAG,iBAAiB,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,iBAAiB,EAAE,CAAC;IAEpC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC,CAAC,CAAC;AAEH;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAuB;IAC1D,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,wEAAwE;IACxE,2EAA2E;IAC3E,IAAI,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC;QACxC,OAAO,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IAC7C,CAAC;IAED,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACpC,OAAO,oBAAoB,MAAM,CAAC,IAAI,EAAE,CAAC;IAC3C,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACrB,OAAO,oBAAoB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IAChD,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,eAAe,EAAE,CAAC;IAC7C,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,oBAAoB,YAAY,EAAE,CAAC;IAC5C,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;AACpE,CAAC","sourcesContent":["import { getWorkflowPort } from '@workflow/utils/get-port';\nimport { once } from './util.js';\n\nconst getDataDirFromEnv = () => {\n return process.env.WORKFLOW_LOCAL_DATA_DIR || '.workflow-data';\n};\n\nexport const DEFAULT_RESOLVE_DATA_OPTION = 'all';\n\nconst getBaseUrlFromEnv = () => {\n return process.env.WORKFLOW_LOCAL_BASE_URL;\n};\n\nexport type Config = {\n dataDir: string;\n port?: number;\n baseUrl?: string;\n /**\n * Optional tag to scope filesystem operations.\n * When set, files are written as `{id}.{tag}.json` and `clear()` only deletes\n * files matching this tag. Used by vitest to isolate test data in the shared\n * `.workflow-data` directory.\n */\n tag?: string;\n /**\n * Override the flush interval (in ms) for buffered stream writes.\n * Default is 10ms. Set to 0 for immediate flushing.\n */\n streamFlushIntervalMs?: number;\n};\n\nexport const config = once<Config>(() => {\n const dataDir = getDataDirFromEnv();\n const baseUrl = getBaseUrlFromEnv();\n\n return { dataDir, baseUrl };\n});\n\n/**\n * Resolves the base URL for queue requests following the priority order:\n * 1. config.baseUrl (highest priority - full override from args)\n * 2. WORKFLOW_LOCAL_BASE_URL env var (checked directly to handle late env var setting)\n * 3. config.port (explicit port override from args)\n * 4. PORT env var (explicit configuration)\n * 5. Auto-detected port via getPort (detect actual listening port)\n */\nexport async function resolveBaseUrl(config: Partial<Config>): Promise<string> {\n if (config.baseUrl) {\n return config.baseUrl;\n }\n\n // Check env var directly in case it was set after the config was cached\n // This is important for CLI tools that set the env var after module import\n if (process.env.WORKFLOW_LOCAL_BASE_URL) {\n return process.env.WORKFLOW_LOCAL_BASE_URL;\n }\n\n if (typeof config.port === 'number') {\n return `http://localhost:${config.port}`;\n }\n\n if (process.env.PORT) {\n return `http://localhost:${process.env.PORT}`;\n }\n\n const detectedPort = await getWorkflowPort();\n if (detectedPort) {\n return `http://localhost:${detectedPort}`;\n }\n\n throw new Error('Unable to resolve base URL for workflow queue.');\n}\n"]}
|
package/dist/fs.d.ts
CHANGED
|
@@ -1,16 +1,117 @@
|
|
|
1
|
+
import { WorkflowWorldError } from '@workflow/errors';
|
|
1
2
|
import type { PaginatedResponse } from '@workflow/world';
|
|
2
3
|
import { z } from 'zod';
|
|
3
|
-
|
|
4
|
+
/**
|
|
5
|
+
* Thrown when a caller-supplied entity ID contains characters that would
|
|
6
|
+
* escape the storage root (path traversal) or otherwise produce an unsafe
|
|
7
|
+
* filename. Extends {@link WorkflowWorldError} so the platform's standard
|
|
8
|
+
* error-to-HTTP mapping handles it alongside other world-layer errors.
|
|
9
|
+
*/
|
|
10
|
+
export declare class UnsafeEntityIdError extends WorkflowWorldError {
|
|
11
|
+
constructor(kind: string, value: string);
|
|
12
|
+
static is(value: unknown): value is UnsafeEntityIdError;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Validate that a string is safe to embed in a filesystem path as a single
|
|
16
|
+
* path component. Rejects values that are:
|
|
17
|
+
* - empty
|
|
18
|
+
* - starting with `.` (blocks `.`, `..`, `.locks`, `.tmp`, and other
|
|
19
|
+
* hidden or reserved filenames)
|
|
20
|
+
* - containing `/`, `\`, or a NUL byte
|
|
21
|
+
*
|
|
22
|
+
* This is the primary defense against path-traversal attacks where a
|
|
23
|
+
* request body supplies a `runId` / `stepId` / `correlationId` containing
|
|
24
|
+
* `../` sequences to read or write files outside the storage root.
|
|
25
|
+
* {@link resolveWithinBase} provides a belt-and-suspenders containment
|
|
26
|
+
* check at the point of `path.join` for defense in depth.
|
|
27
|
+
*
|
|
28
|
+
* @param kind - Human-readable label used in the error message (e.g. "runId")
|
|
29
|
+
* @param value - The value to validate; throws {@link UnsafeEntityIdError}
|
|
30
|
+
* if the value is not safe.
|
|
31
|
+
*/
|
|
32
|
+
export declare function assertSafeEntityId(kind: string, value: string): void;
|
|
33
|
+
/**
|
|
34
|
+
* Join `basedir` with an entity-relative path and assert the result stays
|
|
35
|
+
* inside `basedir`. Complements {@link assertSafeEntityId}: per-entry-point
|
|
36
|
+
* validation is the primary defense, and this final check catches any path
|
|
37
|
+
* escape that slipped past (e.g. a new call site that forgot to validate,
|
|
38
|
+
* or an unusual character combination on a future filesystem). Throws
|
|
39
|
+
* {@link UnsafeEntityIdError} if the joined path escapes `basedir`.
|
|
40
|
+
*/
|
|
41
|
+
export declare function resolveWithinBase(basedir: string, ...segments: string[]): string;
|
|
42
|
+
/**
|
|
43
|
+
* Clear the created files cache. Useful for testing or when files are deleted externally.
|
|
44
|
+
*/
|
|
45
|
+
export declare function clearCreatedFilesCache(): void;
|
|
46
|
+
export { ulidToDate } from '@workflow/world';
|
|
47
|
+
/**
|
|
48
|
+
* Strip a tag suffix from a fileId.
|
|
49
|
+
* `wrun_ABC.vitest-0` → `wrun_ABC`
|
|
50
|
+
* `wrun_ABC` → `wrun_ABC` (no-op if no tag)
|
|
51
|
+
*/
|
|
52
|
+
export declare function stripTag(fileId: string): string;
|
|
53
|
+
/**
|
|
54
|
+
* Build the file path for an entity, with optional tag embedded in the filename.
|
|
55
|
+
* `taggedPath('/data', 'runs', 'wrun_ABC', 'vitest-0')` → `/data/runs/wrun_ABC.vitest-0.json`
|
|
56
|
+
* `taggedPath('/data', 'runs', 'wrun_ABC')` → `/data/runs/wrun_ABC.json`
|
|
57
|
+
*
|
|
58
|
+
* The `fileId` and `tag` are validated with {@link assertSafeEntityId} and
|
|
59
|
+
* the result is containment-checked with {@link resolveWithinBase} to
|
|
60
|
+
* prevent path-traversal attacks when values are derived from untrusted
|
|
61
|
+
* request input.
|
|
62
|
+
*/
|
|
63
|
+
export declare function taggedPath(basedir: string, entityDir: string, fileId: string, tag?: string): string;
|
|
64
|
+
/**
|
|
65
|
+
* Read a JSON entity with tagged fallback.
|
|
66
|
+
* When a tag is set, tries the tagged path first, then falls back to the
|
|
67
|
+
* untagged path (so a tagged world can read entities written without a tag).
|
|
68
|
+
*/
|
|
69
|
+
export declare function readJSONWithFallback<T>(basedir: string, entityDir: string, fileId: string, schema: z.ZodType<T>, tag?: string): Promise<T | null>;
|
|
70
|
+
/**
|
|
71
|
+
* List all filenames in a directory that have a specific tag.
|
|
72
|
+
* Returns full filenames (e.g., `wrun_ABC.vitest-0.json`).
|
|
73
|
+
*/
|
|
74
|
+
export declare function listTaggedFiles(dirPath: string, tag: string): Promise<string[]>;
|
|
75
|
+
/**
|
|
76
|
+
* List all filenames in a directory that have a specific tag and extension.
|
|
77
|
+
* Returns full filenames (e.g., `stream-chnk_ABC.vitest-0.bin`).
|
|
78
|
+
*/
|
|
79
|
+
export declare function listTaggedFilesByExtension(dirPath: string, tag: string, extension: string): Promise<string[]>;
|
|
4
80
|
export declare function ensureDir(dirPath: string): Promise<void>;
|
|
5
81
|
interface WriteOptions {
|
|
6
82
|
overwrite?: boolean;
|
|
7
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* Custom JSON replacer that encodes Uint8Array as base64 strings.
|
|
86
|
+
* Format: { __type: 'Uint8Array', data: '<base64>' }
|
|
87
|
+
*/
|
|
88
|
+
export declare function jsonReplacer(_key: string, value: unknown): unknown;
|
|
89
|
+
/**
|
|
90
|
+
* Custom JSON reviver that decodes base64 strings back to Uint8Array.
|
|
91
|
+
*/
|
|
92
|
+
export declare function jsonReviver(_key: string, value: unknown): unknown;
|
|
8
93
|
export declare function writeJSON(filePath: string, data: any, opts?: WriteOptions): Promise<void>;
|
|
94
|
+
/**
|
|
95
|
+
* Writes data to a file using atomic write-rename pattern.
|
|
96
|
+
*
|
|
97
|
+
* Note: While this function uses temp files to avoid partial writes,
|
|
98
|
+
* it does not provide protection against concurrent writes from multiple
|
|
99
|
+
* processes. In a multi-writer scenario, the last writer wins.
|
|
100
|
+
* For production use with multiple writers, consider using a proper
|
|
101
|
+
* database or locking mechanism.
|
|
102
|
+
*/
|
|
9
103
|
export declare function write(filePath: string, data: string | Buffer, opts?: WriteOptions): Promise<void>;
|
|
10
104
|
export declare function readJSON<T>(filePath: string, decoder: z.ZodType<T>): Promise<T | null>;
|
|
11
105
|
export declare function readBuffer(filePath: string): Promise<Buffer>;
|
|
12
106
|
export declare function deleteJSON(filePath: string): Promise<void>;
|
|
107
|
+
/**
|
|
108
|
+
* Atomically create a file using O_CREAT | O_EXCL flags.
|
|
109
|
+
* Returns true if the file was created, false if it already exists.
|
|
110
|
+
* This is atomic at the OS level, safe for concurrent access.
|
|
111
|
+
*/
|
|
112
|
+
export declare function writeExclusive(filePath: string, data: string): Promise<boolean>;
|
|
13
113
|
export declare function listJSONFiles(dirPath: string): Promise<string[]>;
|
|
114
|
+
export declare function listFilesByExtension(dirPath: string, extension: string): Promise<string[]>;
|
|
14
115
|
interface PaginatedFileSystemQueryConfig<T> {
|
|
15
116
|
directory: string;
|
|
16
117
|
schema: z.ZodType<T>;
|
|
@@ -25,5 +126,4 @@ interface PaginatedFileSystemQueryConfig<T> {
|
|
|
25
126
|
export declare function paginatedFileSystemQuery<T extends {
|
|
26
127
|
createdAt: Date;
|
|
27
128
|
}>(config: PaginatedFileSystemQueryConfig<T>): Promise<PaginatedResponse<T>>;
|
|
28
|
-
export {};
|
|
29
129
|
//# sourceMappingURL=fs.d.ts.map
|
package/dist/fs.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAEA,OAAO,EAAuB,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEzD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAgBxB;;;;;GAKG;AACH,qBAAa,mBAAoB,SAAQ,kBAAkB;gBAC7C,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAOvC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,mBAAmB;CAGxD;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAUpE;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,MAAM,EACf,GAAG,QAAQ,EAAE,MAAM,EAAE,GACpB,MAAM,CAOR;AAuCD;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,IAAI,CAE7C;AAED,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAW7C;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAE/C;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CACxB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,GACX,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EACpB,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAcnB;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB;AAED;;;GAGG;AACH,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB;AAED,wBAAsB,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAM9D;AAED,UAAU,YAAY;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAQlE;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAUjE;AAED,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,GAAG,EACT,IAAI,CAAC,EAAE,YAAY,GAClB,OAAO,CAAC,IAAI,CAAC,CAEf;AAED;;;;;;;;GAQG;AACH,wBAAsB,KAAK,CACzB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,GAAG,MAAM,EACrB,IAAI,CAAC,EAAE,YAAY,GAClB,OAAO,CAAC,IAAI,CAAC,CA0Cf;AAED,wBAAsB,QAAQ,CAAC,CAAC,EAC9B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GACpB,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAQnB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAGlE;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMhE;AAED;;;;GAIG;AACH,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,OAAO,CAAC,CAWlB;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAEtE;AAED,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,EAAE,CAAC,CAUnB;AAED,UAAU,8BAA8B,CAAC,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC;IAC9B,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAAC;IAC5C,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC;CACzB;AAqBD,wBAAsB,wBAAwB,CAAC,CAAC,SAAS;IAAE,SAAS,EAAE,IAAI,CAAA;CAAE,EAC1E,MAAM,EAAE,8BAA8B,CAAC,CAAC,CAAC,GACxC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAwJ/B"}
|
package/dist/fs.js
CHANGED
|
@@ -1,16 +1,196 @@
|
|
|
1
1
|
import { promises as fs } from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { EntityConflictError, WorkflowWorldError } from '@workflow/errors';
|
|
4
|
+
import { monotonicFactory } from 'ulid';
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
const ulid = monotonicFactory(() => Math.random());
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Truncate a possibly-untrusted value for inclusion in an error message.
|
|
9
|
+
* Keeps error output actionable for developers in dev mode while limiting
|
|
10
|
+
* the amount of attacker-controlled data reflected back if a `world-local`
|
|
11
|
+
* backend is ever run in a context that surfaces these messages to clients.
|
|
12
|
+
*/
|
|
13
|
+
function truncateForError(value) {
|
|
14
|
+
const s = typeof value === 'string' ? value : String(value);
|
|
15
|
+
const MAX = 48;
|
|
16
|
+
return s.length > MAX ? `${s.slice(0, MAX)}…` : s;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Thrown when a caller-supplied entity ID contains characters that would
|
|
20
|
+
* escape the storage root (path traversal) or otherwise produce an unsafe
|
|
21
|
+
* filename. Extends {@link WorkflowWorldError} so the platform's standard
|
|
22
|
+
* error-to-HTTP mapping handles it alongside other world-layer errors.
|
|
23
|
+
*/
|
|
24
|
+
export class UnsafeEntityIdError extends WorkflowWorldError {
|
|
25
|
+
constructor(kind, value) {
|
|
26
|
+
super(`Unsafe ${kind} "${truncateForError(value)}": must not be empty, start with ".", or contain path separators or null bytes`);
|
|
27
|
+
this.name = 'UnsafeEntityIdError';
|
|
28
|
+
}
|
|
29
|
+
static is(value) {
|
|
30
|
+
return value instanceof Error && value.name === 'UnsafeEntityIdError';
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Validate that a string is safe to embed in a filesystem path as a single
|
|
35
|
+
* path component. Rejects values that are:
|
|
36
|
+
* - empty
|
|
37
|
+
* - starting with `.` (blocks `.`, `..`, `.locks`, `.tmp`, and other
|
|
38
|
+
* hidden or reserved filenames)
|
|
39
|
+
* - containing `/`, `\`, or a NUL byte
|
|
40
|
+
*
|
|
41
|
+
* This is the primary defense against path-traversal attacks where a
|
|
42
|
+
* request body supplies a `runId` / `stepId` / `correlationId` containing
|
|
43
|
+
* `../` sequences to read or write files outside the storage root.
|
|
44
|
+
* {@link resolveWithinBase} provides a belt-and-suspenders containment
|
|
45
|
+
* check at the point of `path.join` for defense in depth.
|
|
46
|
+
*
|
|
47
|
+
* @param kind - Human-readable label used in the error message (e.g. "runId")
|
|
48
|
+
* @param value - The value to validate; throws {@link UnsafeEntityIdError}
|
|
49
|
+
* if the value is not safe.
|
|
50
|
+
*/
|
|
51
|
+
export function assertSafeEntityId(kind, value) {
|
|
52
|
+
if (value.length === 0 ||
|
|
53
|
+
value.startsWith('.') ||
|
|
54
|
+
value.includes('/') ||
|
|
55
|
+
value.includes('\\') ||
|
|
56
|
+
value.includes('\0')) {
|
|
57
|
+
throw new UnsafeEntityIdError(kind, value);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Join `basedir` with an entity-relative path and assert the result stays
|
|
62
|
+
* inside `basedir`. Complements {@link assertSafeEntityId}: per-entry-point
|
|
63
|
+
* validation is the primary defense, and this final check catches any path
|
|
64
|
+
* escape that slipped past (e.g. a new call site that forgot to validate,
|
|
65
|
+
* or an unusual character combination on a future filesystem). Throws
|
|
66
|
+
* {@link UnsafeEntityIdError} if the joined path escapes `basedir`.
|
|
67
|
+
*/
|
|
68
|
+
export function resolveWithinBase(basedir, ...segments) {
|
|
69
|
+
const resolvedBase = path.resolve(basedir);
|
|
70
|
+
const joined = path.resolve(basedir, ...segments);
|
|
71
|
+
if (joined !== resolvedBase && !joined.startsWith(resolvedBase + path.sep)) {
|
|
72
|
+
throw new UnsafeEntityIdError('path', segments.join('/'));
|
|
73
|
+
}
|
|
74
|
+
return joined;
|
|
75
|
+
}
|
|
76
|
+
const isWindows = process.platform === 'win32';
|
|
77
|
+
/**
|
|
78
|
+
* Execute a filesystem operation with retry logic on Windows.
|
|
79
|
+
* On Windows, file operations can fail with EPERM/EBUSY/EACCES when files
|
|
80
|
+
* are briefly locked by another process or antivirus. This wrapper adds
|
|
81
|
+
* exponential backoff retry logic. On non-Windows platforms, executes directly.
|
|
82
|
+
*/
|
|
83
|
+
async function withWindowsRetry(fn, maxRetries = 5) {
|
|
84
|
+
if (!isWindows)
|
|
85
|
+
return fn();
|
|
86
|
+
const retryableErrors = ['EPERM', 'EBUSY', 'EACCES'];
|
|
87
|
+
const baseDelayMs = 10;
|
|
88
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
89
|
+
try {
|
|
90
|
+
return await fn();
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
const isRetryable = attempt < maxRetries && retryableErrors.includes(error.code);
|
|
94
|
+
if (!isRetryable)
|
|
95
|
+
throw error;
|
|
96
|
+
// Exponential backoff with jitter
|
|
97
|
+
const delay = baseDelayMs * 2 ** attempt + Math.random() * baseDelayMs;
|
|
98
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
// TypeScript: unreachable, but satisfies return type
|
|
102
|
+
throw new Error('Retry loop exited unexpectedly');
|
|
103
|
+
}
|
|
104
|
+
// In-memory cache of created files to avoid expensive fs.access() calls
|
|
105
|
+
// This is safe because we only write once per file path (no overwrites without explicit flag)
|
|
106
|
+
const createdFilesCache = new Set();
|
|
107
|
+
/**
|
|
108
|
+
* Clear the created files cache. Useful for testing or when files are deleted externally.
|
|
109
|
+
*/
|
|
110
|
+
export function clearCreatedFilesCache() {
|
|
111
|
+
createdFilesCache.clear();
|
|
112
|
+
}
|
|
113
|
+
export { ulidToDate } from '@workflow/world';
|
|
114
|
+
/**
|
|
115
|
+
* Regex matching a tag suffix on a fileId (after `.json` has been stripped).
|
|
116
|
+
* E.g., `wrun_ABC.vitest-0` → the `.vitest-0` part.
|
|
117
|
+
* Tags start with a letter and contain alphanumeric chars and hyphens.
|
|
118
|
+
* Entity IDs (ULIDs, step_N, etc.) never contain dots, so the first dot
|
|
119
|
+
* always marks the tag boundary.
|
|
120
|
+
*/
|
|
121
|
+
const TAG_PATTERN = /\.[a-zA-Z][a-zA-Z0-9-]*$/;
|
|
122
|
+
/**
|
|
123
|
+
* Strip a tag suffix from a fileId.
|
|
124
|
+
* `wrun_ABC.vitest-0` → `wrun_ABC`
|
|
125
|
+
* `wrun_ABC` → `wrun_ABC` (no-op if no tag)
|
|
126
|
+
*/
|
|
127
|
+
export function stripTag(fileId) {
|
|
128
|
+
return fileId.replace(TAG_PATTERN, '');
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Build the file path for an entity, with optional tag embedded in the filename.
|
|
132
|
+
* `taggedPath('/data', 'runs', 'wrun_ABC', 'vitest-0')` → `/data/runs/wrun_ABC.vitest-0.json`
|
|
133
|
+
* `taggedPath('/data', 'runs', 'wrun_ABC')` → `/data/runs/wrun_ABC.json`
|
|
134
|
+
*
|
|
135
|
+
* The `fileId` and `tag` are validated with {@link assertSafeEntityId} and
|
|
136
|
+
* the result is containment-checked with {@link resolveWithinBase} to
|
|
137
|
+
* prevent path-traversal attacks when values are derived from untrusted
|
|
138
|
+
* request input.
|
|
139
|
+
*/
|
|
140
|
+
export function taggedPath(basedir, entityDir, fileId, tag) {
|
|
141
|
+
assertSafeEntityId('fileId', fileId);
|
|
142
|
+
if (tag !== undefined)
|
|
143
|
+
assertSafeEntityId('tag', tag);
|
|
144
|
+
const filename = tag ? `${fileId}.${tag}.json` : `${fileId}.json`;
|
|
145
|
+
return resolveWithinBase(basedir, entityDir, filename);
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Read a JSON entity with tagged fallback.
|
|
149
|
+
* When a tag is set, tries the tagged path first, then falls back to the
|
|
150
|
+
* untagged path (so a tagged world can read entities written without a tag).
|
|
151
|
+
*/
|
|
152
|
+
export async function readJSONWithFallback(basedir, entityDir, fileId, schema, tag) {
|
|
153
|
+
assertSafeEntityId('fileId', fileId);
|
|
154
|
+
if (tag !== undefined)
|
|
155
|
+
assertSafeEntityId('tag', tag);
|
|
156
|
+
if (tag) {
|
|
157
|
+
const result = await readJSON(resolveWithinBase(basedir, entityDir, `${fileId}.${tag}.json`), schema);
|
|
158
|
+
if (result !== null)
|
|
159
|
+
return result;
|
|
160
|
+
}
|
|
161
|
+
return readJSON(resolveWithinBase(basedir, entityDir, `${fileId}.json`), schema);
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* List all filenames in a directory that have a specific tag.
|
|
165
|
+
* Returns full filenames (e.g., `wrun_ABC.vitest-0.json`).
|
|
166
|
+
*/
|
|
167
|
+
export async function listTaggedFiles(dirPath, tag) {
|
|
168
|
+
const suffix = `.${tag}.json`;
|
|
169
|
+
try {
|
|
170
|
+
const files = await fs.readdir(dirPath);
|
|
171
|
+
return files.filter((f) => f.endsWith(suffix));
|
|
172
|
+
}
|
|
173
|
+
catch (error) {
|
|
174
|
+
if (error.code === 'ENOENT')
|
|
175
|
+
return [];
|
|
176
|
+
throw error;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* List all filenames in a directory that have a specific tag and extension.
|
|
181
|
+
* Returns full filenames (e.g., `stream-chnk_ABC.vitest-0.bin`).
|
|
182
|
+
*/
|
|
183
|
+
export async function listTaggedFilesByExtension(dirPath, tag, extension) {
|
|
184
|
+
const suffix = `.${tag}${extension}`;
|
|
185
|
+
try {
|
|
186
|
+
const files = await fs.readdir(dirPath);
|
|
187
|
+
return files.filter((f) => f.endsWith(suffix));
|
|
188
|
+
}
|
|
189
|
+
catch (error) {
|
|
190
|
+
if (error.code === 'ENOENT')
|
|
191
|
+
return [];
|
|
192
|
+
throw error;
|
|
12
193
|
}
|
|
13
|
-
return new Date(decodeTime(ulid.data));
|
|
14
194
|
}
|
|
15
195
|
export async function ensureDir(dirPath) {
|
|
16
196
|
try {
|
|
@@ -20,14 +200,56 @@ export async function ensureDir(dirPath) {
|
|
|
20
200
|
// Ignore if already exists
|
|
21
201
|
}
|
|
22
202
|
}
|
|
203
|
+
/**
|
|
204
|
+
* Custom JSON replacer that encodes Uint8Array as base64 strings.
|
|
205
|
+
* Format: { __type: 'Uint8Array', data: '<base64>' }
|
|
206
|
+
*/
|
|
207
|
+
export function jsonReplacer(_key, value) {
|
|
208
|
+
if (value instanceof Uint8Array) {
|
|
209
|
+
return {
|
|
210
|
+
__type: 'Uint8Array',
|
|
211
|
+
data: Buffer.from(value).toString('base64'),
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
return value;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Custom JSON reviver that decodes base64 strings back to Uint8Array.
|
|
218
|
+
*/
|
|
219
|
+
export function jsonReviver(_key, value) {
|
|
220
|
+
if (value !== null &&
|
|
221
|
+
typeof value === 'object' &&
|
|
222
|
+
value.__type === 'Uint8Array' &&
|
|
223
|
+
typeof value.data === 'string') {
|
|
224
|
+
return new Uint8Array(Buffer.from(value.data, 'base64'));
|
|
225
|
+
}
|
|
226
|
+
return value;
|
|
227
|
+
}
|
|
23
228
|
export async function writeJSON(filePath, data, opts) {
|
|
24
|
-
return write(filePath, JSON.stringify(data,
|
|
229
|
+
return write(filePath, JSON.stringify(data, jsonReplacer, 2), opts);
|
|
25
230
|
}
|
|
231
|
+
/**
|
|
232
|
+
* Writes data to a file using atomic write-rename pattern.
|
|
233
|
+
*
|
|
234
|
+
* Note: While this function uses temp files to avoid partial writes,
|
|
235
|
+
* it does not provide protection against concurrent writes from multiple
|
|
236
|
+
* processes. In a multi-writer scenario, the last writer wins.
|
|
237
|
+
* For production use with multiple writers, consider using a proper
|
|
238
|
+
* database or locking mechanism.
|
|
239
|
+
*/
|
|
26
240
|
export async function write(filePath, data, opts) {
|
|
27
241
|
if (!opts?.overwrite) {
|
|
242
|
+
// Fast path: check in-memory cache first to avoid expensive fs.access() calls
|
|
243
|
+
// This provides significant performance improvement when creating many files
|
|
244
|
+
if (createdFilesCache.has(filePath)) {
|
|
245
|
+
throw new EntityConflictError(`File ${filePath} already exists and 'overwrite' is false`);
|
|
246
|
+
}
|
|
247
|
+
// Slow path: check filesystem for files created before this process started
|
|
28
248
|
try {
|
|
29
249
|
await fs.access(filePath);
|
|
30
|
-
|
|
250
|
+
// File exists on disk, add to cache for future checks
|
|
251
|
+
createdFilesCache.add(filePath);
|
|
252
|
+
throw new EntityConflictError(`File ${filePath} already exists and 'overwrite' is false`);
|
|
31
253
|
}
|
|
32
254
|
catch (error) {
|
|
33
255
|
// If file doesn't exist (ENOENT), continue with write
|
|
@@ -37,20 +259,27 @@ export async function write(filePath, data, opts) {
|
|
|
37
259
|
}
|
|
38
260
|
}
|
|
39
261
|
const tempPath = `${filePath}.tmp.${ulid()}`;
|
|
262
|
+
let tempFileCreated = false;
|
|
40
263
|
try {
|
|
41
264
|
await ensureDir(path.dirname(filePath));
|
|
42
265
|
await fs.writeFile(tempPath, data);
|
|
43
|
-
|
|
266
|
+
tempFileCreated = true;
|
|
267
|
+
await withWindowsRetry(() => fs.rename(tempPath, filePath));
|
|
268
|
+
// Track this file in cache so future writes know it exists
|
|
269
|
+
createdFilesCache.add(filePath);
|
|
44
270
|
}
|
|
45
271
|
catch (error) {
|
|
46
|
-
|
|
272
|
+
// Only try to clean up temp file if it was actually created
|
|
273
|
+
if (tempFileCreated) {
|
|
274
|
+
await withWindowsRetry(() => fs.unlink(tempPath), 3).catch(() => { });
|
|
275
|
+
}
|
|
47
276
|
throw error;
|
|
48
277
|
}
|
|
49
278
|
}
|
|
50
279
|
export async function readJSON(filePath, decoder) {
|
|
51
280
|
try {
|
|
52
281
|
const content = await fs.readFile(filePath, 'utf-8');
|
|
53
|
-
return decoder.parse(JSON.parse(content));
|
|
282
|
+
return decoder.parse(JSON.parse(content, jsonReviver));
|
|
54
283
|
}
|
|
55
284
|
catch (error) {
|
|
56
285
|
if (error.code === 'ENOENT')
|
|
@@ -71,12 +300,33 @@ export async function deleteJSON(filePath) {
|
|
|
71
300
|
throw error;
|
|
72
301
|
}
|
|
73
302
|
}
|
|
303
|
+
/**
|
|
304
|
+
* Atomically create a file using O_CREAT | O_EXCL flags.
|
|
305
|
+
* Returns true if the file was created, false if it already exists.
|
|
306
|
+
* This is atomic at the OS level, safe for concurrent access.
|
|
307
|
+
*/
|
|
308
|
+
export async function writeExclusive(filePath, data) {
|
|
309
|
+
await ensureDir(path.dirname(filePath));
|
|
310
|
+
try {
|
|
311
|
+
await fs.writeFile(filePath, data, { flag: 'wx' });
|
|
312
|
+
return true;
|
|
313
|
+
}
|
|
314
|
+
catch (error) {
|
|
315
|
+
if (error.code === 'EEXIST') {
|
|
316
|
+
return false;
|
|
317
|
+
}
|
|
318
|
+
throw error;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
74
321
|
export async function listJSONFiles(dirPath) {
|
|
322
|
+
return listFilesByExtension(dirPath, '.json');
|
|
323
|
+
}
|
|
324
|
+
export async function listFilesByExtension(dirPath, extension) {
|
|
75
325
|
try {
|
|
76
326
|
const files = await fs.readdir(dirPath);
|
|
77
327
|
return files
|
|
78
|
-
.filter((f) => f.endsWith(
|
|
79
|
-
.map((f) => f.
|
|
328
|
+
.filter((f) => f.endsWith(extension))
|
|
329
|
+
.map((f) => f.slice(0, -extension.length));
|
|
80
330
|
}
|
|
81
331
|
catch (error) {
|
|
82
332
|
if (error.code === 'ENOENT')
|
|
@@ -98,6 +348,14 @@ function createCursor(timestamp, id) {
|
|
|
98
348
|
}
|
|
99
349
|
export async function paginatedFileSystemQuery(config) {
|
|
100
350
|
const { directory, schema, filePrefix, filter, sortOrder = 'desc', limit = 20, cursor, getCreatedAt, getId, } = config;
|
|
351
|
+
// Validate filePrefix (typically `${runId}-`) so request-derived prefixes
|
|
352
|
+
// consistently reject unsafe characters. filePrefix is only used below to
|
|
353
|
+
// filter readdir() results by prefix — it doesn't participate in path
|
|
354
|
+
// construction — but keeping the validation rule uniform across the
|
|
355
|
+
// storage layer avoids special cases and catches bad values earlier.
|
|
356
|
+
if (filePrefix !== undefined) {
|
|
357
|
+
assertSafeEntityId('filePrefix', filePrefix);
|
|
358
|
+
}
|
|
101
359
|
// 1. Get all JSON files in directory
|
|
102
360
|
const fileIds = await listJSONFiles(directory);
|
|
103
361
|
// 2. Filter by prefix if provided
|
|
@@ -130,21 +388,30 @@ export async function paginatedFileSystemQuery(config) {
|
|
|
130
388
|
: fileTime > cursorTime;
|
|
131
389
|
}
|
|
132
390
|
}
|
|
133
|
-
//
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
}
|
|
137
|
-
else {
|
|
138
|
-
// Even without cursor, skip files where getCreatedAt returns null for consistency
|
|
139
|
-
candidateFileIds = relevantFileIds.filter((fileId) => {
|
|
140
|
-
return getCreatedAt(`${fileId}.json`) !== null;
|
|
391
|
+
// Can't extract timestamp from filename (e.g., steps use sequential IDs).
|
|
392
|
+
// Include the file and defer to JSON-based filtering below.
|
|
393
|
+
return true;
|
|
141
394
|
});
|
|
142
395
|
}
|
|
143
396
|
// 4. Load files individually and collect valid items
|
|
144
397
|
const validItems = [];
|
|
145
398
|
for (const fileId of candidateFileIds) {
|
|
146
399
|
const filePath = path.join(directory, `${fileId}.json`);
|
|
147
|
-
|
|
400
|
+
let item = null;
|
|
401
|
+
try {
|
|
402
|
+
item = await readJSON(filePath, schema);
|
|
403
|
+
}
|
|
404
|
+
catch (error) {
|
|
405
|
+
// We don't expect zod errors to happen, but if the JSON does get malformed,
|
|
406
|
+
// we skip the item. Preferably, we'd have a way to mark items as malformed,
|
|
407
|
+
// so that the UI can display them as such, with richer messaging. In the meantime,
|
|
408
|
+
// we just log a warning and skip the item.
|
|
409
|
+
if (error instanceof z.ZodError) {
|
|
410
|
+
console.warn(`Skipping item ${fileId} due to malformed JSON: ${error.message}`);
|
|
411
|
+
continue;
|
|
412
|
+
}
|
|
413
|
+
throw error;
|
|
414
|
+
}
|
|
148
415
|
if (item) {
|
|
149
416
|
// Apply custom filter early if provided
|
|
150
417
|
if (filter && !filter(item))
|