@workflow/world-local 4.0.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +21 -0
- package/README.md +8 -0
- package/dist/auth.d.ts +3 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +18 -0
- package/dist/auth.js.map +1 -0
- package/dist/config.d.ts +8 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +19 -0
- package/dist/config.js.map +1 -0
- package/dist/fs.d.ts +29 -0
- package/dist/fs.d.ts.map +1 -0
- package/dist/fs.js +210 -0
- package/dist/fs.js.map +1 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +22 -0
- package/dist/index.js.map +1 -0
- package/dist/queue.d.ts +3 -0
- package/dist/queue.d.ts.map +1 -0
- package/dist/queue.js +120 -0
- package/dist/queue.js.map +1 -0
- package/dist/storage.d.ts +3 -0
- package/dist/storage.d.ts.map +1 -0
- package/dist/storage.js +415 -0
- package/dist/storage.js.map +1 -0
- package/dist/streamer.d.ts +19 -0
- package/dist/streamer.d.ts.map +1 -0
- package/dist/streamer.js +127 -0
- package/dist/streamer.js.map +1 -0
- package/dist/util.d.ts +14 -0
- package/dist/util.d.ts.map +1 -0
- package/dist/util.js +21 -0
- package/dist/util.js.map +1 -0
- package/package.json +48 -0
package/LICENSE.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Vercel, Inc.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# @workflow/world-local
|
|
2
|
+
|
|
3
|
+
Filesystem-based workflow backend for local development and testing.
|
|
4
|
+
|
|
5
|
+
Stores workflow data as JSON files on disk and provides in-memory queuing. Automatically detects development server port for queue transport.
|
|
6
|
+
|
|
7
|
+
Used by default on `next dev` and `next start`.
|
|
8
|
+
|
package/dist/auth.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth.d.ts","sourceRoot":"","sources":["../src/auth.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAEpD,eAAO,MAAM,IAAI,EAAE,YAiBlB,CAAC"}
|
package/dist/auth.js
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export const auth = {
|
|
2
|
+
async getAuthInfo() {
|
|
3
|
+
return {
|
|
4
|
+
ownerId: 'embedded-owner',
|
|
5
|
+
projectId: 'embedded-project',
|
|
6
|
+
environment: 'embedded',
|
|
7
|
+
userId: 'embedded-user',
|
|
8
|
+
};
|
|
9
|
+
},
|
|
10
|
+
async checkHealth() {
|
|
11
|
+
return {
|
|
12
|
+
success: true,
|
|
13
|
+
data: { healthy: true },
|
|
14
|
+
message: 'Embedded backend is healthy',
|
|
15
|
+
};
|
|
16
|
+
},
|
|
17
|
+
};
|
|
18
|
+
//# sourceMappingURL=auth.js.map
|
package/dist/auth.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth.js","sourceRoot":"","sources":["../src/auth.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,IAAI,GAAiB;IAChC,KAAK,CAAC,WAAW;QACf,OAAO;YACL,OAAO,EAAE,gBAAgB;YACzB,SAAS,EAAE,kBAAkB;YAC7B,WAAW,EAAE,UAAU;YACvB,MAAM,EAAE,eAAe;SACxB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,WAAW;QACf,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;YACvB,OAAO,EAAE,6BAA6B;SACvC,CAAC;IACJ,CAAC;CACF,CAAC"}
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAMA,eAAO,MAAM,2BAA2B,QAAQ,CAAC;AAWjD,eAAO,MAAM,MAAM;;;;;CAKjB,CAAC"}
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { once } from './util.js';
|
|
2
|
+
const getDataDirFromEnv = () => {
|
|
3
|
+
return process.env.WORKFLOW_EMBEDDED_DATA_DIR || '.workflow-data';
|
|
4
|
+
};
|
|
5
|
+
export const DEFAULT_RESOLVE_DATA_OPTION = 'all';
|
|
6
|
+
const getPortFromEnv = () => {
|
|
7
|
+
const port = process.env.PORT;
|
|
8
|
+
if (port) {
|
|
9
|
+
return Number(port);
|
|
10
|
+
}
|
|
11
|
+
//
|
|
12
|
+
return 3000;
|
|
13
|
+
};
|
|
14
|
+
export const config = once(() => {
|
|
15
|
+
const dataDir = getDataDirFromEnv();
|
|
16
|
+
const port = getPortFromEnv();
|
|
17
|
+
return { dataDir, port };
|
|
18
|
+
});
|
|
19
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,iBAAiB,GAAG,GAAG,EAAE;IAC7B,OAAO,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,gBAAgB,CAAC;AACpE,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,2BAA2B,GAAG,KAAK,CAAC;AAEjD,MAAM,cAAc,GAAG,GAAG,EAAE;IAC1B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;IAC9B,IAAI,IAAI,EAAE,CAAC;QACT,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;IACtB,CAAC;IACD,EAAE;IACF,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE;IAC9B,MAAM,OAAO,GAAG,iBAAiB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAE9B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC,CAAC,CAAC"}
|
package/dist/fs.d.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { PaginatedResponse } from '@workflow/world';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
export declare function ulidToDate(maybeUlid: string): Date | null;
|
|
4
|
+
export declare function ensureDir(dirPath: string): Promise<void>;
|
|
5
|
+
interface WriteOptions {
|
|
6
|
+
overwrite?: boolean;
|
|
7
|
+
}
|
|
8
|
+
export declare function writeJSON(filePath: string, data: any, opts?: WriteOptions): Promise<void>;
|
|
9
|
+
export declare function write(filePath: string, data: string | Buffer, opts?: WriteOptions): Promise<void>;
|
|
10
|
+
export declare function readJSON<T>(filePath: string, decoder: z.ZodType<T>): Promise<T | null>;
|
|
11
|
+
export declare function readBuffer(filePath: string): Promise<Buffer>;
|
|
12
|
+
export declare function deleteJSON(filePath: string): Promise<void>;
|
|
13
|
+
export declare function listJSONFiles(dirPath: string): Promise<string[]>;
|
|
14
|
+
interface PaginatedFileSystemQueryConfig<T> {
|
|
15
|
+
directory: string;
|
|
16
|
+
schema: z.ZodType<T>;
|
|
17
|
+
filePrefix?: string;
|
|
18
|
+
filter?: (item: T) => boolean;
|
|
19
|
+
sortOrder?: 'asc' | 'desc';
|
|
20
|
+
limit?: number;
|
|
21
|
+
cursor?: string;
|
|
22
|
+
getCreatedAt(filename: string): Date | null;
|
|
23
|
+
getId?(item: T): string;
|
|
24
|
+
}
|
|
25
|
+
export declare function paginatedFileSystemQuery<T extends {
|
|
26
|
+
createdAt: Date;
|
|
27
|
+
}>(config: PaginatedFileSystemQueryConfig<T>): Promise<PaginatedResponse<T>>;
|
|
28
|
+
export {};
|
|
29
|
+
//# sourceMappingURL=fs.d.ts.map
|
package/dist/fs.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEzD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAOzD;AAED,wBAAsB,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAM9D;AAED,UAAU,YAAY;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,GAAG,EACT,IAAI,CAAC,EAAE,YAAY,GAClB,OAAO,CAAC,IAAI,CAAC,CAEf;AAED,wBAAsB,KAAK,CACzB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,GAAG,MAAM,EACrB,IAAI,CAAC,EAAE,YAAY,GAClB,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,QAAQ,CAAC,CAAC,EAC9B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GACpB,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAQnB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAGlE;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMhE;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAUtE;AAED,UAAU,8BAA8B,CAAC,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC;IAC9B,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAAC;IAC5C,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC;CACzB;AAqBD,wBAAsB,wBAAwB,CAAC,CAAC,SAAS;IAAE,SAAS,EAAE,IAAI,CAAA;CAAE,EAC1E,MAAM,EAAE,8BAA8B,CAAC,CAAC,CAAC,GACxC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAoI/B"}
|
package/dist/fs.js
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import { promises as fs } from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { WorkflowAPIError } from '@workflow/errors';
|
|
4
|
+
import { decodeTime, monotonicFactory } from 'ulid';
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
const ulid = monotonicFactory(() => Math.random());
|
|
7
|
+
const Ulid = z.string().ulid();
|
|
8
|
+
export function ulidToDate(maybeUlid) {
|
|
9
|
+
const ulid = Ulid.safeParse(maybeUlid);
|
|
10
|
+
if (!ulid.success) {
|
|
11
|
+
return null;
|
|
12
|
+
}
|
|
13
|
+
return new Date(decodeTime(ulid.data));
|
|
14
|
+
}
|
|
15
|
+
export async function ensureDir(dirPath) {
|
|
16
|
+
try {
|
|
17
|
+
await fs.mkdir(dirPath, { recursive: true });
|
|
18
|
+
}
|
|
19
|
+
catch (_error) {
|
|
20
|
+
// Ignore if already exists
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
export async function writeJSON(filePath, data, opts) {
|
|
24
|
+
return write(filePath, JSON.stringify(data, null, 2), opts);
|
|
25
|
+
}
|
|
26
|
+
export async function write(filePath, data, opts) {
|
|
27
|
+
if (!opts?.overwrite) {
|
|
28
|
+
try {
|
|
29
|
+
await fs.access(filePath);
|
|
30
|
+
throw new WorkflowAPIError(`File ${filePath} already exists and 'overwrite' is false`, { status: 409 });
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
// If file doesn't exist (ENOENT), continue with write
|
|
34
|
+
if (error.code !== 'ENOENT') {
|
|
35
|
+
throw error;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
const tempPath = `${filePath}.tmp.${ulid()}`;
|
|
40
|
+
try {
|
|
41
|
+
await ensureDir(path.dirname(filePath));
|
|
42
|
+
await fs.writeFile(tempPath, data);
|
|
43
|
+
await fs.rename(tempPath, filePath);
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
await fs.unlink(tempPath).catch(() => { });
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
export async function readJSON(filePath, decoder) {
|
|
51
|
+
try {
|
|
52
|
+
const content = await fs.readFile(filePath, 'utf-8');
|
|
53
|
+
return decoder.parse(JSON.parse(content));
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
if (error.code === 'ENOENT')
|
|
57
|
+
return null;
|
|
58
|
+
throw error;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
export async function readBuffer(filePath) {
|
|
62
|
+
const content = await fs.readFile(filePath);
|
|
63
|
+
return content;
|
|
64
|
+
}
|
|
65
|
+
export async function deleteJSON(filePath) {
|
|
66
|
+
try {
|
|
67
|
+
await fs.unlink(filePath);
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
if (error.code !== 'ENOENT')
|
|
71
|
+
throw error;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
export async function listJSONFiles(dirPath) {
|
|
75
|
+
try {
|
|
76
|
+
const files = await fs.readdir(dirPath);
|
|
77
|
+
return files
|
|
78
|
+
.filter((f) => f.endsWith('.json'))
|
|
79
|
+
.map((f) => f.replace('.json', ''));
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
if (error.code === 'ENOENT')
|
|
83
|
+
return [];
|
|
84
|
+
throw error;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function parseCursor(cursor) {
|
|
88
|
+
if (!cursor)
|
|
89
|
+
return null;
|
|
90
|
+
const parts = cursor.split('|');
|
|
91
|
+
return {
|
|
92
|
+
timestamp: new Date(parts[0]),
|
|
93
|
+
id: parts[1] || null,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function createCursor(timestamp, id) {
|
|
97
|
+
return id ? `${timestamp.toISOString()}|${id}` : timestamp.toISOString();
|
|
98
|
+
}
|
|
99
|
+
export async function paginatedFileSystemQuery(config) {
|
|
100
|
+
const { directory, schema, filePrefix, filter, sortOrder = 'desc', limit = 20, cursor, getCreatedAt, getId, } = config;
|
|
101
|
+
// 1. Get all JSON files in directory
|
|
102
|
+
const fileIds = await listJSONFiles(directory);
|
|
103
|
+
// 2. Filter by prefix if provided
|
|
104
|
+
const relevantFileIds = filePrefix
|
|
105
|
+
? fileIds.filter((fileId) => fileId.startsWith(filePrefix))
|
|
106
|
+
: fileIds;
|
|
107
|
+
// 3. ULID Optimization: Filter by cursor using filename timestamps before loading JSON
|
|
108
|
+
const parsedCursor = parseCursor(cursor);
|
|
109
|
+
let candidateFileIds = relevantFileIds;
|
|
110
|
+
if (parsedCursor) {
|
|
111
|
+
candidateFileIds = relevantFileIds.filter((fileId) => {
|
|
112
|
+
const filenameDate = getCreatedAt(`${fileId}.json`);
|
|
113
|
+
if (filenameDate) {
|
|
114
|
+
// Use filename timestamp for cursor filtering
|
|
115
|
+
// We need to be careful here: if parsedCursor has an ID (for tie-breaking),
|
|
116
|
+
// we need to include items with the same timestamp for later ID-based filtering.
|
|
117
|
+
// If no ID, we can use strict inequality for optimization.
|
|
118
|
+
const cursorTime = parsedCursor.timestamp.getTime();
|
|
119
|
+
const fileTime = filenameDate.getTime();
|
|
120
|
+
if (parsedCursor.id) {
|
|
121
|
+
// Tie-breaking mode: include items at or near cursor timestamp
|
|
122
|
+
return sortOrder === 'desc'
|
|
123
|
+
? fileTime <= cursorTime
|
|
124
|
+
: fileTime >= cursorTime;
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
// No tie-breaking: strict inequality
|
|
128
|
+
return sortOrder === 'desc'
|
|
129
|
+
? fileTime < cursorTime
|
|
130
|
+
: fileTime > cursorTime;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
// Skip files where we can't extract timestamp - no optimization benefit
|
|
134
|
+
return false;
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
else {
|
|
138
|
+
// Even without cursor, skip files where getCreatedAt returns null for consistency
|
|
139
|
+
candidateFileIds = relevantFileIds.filter((fileId) => {
|
|
140
|
+
return getCreatedAt(`${fileId}.json`) !== null;
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
// 4. Load files individually and collect valid items
|
|
144
|
+
const validItems = [];
|
|
145
|
+
for (const fileId of candidateFileIds) {
|
|
146
|
+
const filePath = path.join(directory, `${fileId}.json`);
|
|
147
|
+
const item = await readJSON(filePath, schema);
|
|
148
|
+
if (item) {
|
|
149
|
+
// Apply custom filter early if provided
|
|
150
|
+
if (filter && !filter(item))
|
|
151
|
+
continue;
|
|
152
|
+
// Double-check cursor filtering with actual createdAt from JSON
|
|
153
|
+
// (in case ULID timestamp differs from stored createdAt)
|
|
154
|
+
if (parsedCursor) {
|
|
155
|
+
const itemTime = item.createdAt.getTime();
|
|
156
|
+
const cursorTime = parsedCursor.timestamp.getTime();
|
|
157
|
+
if (sortOrder === 'desc') {
|
|
158
|
+
// For descending order, skip items >= cursor
|
|
159
|
+
if (itemTime > cursorTime)
|
|
160
|
+
continue;
|
|
161
|
+
// If timestamps are equal, use ID for tie-breaking (skip if ID >= cursorId)
|
|
162
|
+
if (itemTime === cursorTime && parsedCursor.id && getId) {
|
|
163
|
+
const itemId = getId(item);
|
|
164
|
+
if (itemId >= parsedCursor.id)
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
else {
|
|
169
|
+
// For ascending order, skip items <= cursor
|
|
170
|
+
if (itemTime < cursorTime)
|
|
171
|
+
continue;
|
|
172
|
+
// If timestamps are equal, use ID for tie-breaking (skip if ID <= cursorId)
|
|
173
|
+
if (itemTime === cursorTime && parsedCursor.id && getId) {
|
|
174
|
+
const itemId = getId(item);
|
|
175
|
+
if (itemId <= parsedCursor.id)
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
validItems.push(item);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
// 5. Sort by createdAt (and by ID for tie-breaking if getId is provided)
|
|
184
|
+
validItems.sort((a, b) => {
|
|
185
|
+
const aTime = a.createdAt.getTime();
|
|
186
|
+
const bTime = b.createdAt.getTime();
|
|
187
|
+
const timeComparison = sortOrder === 'asc' ? aTime - bTime : bTime - aTime;
|
|
188
|
+
// If timestamps are equal and we have getId, use ID for stable sorting
|
|
189
|
+
if (timeComparison === 0 && getId) {
|
|
190
|
+
const aId = getId(a);
|
|
191
|
+
const bId = getId(b);
|
|
192
|
+
return sortOrder === 'asc'
|
|
193
|
+
? aId.localeCompare(bId)
|
|
194
|
+
: bId.localeCompare(aId);
|
|
195
|
+
}
|
|
196
|
+
return timeComparison;
|
|
197
|
+
});
|
|
198
|
+
// 6. Apply pagination
|
|
199
|
+
const hasMore = validItems.length > limit;
|
|
200
|
+
const items = hasMore ? validItems.slice(0, limit) : validItems;
|
|
201
|
+
const nextCursor = hasMore && items.length > 0
|
|
202
|
+
? createCursor(items[items.length - 1].createdAt, getId?.(items[items.length - 1]))
|
|
203
|
+
: null;
|
|
204
|
+
return {
|
|
205
|
+
data: items,
|
|
206
|
+
cursor: nextCursor,
|
|
207
|
+
hasMore,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
//# sourceMappingURL=fs.js.map
|
package/dist/fs.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fs.js","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,MAAM,CAAC;AACpD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAEnD,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;AAE/B,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IACvC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAe;IAC7C,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,MAAM,EAAE,CAAC;QAChB,2BAA2B;IAC7B,CAAC;AACH,CAAC;AAMD,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAgB,EAChB,IAAS,EACT,IAAmB;IAEnB,OAAO,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,QAAgB,EAChB,IAAqB,EACrB,IAAmB;IAEnB,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC1B,MAAM,IAAI,gBAAgB,CACxB,QAAQ,QAAQ,0CAA0C,EAC1D,EAAE,MAAM,EAAE,GAAG,EAAE,CAChB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,sDAAsD;YACtD,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,QAAQ,QAAQ,IAAI,EAAE,EAAE,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACxC,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC1C,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,QAAgB,EAChB,OAAqB;IAErB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACrD,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAAa,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAClD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB;IAC/C,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC5C,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB;IAC/C,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAAa,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC;IACpD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAe;IACjD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxC,OAAO,KAAK;aACT,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAAa,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAChD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAmBD,SAAS,WAAW,CAAC,MAA0B;IAC7C,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC7B,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI;KACrB,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,SAAe,EAAE,EAAsB;IAC3D,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;AAC3E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,MAAyC;IAEzC,MAAM,EACJ,SAAS,EACT,MAAM,EACN,UAAU,EACV,MAAM,EACN,SAAS,GAAG,MAAM,EAClB,KAAK,GAAG,EAAE,EACV,MAAM,EACN,YAAY,EACZ,KAAK,GACN,GAAG,MAAM,CAAC;IAEX,qCAAqC;IACrC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;IAE/C,kCAAkC;IAClC,MAAM,eAAe,GAAG,UAAU;QAChC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC3D,CAAC,CAAC,OAAO,CAAC;IAEZ,uFAAuF;IACvF,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,gBAAgB,GAAG,eAAe,CAAC;IAEvC,IAAI,YAAY,EAAE,CAAC;QACjB,gBAAgB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE;YACnD,MAAM,YAAY,GAAG,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC;YACpD,IAAI,YAAY,EAAE,CAAC;gBACjB,8CAA8C;gBAC9C,4EAA4E;gBAC5E,iFAAiF;gBACjF,2DAA2D;gBAC3D,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC;gBAExC,IAAI,YAAY,CAAC,EAAE,EAAE,CAAC;oBACpB,+DAA+D;oBAC/D,OAAO,SAAS,KAAK,MAAM;wBACzB,CAAC,CAAC,QAAQ,IAAI,UAAU;wBACxB,CAAC,CAAC,QAAQ,IAAI,UAAU,CAAC;gBAC7B,CAAC;qBAAM,CAAC;oBACN,qCAAqC;oBACrC,OAAO,SAAS,KAAK,MAAM;wBACzB,CAAC,CAAC,QAAQ,GAAG,UAAU;wBACvB,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC;gBAC5B,CAAC;YACH,CAAC;YACD,wEAAwE;YACxE,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,kFAAkF;QAClF,gBAAgB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE;YACnD,OAAO,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,KAAK,IAAI,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,qDAAqD;IACrD,MAAM,UAAU,GAAQ,EAAE,CAAC;IAE3B,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAE9C,IAAI,IAAI,EAAE,CAAC;YACT,wCAAwC;YACxC,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;gBAAE,SAAS;YAEtC,gEAAgE;YAChE,yDAAyD;YACzD,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;gBAC1C,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;gBAEpD,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;oBACzB,6CAA6C;oBAC7C,IAAI,QAAQ,GAAG,UAAU;wBAAE,SAAS;oBACpC,4EAA4E;oBAC5E,IAAI,QAAQ,KAAK,UAAU,IAAI,YAAY,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;wBACxD,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;wBAC3B,IAAI,MAAM,IAAI,YAAY,CAAC,EAAE;4BAAE,SAAS;oBAC1C,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,4CAA4C;oBAC5C,IAAI,QAAQ,GAAG,UAAU;wBAAE,SAAS;oBACpC,4EAA4E;oBAC5E,IAAI,QAAQ,KAAK,UAAU,IAAI,YAAY,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;wBACxD,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;wBAC3B,IAAI,MAAM,IAAI,YAAY,CAAC,EAAE;4BAAE,SAAS;oBAC1C,CAAC;gBACH,CAAC;YACH,CAAC;YAED,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACvB,MAAM,KAAK,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACpC,MAAM,cAAc,GAAG,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;QAE3E,uEAAuE;QACvE,IAAI,cAAc,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACrB,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACrB,OAAO,SAAS,KAAK,KAAK;gBACxB,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC;gBACxB,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,cAAc,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,sBAAsB;IACtB,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,GAAG,KAAK,CAAC;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAChE,MAAM,UAAU,GACd,OAAO,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QACzB,CAAC,CAAC,YAAY,CACV,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,EACjC,KAAK,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CACjC;QACH,CAAC,CAAC,IAAI,CAAC;IAEX,OAAO;QACL,IAAI,EAAE,KAAK;QACX,MAAM,EAAE,UAAU;QAClB,OAAO;KACR,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { World } from '@workflow/world';
|
|
2
|
+
/**
|
|
3
|
+
* Creates an embedded world instance that combines queue, storage, streamer, and authentication functionalities.
|
|
4
|
+
*
|
|
5
|
+
* @param dataDir - The directory to use for storage. If not provided, the default data dir will be used.
|
|
6
|
+
* @param port - The port to use for the queue. If not provided, the default port will be used.
|
|
7
|
+
*/
|
|
8
|
+
export declare function createEmbeddedWorld({ dataDir, port, }: {
|
|
9
|
+
dataDir?: string;
|
|
10
|
+
port?: number;
|
|
11
|
+
}): World;
|
|
12
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAO7C;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,EAClC,OAAO,EACP,IAAI,GACL,EAAE;IACD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,KAAK,CASR"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { auth } from './auth.js';
|
|
2
|
+
import { config } from './config.js';
|
|
3
|
+
import { createQueue } from './queue.js';
|
|
4
|
+
import { createStorage } from './storage.js';
|
|
5
|
+
import { createStreamer } from './streamer.js';
|
|
6
|
+
/**
|
|
7
|
+
* Creates an embedded world instance that combines queue, storage, streamer, and authentication functionalities.
|
|
8
|
+
*
|
|
9
|
+
* @param dataDir - The directory to use for storage. If not provided, the default data dir will be used.
|
|
10
|
+
* @param port - The port to use for the queue. If not provided, the default port will be used.
|
|
11
|
+
*/
|
|
12
|
+
export function createEmbeddedWorld({ dataDir, port, }) {
|
|
13
|
+
const dir = dataDir ?? config.value.dataDir;
|
|
14
|
+
const queuePort = port ?? config.value.port;
|
|
15
|
+
return {
|
|
16
|
+
...createQueue(queuePort),
|
|
17
|
+
...createStorage(dir),
|
|
18
|
+
...createStreamer(dir),
|
|
19
|
+
...auth,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,EAClC,OAAO,EACP,IAAI,GAIL;IACC,MAAM,GAAG,GAAG,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;IAC5C,MAAM,SAAS,GAAG,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;IAC5C,OAAO;QACL,GAAG,WAAW,CAAC,SAAS,CAAC;QACzB,GAAG,aAAa,CAAC,GAAG,CAAC;QACrB,GAAG,cAAc,CAAC,GAAG,CAAC;QACtB,GAAG,IAAI;KACR,CAAC;AACJ,CAAC"}
|
package/dist/queue.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAEA,OAAO,EAAa,KAAK,KAAK,EAAkB,MAAM,iBAAiB,CAAC;AAIxE,wBAAgB,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,KAAK,CA4IhD"}
|
package/dist/queue.js
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { setTimeout } from 'node:timers/promises';
|
|
2
|
+
import { JsonTransport } from '@vercel/queue';
|
|
3
|
+
import { MessageId, ValidQueueName } from '@workflow/world';
|
|
4
|
+
import { monotonicFactory } from 'ulid';
|
|
5
|
+
import z from 'zod';
|
|
6
|
+
export function createQueue(port) {
|
|
7
|
+
const transport = new JsonTransport();
|
|
8
|
+
const generateId = monotonicFactory();
|
|
9
|
+
/**
|
|
10
|
+
* holds inflight messages by idempotency key to ensure
|
|
11
|
+
* that we don't queue the same message multiple times
|
|
12
|
+
*/
|
|
13
|
+
const inflightMessages = new Map();
|
|
14
|
+
const queue = async (queueName, message, opts) => {
|
|
15
|
+
const cleanup = [];
|
|
16
|
+
if (opts?.idempotencyKey) {
|
|
17
|
+
const existing = inflightMessages.get(opts.idempotencyKey);
|
|
18
|
+
if (existing) {
|
|
19
|
+
return { messageId: existing };
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
const body = transport.serialize(message);
|
|
23
|
+
let pathname;
|
|
24
|
+
if (queueName.startsWith('__wkf_step_')) {
|
|
25
|
+
pathname = `step`;
|
|
26
|
+
}
|
|
27
|
+
else if (queueName.startsWith('__wkf_workflow_')) {
|
|
28
|
+
pathname = `flow`;
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
throw new Error('Unknown queue name prefix');
|
|
32
|
+
}
|
|
33
|
+
const messageId = MessageId.parse(`msg_${generateId()}`);
|
|
34
|
+
if (opts?.idempotencyKey) {
|
|
35
|
+
const key = opts.idempotencyKey;
|
|
36
|
+
inflightMessages.set(key, messageId);
|
|
37
|
+
cleanup.push(() => {
|
|
38
|
+
inflightMessages.delete(key);
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
(async () => {
|
|
42
|
+
let defaultRetriesLeft = 3;
|
|
43
|
+
for (let attempt = 0; defaultRetriesLeft > 0; attempt++) {
|
|
44
|
+
defaultRetriesLeft--;
|
|
45
|
+
const response = await fetch(`http://localhost:${port}/.well-known/workflow/v1/${pathname}`, {
|
|
46
|
+
method: 'POST',
|
|
47
|
+
duplex: 'half',
|
|
48
|
+
headers: {
|
|
49
|
+
'x-vqs-queue-name': queueName,
|
|
50
|
+
'x-vqs-message-id': messageId,
|
|
51
|
+
'x-vqs-message-attempt': String(attempt + 1),
|
|
52
|
+
},
|
|
53
|
+
body,
|
|
54
|
+
});
|
|
55
|
+
if (response.ok) {
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
const text = await response.text();
|
|
59
|
+
if (response.status === 503) {
|
|
60
|
+
try {
|
|
61
|
+
const retryIn = Number(JSON.parse(text).retryIn);
|
|
62
|
+
await setTimeout(retryIn * 1000);
|
|
63
|
+
defaultRetriesLeft++;
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
catch { }
|
|
67
|
+
}
|
|
68
|
+
console.error(`[embedded world] Failed to queue message`, {
|
|
69
|
+
queueName,
|
|
70
|
+
text,
|
|
71
|
+
status: response.status,
|
|
72
|
+
headers: Object.fromEntries(response.headers.entries()),
|
|
73
|
+
body: body.toString(),
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
console.error(`[embedded world] Reached max retries of embedded world queue implementation`);
|
|
77
|
+
})().finally(() => {
|
|
78
|
+
for (const fn of cleanup) {
|
|
79
|
+
fn();
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
return { messageId };
|
|
83
|
+
};
|
|
84
|
+
const HeaderParser = z.object({
|
|
85
|
+
'x-vqs-queue-name': ValidQueueName,
|
|
86
|
+
'x-vqs-message-id': MessageId,
|
|
87
|
+
'x-vqs-message-attempt': z.coerce.number(),
|
|
88
|
+
});
|
|
89
|
+
const createQueueHandler = (prefix, handler) => {
|
|
90
|
+
return async (req) => {
|
|
91
|
+
const headers = HeaderParser.safeParse(Object.fromEntries(req.headers));
|
|
92
|
+
if (!headers.success || !req.body) {
|
|
93
|
+
return Response.json({ error: 'Missing required headers' }, { status: 400 });
|
|
94
|
+
}
|
|
95
|
+
const queueName = headers.data['x-vqs-queue-name'];
|
|
96
|
+
const messageId = headers.data['x-vqs-message-id'];
|
|
97
|
+
const attempt = headers.data['x-vqs-message-attempt'];
|
|
98
|
+
if (!queueName.startsWith(prefix)) {
|
|
99
|
+
return Response.json({ error: 'Unhandled queue' }, { status: 400 });
|
|
100
|
+
}
|
|
101
|
+
const body = await new JsonTransport().deserialize(req.body);
|
|
102
|
+
try {
|
|
103
|
+
const response = await handler(body, { attempt, queueName, messageId });
|
|
104
|
+
const retryIn = typeof response === 'undefined' ? null : response.timeoutSeconds;
|
|
105
|
+
if (retryIn) {
|
|
106
|
+
return Response.json({ retryIn }, { status: 503 });
|
|
107
|
+
}
|
|
108
|
+
return Response.json({ ok: true });
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
return Response.json(String(error), { status: 500 });
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
};
|
|
115
|
+
const getDeploymentId = async () => {
|
|
116
|
+
return 'dpl_embedded';
|
|
117
|
+
};
|
|
118
|
+
return { queue, createQueueHandler, getDeploymentId };
|
|
119
|
+
}
|
|
120
|
+
//# sourceMappingURL=queue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAc,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,CAAC,MAAM,KAAK,CAAC;AAEpB,MAAM,UAAU,WAAW,CAAC,IAAa;IACvC,MAAM,SAAS,GAAG,IAAI,aAAa,EAAE,CAAC;IACtC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;IAEtC;;;OAGG;IACH,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAqB,CAAC;IAEtD,MAAM,KAAK,GAAmB,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;QAC/D,MAAM,OAAO,GAAG,EAAoB,CAAC;QAErC,IAAI,IAAI,EAAE,cAAc,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC3D,IAAI,QAAQ,EAAE,CAAC;gBACb,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;YACjC,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC1C,IAAI,QAAgB,CAAC;QACrB,IAAI,SAAS,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;YACxC,QAAQ,GAAG,MAAM,CAAC;QACpB,CAAC;aAAM,IAAI,SAAS,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACnD,QAAQ,GAAG,MAAM,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QACD,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,UAAU,EAAE,EAAE,CAAC,CAAC;QAEzD,IAAI,IAAI,EAAE,cAAc,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC;YAChC,gBAAgB,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YACrC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;gBAChB,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC/B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,CAAC,KAAK,IAAI,EAAE;YACV,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAC3B,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,kBAAkB,GAAG,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;gBACxD,kBAAkB,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAC1B,oBAAoB,IAAI,4BAA4B,QAAQ,EAAE,EAC9D;oBACE,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE;wBACP,kBAAkB,EAAE,SAAS;wBAC7B,kBAAkB,EAAE,SAAS;wBAC7B,uBAAuB,EAAE,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC;qBAC7C;oBACD,IAAI;iBACL,CACF,CAAC;gBAEF,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;oBAChB,OAAO;gBACT,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAEnC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC;wBACjD,MAAM,UAAU,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;wBACjC,kBAAkB,EAAE,CAAC;wBACrB,SAAS;oBACX,CAAC;oBAAC,MAAM,CAAC,CAAA,CAAC;gBACZ,CAAC;gBAED,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE;oBACxD,SAAS;oBACT,IAAI;oBACJ,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACvD,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE;iBACtB,CAAC,CAAC;YACL,CAAC;YAED,OAAO,CAAC,KAAK,CACX,6EAA6E,CAC9E,CAAC;QACJ,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YAChB,KAAK,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;gBACzB,EAAE,EAAE,CAAC;YACP,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,EAAE,SAAS,EAAE,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;QAC5B,kBAAkB,EAAE,cAAc;QAClC,kBAAkB,EAAE,SAAS;QAC7B,uBAAuB,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE;KAC3C,CAAC,CAAC;IAEH,MAAM,kBAAkB,GAAgC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;QAC1E,OAAO,KAAK,EAAE,GAAG,EAAE,EAAE;YACnB,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;YAExE,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;gBAClC,OAAO,QAAQ,CAAC,IAAI,CAClB,EAAE,KAAK,EAAE,0BAA0B,EAAE,EACrC,EAAE,MAAM,EAAE,GAAG,EAAE,CAChB,CAAC;YACJ,CAAC;YAED,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACnD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACnD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAEtD,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClC,OAAO,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YACtE,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,aAAa,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7D,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;gBACxE,MAAM,OAAO,GACX,OAAO,QAAQ,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC;gBAEnE,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;gBACrD,CAAC;gBAED,OAAO,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;YACrC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,eAAe,GAA6B,KAAK,IAAI,EAAE;QAC3D,OAAO,cAAc,CAAC;IACxB,CAAC,CAAC;IAEF,OAAO,EAAE,KAAK,EAAE,kBAAkB,EAAE,eAAe,EAAE,CAAC;AACxD,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../src/storage.ts"],"names":[],"mappings":"AAEA,OAAO,EAOL,KAAK,OAAO,EAGb,MAAM,iBAAiB,CAAC;AAoFzB,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAqZtD"}
|
package/dist/storage.js
ADDED
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { WorkflowRunNotFoundError } from '@workflow/errors';
|
|
3
|
+
import { EventSchema, HookSchema, StepSchema, WorkflowRunSchema, } from '@workflow/world';
|
|
4
|
+
import { monotonicFactory } from 'ulid';
|
|
5
|
+
import { DEFAULT_RESOLVE_DATA_OPTION } from './config.js';
|
|
6
|
+
import { deleteJSON, listJSONFiles, paginatedFileSystemQuery, readJSON, ulidToDate, writeJSON, } from './fs.js';
|
|
7
|
+
// Create a monotonic ULID factory that ensures ULIDs are always increasing
|
|
8
|
+
// even when generated within the same millisecond
|
|
9
|
+
const monotonicUlid = monotonicFactory(() => Math.random());
|
|
10
|
+
// Helper functions to filter data based on resolveData setting
|
|
11
|
+
function filterRunData(run, resolveData) {
|
|
12
|
+
if (resolveData === 'none') {
|
|
13
|
+
return {
|
|
14
|
+
...run,
|
|
15
|
+
input: [],
|
|
16
|
+
output: undefined,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
return run;
|
|
20
|
+
}
|
|
21
|
+
function filterStepData(step, resolveData) {
|
|
22
|
+
if (resolveData === 'none') {
|
|
23
|
+
return {
|
|
24
|
+
...step,
|
|
25
|
+
input: [],
|
|
26
|
+
output: undefined,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
return step;
|
|
30
|
+
}
|
|
31
|
+
function filterEventData(event, resolveData) {
|
|
32
|
+
if (resolveData === 'none') {
|
|
33
|
+
const { eventData: _eventData, ...rest } = event;
|
|
34
|
+
return rest;
|
|
35
|
+
}
|
|
36
|
+
return event;
|
|
37
|
+
}
|
|
38
|
+
function filterHookData(hook, resolveData) {
|
|
39
|
+
if (resolveData === 'none') {
|
|
40
|
+
const { metadata: _metadata, ...rest } = hook;
|
|
41
|
+
return rest;
|
|
42
|
+
}
|
|
43
|
+
return hook;
|
|
44
|
+
}
|
|
45
|
+
const getObjectCreatedAt = (idPrefix) => (filename) => {
|
|
46
|
+
const replaceRegex = new RegExp(`^${idPrefix}_`, 'g');
|
|
47
|
+
const dashIndex = filename.indexOf('-');
|
|
48
|
+
if (dashIndex === -1) {
|
|
49
|
+
// No dash - extract ULID from the filename (e.g., wrun_ULID.json, evnt_ULID.json)
|
|
50
|
+
const ulid = filename.replace(/\.json$/, '').replace(replaceRegex, '');
|
|
51
|
+
return ulidToDate(ulid);
|
|
52
|
+
}
|
|
53
|
+
// For composite keys like {runId}-{stepId}, extract from the appropriate part
|
|
54
|
+
if (idPrefix === 'step') {
|
|
55
|
+
// For steps: wrun_ULID-step_123.json - extract from the runId part
|
|
56
|
+
const runId = filename.substring(0, dashIndex);
|
|
57
|
+
const ulid = runId.replace(/^wrun_/, '');
|
|
58
|
+
return ulidToDate(ulid);
|
|
59
|
+
}
|
|
60
|
+
// For events: wrun_ULID-evnt_ULID.json - extract from the eventId part
|
|
61
|
+
const id = filename.substring(dashIndex + 1).replace(/\.json$/, '');
|
|
62
|
+
const ulid = id.replace(replaceRegex, '');
|
|
63
|
+
return ulidToDate(ulid);
|
|
64
|
+
};
|
|
65
|
+
export function createStorage(basedir) {
|
|
66
|
+
return {
|
|
67
|
+
runs: {
|
|
68
|
+
async create(data) {
|
|
69
|
+
const runId = `wrun_${monotonicUlid()}`;
|
|
70
|
+
const now = new Date();
|
|
71
|
+
const result = {
|
|
72
|
+
runId,
|
|
73
|
+
deploymentId: data.deploymentId,
|
|
74
|
+
status: 'pending',
|
|
75
|
+
workflowName: data.workflowName,
|
|
76
|
+
executionContext: data.executionContext,
|
|
77
|
+
input: data.input || [],
|
|
78
|
+
output: undefined,
|
|
79
|
+
error: undefined,
|
|
80
|
+
errorCode: undefined,
|
|
81
|
+
startedAt: undefined,
|
|
82
|
+
completedAt: undefined,
|
|
83
|
+
createdAt: now,
|
|
84
|
+
updatedAt: now,
|
|
85
|
+
};
|
|
86
|
+
const runPath = path.join(basedir, 'runs', `${runId}.json`);
|
|
87
|
+
await writeJSON(runPath, result);
|
|
88
|
+
return result;
|
|
89
|
+
},
|
|
90
|
+
async get(id, params) {
|
|
91
|
+
const runPath = path.join(basedir, 'runs', `${id}.json`);
|
|
92
|
+
const run = await readJSON(runPath, WorkflowRunSchema);
|
|
93
|
+
if (!run) {
|
|
94
|
+
throw new WorkflowRunNotFoundError(id);
|
|
95
|
+
}
|
|
96
|
+
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
97
|
+
return filterRunData(run, resolveData);
|
|
98
|
+
},
|
|
99
|
+
async update(id, data) {
|
|
100
|
+
const runPath = path.join(basedir, 'runs', `${id}.json`);
|
|
101
|
+
const run = await readJSON(runPath, WorkflowRunSchema);
|
|
102
|
+
if (!run) {
|
|
103
|
+
throw new WorkflowRunNotFoundError(id);
|
|
104
|
+
}
|
|
105
|
+
const now = new Date();
|
|
106
|
+
const updatedRun = {
|
|
107
|
+
...run,
|
|
108
|
+
...data,
|
|
109
|
+
updatedAt: now,
|
|
110
|
+
};
|
|
111
|
+
// Only set startedAt the first time the run transitions to 'running'
|
|
112
|
+
if (data.status === 'running' && !updatedRun.startedAt) {
|
|
113
|
+
updatedRun.startedAt = now;
|
|
114
|
+
}
|
|
115
|
+
if (data.status === 'completed' ||
|
|
116
|
+
data.status === 'failed' ||
|
|
117
|
+
data.status === 'cancelled') {
|
|
118
|
+
updatedRun.completedAt = now;
|
|
119
|
+
}
|
|
120
|
+
await writeJSON(runPath, updatedRun, { overwrite: true });
|
|
121
|
+
return updatedRun;
|
|
122
|
+
},
|
|
123
|
+
async list(params) {
|
|
124
|
+
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
125
|
+
const result = await paginatedFileSystemQuery({
|
|
126
|
+
directory: path.join(basedir, 'runs'),
|
|
127
|
+
schema: WorkflowRunSchema,
|
|
128
|
+
filter: params?.workflowName
|
|
129
|
+
? (run) => run.workflowName === params.workflowName
|
|
130
|
+
: undefined,
|
|
131
|
+
sortOrder: params?.pagination?.sortOrder ?? 'desc',
|
|
132
|
+
limit: params?.pagination?.limit,
|
|
133
|
+
cursor: params?.pagination?.cursor,
|
|
134
|
+
getCreatedAt: getObjectCreatedAt('wrun'),
|
|
135
|
+
getId: (run) => run.runId,
|
|
136
|
+
});
|
|
137
|
+
// If resolveData is "none", replace input/output with empty data
|
|
138
|
+
if (resolveData === 'none') {
|
|
139
|
+
return {
|
|
140
|
+
...result,
|
|
141
|
+
data: result.data.map((run) => ({
|
|
142
|
+
...run,
|
|
143
|
+
input: [],
|
|
144
|
+
output: undefined,
|
|
145
|
+
})),
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
return result;
|
|
149
|
+
},
|
|
150
|
+
async cancel(id, params) {
|
|
151
|
+
const run = await this.update(id, { status: 'cancelled' });
|
|
152
|
+
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
153
|
+
return filterRunData(run, resolveData);
|
|
154
|
+
},
|
|
155
|
+
async pause(id, params) {
|
|
156
|
+
const run = await this.update(id, { status: 'paused' });
|
|
157
|
+
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
158
|
+
return filterRunData(run, resolveData);
|
|
159
|
+
},
|
|
160
|
+
async resume(id, params) {
|
|
161
|
+
const run = await this.update(id, { status: 'running' });
|
|
162
|
+
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
163
|
+
return filterRunData(run, resolveData);
|
|
164
|
+
},
|
|
165
|
+
},
|
|
166
|
+
steps: {
|
|
167
|
+
async create(runId, data) {
|
|
168
|
+
const now = new Date();
|
|
169
|
+
const result = {
|
|
170
|
+
runId,
|
|
171
|
+
stepId: data.stepId,
|
|
172
|
+
stepName: data.stepName,
|
|
173
|
+
status: 'pending',
|
|
174
|
+
input: data.input,
|
|
175
|
+
output: undefined,
|
|
176
|
+
error: undefined,
|
|
177
|
+
errorCode: undefined,
|
|
178
|
+
attempt: 0,
|
|
179
|
+
startedAt: undefined,
|
|
180
|
+
completedAt: undefined,
|
|
181
|
+
createdAt: now,
|
|
182
|
+
updatedAt: now,
|
|
183
|
+
};
|
|
184
|
+
const compositeKey = `${runId}-${data.stepId}`;
|
|
185
|
+
const stepPath = path.join(basedir, 'steps', `${compositeKey}.json`);
|
|
186
|
+
await writeJSON(stepPath, result);
|
|
187
|
+
return result;
|
|
188
|
+
},
|
|
189
|
+
async get(runId, stepId, params) {
|
|
190
|
+
if (!runId) {
|
|
191
|
+
const fileIds = await listJSONFiles(path.join(basedir, 'steps'));
|
|
192
|
+
const fileId = fileIds.find((fileId) => fileId.endsWith(`-${stepId}`));
|
|
193
|
+
if (!fileId) {
|
|
194
|
+
throw new Error(`Step ${stepId} not found`);
|
|
195
|
+
}
|
|
196
|
+
runId = fileId.split('-')[0];
|
|
197
|
+
}
|
|
198
|
+
const compositeKey = `${runId}-${stepId}`;
|
|
199
|
+
const stepPath = path.join(basedir, 'steps', `${compositeKey}.json`);
|
|
200
|
+
const step = await readJSON(stepPath, StepSchema);
|
|
201
|
+
if (!step) {
|
|
202
|
+
throw new Error(`Step ${stepId} in run ${runId} not found`);
|
|
203
|
+
}
|
|
204
|
+
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
205
|
+
return filterStepData(step, resolveData);
|
|
206
|
+
},
|
|
207
|
+
async update(runId, stepId, data) {
|
|
208
|
+
const compositeKey = `${runId}-${stepId}`;
|
|
209
|
+
const stepPath = path.join(basedir, 'steps', `${compositeKey}.json`);
|
|
210
|
+
const step = await readJSON(stepPath, StepSchema);
|
|
211
|
+
if (!step) {
|
|
212
|
+
throw new Error(`Step ${stepId} in run ${runId} not found`);
|
|
213
|
+
}
|
|
214
|
+
const now = new Date();
|
|
215
|
+
const updatedStep = {
|
|
216
|
+
...step,
|
|
217
|
+
...data,
|
|
218
|
+
updatedAt: now,
|
|
219
|
+
};
|
|
220
|
+
// Only set startedAt the first time the step transitions to 'running'
|
|
221
|
+
if (data.status === 'running' && !updatedStep.startedAt) {
|
|
222
|
+
updatedStep.startedAt = now;
|
|
223
|
+
}
|
|
224
|
+
if (data.status === 'completed' || data.status === 'failed') {
|
|
225
|
+
updatedStep.completedAt = now;
|
|
226
|
+
}
|
|
227
|
+
await writeJSON(stepPath, updatedStep, { overwrite: true });
|
|
228
|
+
return updatedStep;
|
|
229
|
+
},
|
|
230
|
+
async list(params) {
|
|
231
|
+
const resolveData = params.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
232
|
+
const result = await paginatedFileSystemQuery({
|
|
233
|
+
directory: path.join(basedir, 'steps'),
|
|
234
|
+
schema: StepSchema,
|
|
235
|
+
filePrefix: `${params.runId}-`,
|
|
236
|
+
sortOrder: params.pagination?.sortOrder ?? 'desc',
|
|
237
|
+
limit: params.pagination?.limit,
|
|
238
|
+
cursor: params.pagination?.cursor,
|
|
239
|
+
getCreatedAt: getObjectCreatedAt('step'),
|
|
240
|
+
getId: (step) => step.stepId,
|
|
241
|
+
});
|
|
242
|
+
// If resolveData is "none", replace input/output with empty data
|
|
243
|
+
if (resolveData === 'none') {
|
|
244
|
+
return {
|
|
245
|
+
...result,
|
|
246
|
+
data: result.data.map((step) => ({
|
|
247
|
+
...step,
|
|
248
|
+
input: [],
|
|
249
|
+
output: undefined,
|
|
250
|
+
})),
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
return result;
|
|
254
|
+
},
|
|
255
|
+
},
|
|
256
|
+
// Events - filesystem-backed storage
|
|
257
|
+
events: {
|
|
258
|
+
async create(runId, data, params) {
|
|
259
|
+
const eventId = `evnt_${monotonicUlid()}`;
|
|
260
|
+
const now = new Date();
|
|
261
|
+
const result = {
|
|
262
|
+
...data,
|
|
263
|
+
runId,
|
|
264
|
+
eventId,
|
|
265
|
+
createdAt: now,
|
|
266
|
+
};
|
|
267
|
+
// Store event using composite key {runId}-{eventId}
|
|
268
|
+
const compositeKey = `${runId}-${eventId}`;
|
|
269
|
+
const eventPath = path.join(basedir, 'events', `${compositeKey}.json`);
|
|
270
|
+
await writeJSON(eventPath, result);
|
|
271
|
+
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
272
|
+
return filterEventData(result, resolveData);
|
|
273
|
+
},
|
|
274
|
+
async list(params) {
|
|
275
|
+
const { runId } = params;
|
|
276
|
+
const resolveData = params.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
277
|
+
const result = await paginatedFileSystemQuery({
|
|
278
|
+
directory: path.join(basedir, 'events'),
|
|
279
|
+
schema: EventSchema,
|
|
280
|
+
filePrefix: `${runId}-`,
|
|
281
|
+
// Events in chronological order (oldest first) by default,
|
|
282
|
+
// different from the default for other list calls.
|
|
283
|
+
sortOrder: params.pagination?.sortOrder ?? 'asc',
|
|
284
|
+
limit: params.pagination?.limit,
|
|
285
|
+
cursor: params.pagination?.cursor,
|
|
286
|
+
getCreatedAt: getObjectCreatedAt('evnt'),
|
|
287
|
+
getId: (event) => event.eventId,
|
|
288
|
+
});
|
|
289
|
+
// If resolveData is "none", remove eventData from events
|
|
290
|
+
if (resolveData === 'none') {
|
|
291
|
+
return {
|
|
292
|
+
...result,
|
|
293
|
+
data: result.data.map((event) => {
|
|
294
|
+
const { eventData: _eventData, ...rest } = event;
|
|
295
|
+
return rest;
|
|
296
|
+
}),
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
return result;
|
|
300
|
+
},
|
|
301
|
+
async listByCorrelationId(params) {
|
|
302
|
+
const correlationId = params.correlationId;
|
|
303
|
+
const resolveData = params.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
304
|
+
const result = await paginatedFileSystemQuery({
|
|
305
|
+
directory: path.join(basedir, 'events'),
|
|
306
|
+
schema: EventSchema,
|
|
307
|
+
// No filePrefix - search all events
|
|
308
|
+
filter: (event) => event.correlationId === correlationId,
|
|
309
|
+
// Events in chronological order (oldest first) by default,
|
|
310
|
+
// different from the default for other list calls.
|
|
311
|
+
sortOrder: params.pagination?.sortOrder ?? 'asc',
|
|
312
|
+
limit: params.pagination?.limit,
|
|
313
|
+
cursor: params.pagination?.cursor,
|
|
314
|
+
getCreatedAt: getObjectCreatedAt('evnt'),
|
|
315
|
+
getId: (event) => event.eventId,
|
|
316
|
+
});
|
|
317
|
+
// If resolveData is "none", remove eventData from events
|
|
318
|
+
if (resolveData === 'none') {
|
|
319
|
+
return {
|
|
320
|
+
...result,
|
|
321
|
+
data: result.data.map((event) => {
|
|
322
|
+
const { eventData: _eventData, ...rest } = event;
|
|
323
|
+
return rest;
|
|
324
|
+
}),
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
return result;
|
|
328
|
+
},
|
|
329
|
+
},
|
|
330
|
+
// Hooks
|
|
331
|
+
hooks: {
|
|
332
|
+
async create(runId, data) {
|
|
333
|
+
const now = new Date();
|
|
334
|
+
const result = {
|
|
335
|
+
runId,
|
|
336
|
+
hookId: data.hookId,
|
|
337
|
+
token: data.token,
|
|
338
|
+
metadata: data.metadata,
|
|
339
|
+
ownerId: 'embedded-owner',
|
|
340
|
+
projectId: 'embedded-project',
|
|
341
|
+
environment: 'embedded',
|
|
342
|
+
createdAt: now,
|
|
343
|
+
};
|
|
344
|
+
const hookPath = path.join(basedir, 'hooks', `${data.hookId}.json`);
|
|
345
|
+
await writeJSON(hookPath, result);
|
|
346
|
+
return result;
|
|
347
|
+
},
|
|
348
|
+
async get(hookId, params) {
|
|
349
|
+
const hookPath = path.join(basedir, 'hooks', `${hookId}.json`);
|
|
350
|
+
// Try webhook schema first (which includes response), fall back to HookSchema
|
|
351
|
+
const hook = await readJSON(hookPath, HookSchema);
|
|
352
|
+
if (!hook) {
|
|
353
|
+
throw new Error(`Hook ${hookId} not found`);
|
|
354
|
+
}
|
|
355
|
+
const resolveData = params?.resolveData || DEFAULT_RESOLVE_DATA_OPTION;
|
|
356
|
+
return filterHookData(hook, resolveData);
|
|
357
|
+
},
|
|
358
|
+
async getByToken(token) {
|
|
359
|
+
// Need to search through all hooks to find one with matching token
|
|
360
|
+
const hooksDir = path.join(basedir, 'hooks');
|
|
361
|
+
const files = await listJSONFiles(hooksDir);
|
|
362
|
+
for (const file of files) {
|
|
363
|
+
const hookPath = path.join(hooksDir, `${file}.json`);
|
|
364
|
+
const hook = await readJSON(hookPath, HookSchema);
|
|
365
|
+
if (hook && hook.token === token) {
|
|
366
|
+
return hook;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
throw new Error(`Hook with token ${token} not found`);
|
|
370
|
+
},
|
|
371
|
+
async list(params) {
|
|
372
|
+
const hooksDir = path.join(basedir, 'hooks');
|
|
373
|
+
const resolveData = params.resolveData || DEFAULT_RESOLVE_DATA_OPTION;
|
|
374
|
+
const result = await paginatedFileSystemQuery({
|
|
375
|
+
directory: hooksDir,
|
|
376
|
+
schema: HookSchema,
|
|
377
|
+
sortOrder: params.pagination?.sortOrder,
|
|
378
|
+
limit: params.pagination?.limit,
|
|
379
|
+
cursor: params.pagination?.cursor,
|
|
380
|
+
filePrefix: undefined, // Hooks don't have ULIDs, so we can't optimize by filename
|
|
381
|
+
filter: (hook) => {
|
|
382
|
+
// Filter by runId if provided
|
|
383
|
+
if (params.runId && hook.runId !== params.runId) {
|
|
384
|
+
return false;
|
|
385
|
+
}
|
|
386
|
+
return true;
|
|
387
|
+
},
|
|
388
|
+
getCreatedAt: () => {
|
|
389
|
+
// Hook files don't have ULID timestamps in filename
|
|
390
|
+
// We need to read the file to get createdAt, but that's inefficient
|
|
391
|
+
// So we return the hook's createdAt directly (item.createdAt will be used for sorting)
|
|
392
|
+
// Return a dummy date to pass the null check, actual sorting uses item.createdAt
|
|
393
|
+
return new Date(0);
|
|
394
|
+
},
|
|
395
|
+
getId: (hook) => hook.hookId,
|
|
396
|
+
});
|
|
397
|
+
// Transform the data after pagination
|
|
398
|
+
return {
|
|
399
|
+
...result,
|
|
400
|
+
data: result.data.map((hook) => filterHookData(hook, resolveData)),
|
|
401
|
+
};
|
|
402
|
+
},
|
|
403
|
+
async dispose(hookId) {
|
|
404
|
+
const hookPath = path.join(basedir, 'hooks', `${hookId}.json`);
|
|
405
|
+
const hook = await readJSON(hookPath, HookSchema);
|
|
406
|
+
if (!hook) {
|
|
407
|
+
throw new Error(`Hook ${hookId} not found`);
|
|
408
|
+
}
|
|
409
|
+
await deleteJSON(hookPath);
|
|
410
|
+
return hook;
|
|
411
|
+
},
|
|
412
|
+
},
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
//# sourceMappingURL=storage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage.js","sourceRoot":"","sources":["../src/storage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EAEL,WAAW,EAEX,UAAU,EAEV,UAAU,EAGV,iBAAiB,GAClB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,gBAAgB,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,EAAE,2BAA2B,EAAE,MAAM,aAAa,CAAC;AAC1D,OAAO,EACL,UAAU,EACV,aAAa,EACb,wBAAwB,EACxB,QAAQ,EACR,UAAU,EACV,SAAS,GACV,MAAM,SAAS,CAAC;AAEjB,2EAA2E;AAC3E,kDAAkD;AAClD,MAAM,aAAa,GAAG,gBAAgB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAE5D,+DAA+D;AAC/D,SAAS,aAAa,CACpB,GAAgB,EAChB,WAA2B;IAE3B,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;QAC3B,OAAO;YACL,GAAG,GAAG;YACN,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,SAAS;SAClB,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,cAAc,CAAC,IAAU,EAAE,WAA2B;IAC7D,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;QAC3B,OAAO;YACL,GAAG,IAAI;YACP,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,SAAS;SAClB,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,eAAe,CAAC,KAAY,EAAE,WAA2B;IAChE,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;QAC3B,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,GAAG,KAAY,CAAC;QACxD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,IAAU,EAAE,WAA2B;IAC7D,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;QAC3B,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,IAAW,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,kBAAkB,GACtB,CAAC,QAAgB,EAAE,EAAE,CACrB,CAAC,QAAgB,EAAe,EAAE;IAChC,MAAM,YAAY,GAAG,IAAI,MAAM,CAAC,IAAI,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAExC,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;QACrB,kFAAkF;QAClF,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QACvE,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,8EAA8E;IAC9E,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QACxB,mEAAmE;QACnE,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACzC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,uEAAuE;IACvE,MAAM,EAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAC1C,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC,CAAC;AAEJ,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,OAAO;QACL,IAAI,EAAE;YACJ,KAAK,CAAC,MAAM,CAAC,IAAI;gBACf,MAAM,KAAK,GAAG,QAAQ,aAAa,EAAE,EAAE,CAAC;gBACxC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBAEvB,MAAM,MAAM,GAAgB;oBAC1B,KAAK;oBACL,YAAY,EAAE,IAAI,CAAC,YAAY;oBAC/B,MAAM,EAAE,SAAS;oBACjB,YAAY,EAAE,IAAI,CAAC,YAAY;oBAC/B,gBAAgB,EAAE,IAAI,CAAC,gBAEV;oBACb,KAAK,EAAG,IAAI,CAAC,KAAe,IAAI,EAAE;oBAClC,MAAM,EAAE,SAAS;oBACjB,KAAK,EAAE,SAAS;oBAChB,SAAS,EAAE,SAAS;oBACpB,SAAS,EAAE,SAAS;oBACpB,WAAW,EAAE,SAAS;oBACtB,SAAS,EAAE,GAAG;oBACd,SAAS,EAAE,GAAG;iBACf,CAAC;gBAEF,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC;gBAC5D,MAAM,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBACjC,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM;gBAClB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;gBACzD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;gBACvD,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,MAAM,IAAI,wBAAwB,CAAC,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,EAAE,WAAW,IAAI,2BAA2B,CAAC;gBACvE,OAAO,aAAa,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YACzC,CAAC;YAED,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI;gBACnB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;gBACzD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;gBACvD,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,MAAM,IAAI,wBAAwB,CAAC,EAAE,CAAC,CAAC;gBACzC,CAAC;gBAED,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,UAAU,GAAgB;oBAC9B,GAAG,GAAG;oBACN,GAAG,IAAI;oBACP,SAAS,EAAE,GAAG;iBACf,CAAC;gBAEF,qEAAqE;gBACrE,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;oBACvD,UAAU,CAAC,SAAS,GAAG,GAAG,CAAC;gBAC7B,CAAC;gBACD,IACE,IAAI,CAAC,MAAM,KAAK,WAAW;oBAC3B,IAAI,CAAC,MAAM,KAAK,QAAQ;oBACxB,IAAI,CAAC,MAAM,KAAK,WAAW,EAC3B,CAAC;oBACD,UAAU,CAAC,WAAW,GAAG,GAAG,CAAC;gBAC/B,CAAC;gBAED,MAAM,SAAS,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1D,OAAO,UAAU,CAAC;YACpB,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,MAAM;gBACf,MAAM,WAAW,GAAG,MAAM,EAAE,WAAW,IAAI,2BAA2B,CAAC;gBACvE,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC;oBAC5C,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;oBACrC,MAAM,EAAE,iBAAiB;oBACzB,MAAM,EAAE,MAAM,EAAE,YAAY;wBAC1B,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,KAAK,MAAM,CAAC,YAAY;wBACnD,CAAC,CAAC,SAAS;oBACb,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,IAAI,MAAM;oBAClD,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK;oBAChC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;oBAClC,YAAY,EAAE,kBAAkB,CAAC,MAAM,CAAC;oBACxC,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK;iBAC1B,CAAC,CAAC;gBAEH,iEAAiE;gBACjE,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;oBAC3B,OAAO;wBACL,GAAG,MAAM;wBACT,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;4BAC9B,GAAG,GAAG;4BACN,KAAK,EAAE,EAAE;4BACT,MAAM,EAAE,SAAS;yBAClB,CAAC,CAAC;qBACJ,CAAC;gBACJ,CAAC;gBAED,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM;gBACrB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;gBAC3D,MAAM,WAAW,GAAG,MAAM,EAAE,WAAW,IAAI,2BAA2B,CAAC;gBACvE,OAAO,aAAa,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YACzC,CAAC;YAED,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM;gBACpB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACxD,MAAM,WAAW,GAAG,MAAM,EAAE,WAAW,IAAI,2BAA2B,CAAC;gBACvE,OAAO,aAAa,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YACzC,CAAC;YAED,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM;gBACrB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBACzD,MAAM,WAAW,GAAG,MAAM,EAAE,WAAW,IAAI,2BAA2B,CAAC;gBACvE,OAAO,aAAa,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YACzC,CAAC;SACF;QAED,KAAK,EAAE;YACL,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI;gBACtB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBAEvB,MAAM,MAAM,GAAS;oBACnB,KAAK;oBACL,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,MAAM,EAAE,SAAS;oBACjB,KAAK,EAAE,IAAI,CAAC,KAAc;oBAC1B,MAAM,EAAE,SAAS;oBACjB,KAAK,EAAE,SAAS;oBAChB,SAAS,EAAE,SAAS;oBACpB,OAAO,EAAE,CAAC;oBACV,SAAS,EAAE,SAAS;oBACpB,WAAW,EAAE,SAAS;oBACtB,SAAS,EAAE,GAAG;oBACd,SAAS,EAAE,GAAG;iBACf,CAAC;gBAEF,MAAM,YAAY,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,YAAY,OAAO,CAAC,CAAC;gBACrE,MAAM,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBAElC,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,KAAK,CAAC,GAAG,CACP,KAAyB,EACzB,MAAc,EACd,MAAM;gBAEN,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;oBACjE,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CACrC,MAAM,CAAC,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC,CAC9B,CAAC;oBACF,IAAI,CAAC,MAAM,EAAE,CAAC;wBACZ,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,YAAY,CAAC,CAAC;oBAC9C,CAAC;oBACD,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/B,CAAC;gBACD,MAAM,YAAY,GAAG,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,YAAY,OAAO,CAAC,CAAC;gBACrE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAClD,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,WAAW,KAAK,YAAY,CAAC,CAAC;gBAC9D,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,EAAE,WAAW,IAAI,2BAA2B,CAAC;gBACvE,OAAO,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAC3C,CAAC;YAED,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI;gBAC9B,MAAM,YAAY,GAAG,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,YAAY,OAAO,CAAC,CAAC;gBACrE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAClD,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,WAAW,KAAK,YAAY,CAAC,CAAC;gBAC9D,CAAC;gBAED,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,WAAW,GAAS;oBACxB,GAAG,IAAI;oBACP,GAAG,IAAI;oBACP,SAAS,EAAE,GAAG;iBACf,CAAC;gBAEF,sEAAsE;gBACtE,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;oBACxD,WAAW,CAAC,SAAS,GAAG,GAAG,CAAC;gBAC9B,CAAC;gBACD,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;oBAC5D,WAAW,CAAC,WAAW,GAAG,GAAG,CAAC;gBAChC,CAAC;gBAED,MAAM,SAAS,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC5D,OAAO,WAAW,CAAC;YACrB,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,MAAM;gBACf,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,2BAA2B,CAAC;gBACtE,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC;oBAC5C,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;oBACtC,MAAM,EAAE,UAAU;oBAClB,UAAU,EAAE,GAAG,MAAM,CAAC,KAAK,GAAG;oBAC9B,SAAS,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,IAAI,MAAM;oBACjD,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK;oBAC/B,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM;oBACjC,YAAY,EAAE,kBAAkB,CAAC,MAAM,CAAC;oBACxC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM;iBAC7B,CAAC,CAAC;gBAEH,iEAAiE;gBACjE,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;oBAC3B,OAAO;wBACL,GAAG,MAAM;wBACT,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;4BAC/B,GAAG,IAAI;4BACP,KAAK,EAAE,EAAE;4BACT,MAAM,EAAE,SAAS;yBAClB,CAAC,CAAC;qBACJ,CAAC;gBACJ,CAAC;gBAED,OAAO,MAAM,CAAC;YAChB,CAAC;SACF;QAED,qCAAqC;QACrC,MAAM,EAAE;YACN,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM;gBAC9B,MAAM,OAAO,GAAG,QAAQ,aAAa,EAAE,EAAE,CAAC;gBAC1C,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBAEvB,MAAM,MAAM,GAAU;oBACpB,GAAG,IAAI;oBACP,KAAK;oBACL,OAAO;oBACP,SAAS,EAAE,GAAG;iBACf,CAAC;gBAEF,oDAAoD;gBACpD,MAAM,YAAY,GAAG,GAAG,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,YAAY,OAAO,CAAC,CAAC;gBACvE,MAAM,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBAEnC,MAAM,WAAW,GAAG,MAAM,EAAE,WAAW,IAAI,2BAA2B,CAAC;gBACvE,OAAO,eAAe,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAC9C,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,MAAM;gBACf,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;gBACzB,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,2BAA2B,CAAC;gBACtE,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC;oBAC5C,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;oBACvC,MAAM,EAAE,WAAW;oBACnB,UAAU,EAAE,GAAG,KAAK,GAAG;oBACvB,2DAA2D;oBAC3D,mDAAmD;oBACnD,SAAS,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,IAAI,KAAK;oBAChD,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK;oBAC/B,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM;oBACjC,YAAY,EAAE,kBAAkB,CAAC,MAAM,CAAC;oBACxC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO;iBAChC,CAAC,CAAC;gBAEH,yDAAyD;gBACzD,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;oBAC3B,OAAO;wBACL,GAAG,MAAM;wBACT,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;4BAC9B,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,GAAG,KAAY,CAAC;4BACxD,OAAO,IAAI,CAAC;wBACd,CAAC,CAAC;qBACH,CAAC;gBACJ,CAAC;gBAED,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,KAAK,CAAC,mBAAmB,CAAC,MAAM;gBAC9B,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;gBAC3C,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,2BAA2B,CAAC;gBACtE,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC;oBAC5C,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;oBACvC,MAAM,EAAE,WAAW;oBACnB,oCAAoC;oBACpC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,aAAa;oBACxD,2DAA2D;oBAC3D,mDAAmD;oBACnD,SAAS,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,IAAI,KAAK;oBAChD,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK;oBAC/B,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM;oBACjC,YAAY,EAAE,kBAAkB,CAAC,MAAM,CAAC;oBACxC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO;iBAChC,CAAC,CAAC;gBAEH,yDAAyD;gBACzD,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;oBAC3B,OAAO;wBACL,GAAG,MAAM;wBACT,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;4BAC9B,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,GAAG,KAAY,CAAC;4BACxD,OAAO,IAAI,CAAC;wBACd,CAAC,CAAC;qBACH,CAAC;gBACJ,CAAC;gBAED,OAAO,MAAM,CAAC;YAChB,CAAC;SACF;QAED,QAAQ;QACR,KAAK,EAAE;YACL,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI;gBACtB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBAEvB,MAAM,MAAM,GAAG;oBACb,KAAK;oBACL,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,OAAO,EAAE,gBAAgB;oBACzB,SAAS,EAAE,kBAAkB;oBAC7B,WAAW,EAAE,UAAU;oBACvB,SAAS,EAAE,GAAG;iBACP,CAAC;gBAEV,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,OAAO,CAAC,CAAC;gBACpE,MAAM,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBAClC,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM;gBACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;gBAC/D,8EAA8E;gBAC9E,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAClD,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,YAAY,CAAC,CAAC;gBAC9C,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,EAAE,WAAW,IAAI,2BAA2B,CAAC;gBACvE,OAAO,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAC3C,CAAC;YAED,KAAK,CAAC,UAAU,CAAC,KAAK;gBACpB,mEAAmE;gBACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBAC7C,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;gBAE5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC;oBACrD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;oBAClD,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;wBACjC,OAAO,IAAI,CAAC;oBACd,CAAC;gBACH,CAAC;gBAED,MAAM,IAAI,KAAK,CAAC,mBAAmB,KAAK,YAAY,CAAC,CAAC;YACxD,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,MAAM;gBACf,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBAC7C,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,2BAA2B,CAAC;gBAEtE,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC;oBAC5C,SAAS,EAAE,QAAQ;oBACnB,MAAM,EAAE,UAAU;oBAClB,SAAS,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS;oBACvC,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK;oBAC/B,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM;oBACjC,UAAU,EAAE,SAAS,EAAE,2DAA2D;oBAClF,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;wBACf,8BAA8B;wBAC9B,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC;4BAChD,OAAO,KAAK,CAAC;wBACf,CAAC;wBACD,OAAO,IAAI,CAAC;oBACd,CAAC;oBACD,YAAY,EAAE,GAAG,EAAE;wBACjB,oDAAoD;wBACpD,oEAAoE;wBACpE,uFAAuF;wBACvF,iFAAiF;wBACjF,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;oBACrB,CAAC;oBACD,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM;iBAC7B,CAAC,CAAC;gBAEH,sCAAsC;gBACtC,OAAO;oBACL,GAAG,MAAM;oBACT,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;iBACnE,CAAC;YACJ,CAAC;YAED,KAAK,CAAC,OAAO,CAAC,MAAM;gBAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;gBAC/D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAClD,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,YAAY,CAAC,CAAC;gBAC9C,CAAC;gBACD,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAC3B,OAAO,IAAI,CAAC;YACd,CAAC;SACF;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Streamer } from '@workflow/world';
|
|
2
|
+
/**
|
|
3
|
+
* A chunk consists of a boolean `eof` indicating if it's the last chunk,
|
|
4
|
+
* and a `chunk` which is a Buffer of data.
|
|
5
|
+
* The serialized format is:
|
|
6
|
+
* - 1 byte for `eof` (0 or 1)
|
|
7
|
+
* - and the rest is the chunk data.
|
|
8
|
+
*/
|
|
9
|
+
export interface Chunk {
|
|
10
|
+
eof: boolean;
|
|
11
|
+
chunk: Buffer;
|
|
12
|
+
}
|
|
13
|
+
export declare function serializeChunk(chunk: Chunk): Buffer<ArrayBuffer>;
|
|
14
|
+
export declare function deserializeChunk(serialized: Buffer): {
|
|
15
|
+
eof: boolean;
|
|
16
|
+
chunk: Buffer<ArrayBufferLike>;
|
|
17
|
+
};
|
|
18
|
+
export declare function createStreamer(basedir: string): Streamer;
|
|
19
|
+
//# sourceMappingURL=streamer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streamer.d.ts","sourceRoot":"","sources":["../src/streamer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAQhD;;;;;;GAMG;AACH,MAAM,WAAW,KAAK;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,KAAK,uBAG1C;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM;;;EAIlD;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,CAyKxD"}
|
package/dist/streamer.js
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { EventEmitter } from 'node:events';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { monotonicFactory } from 'ulid';
|
|
4
|
+
import { listJSONFiles, readBuffer, write } from './fs.js';
|
|
5
|
+
// Create a monotonic ULID factory that ensures ULIDs are always increasing
|
|
6
|
+
// even when generated within the same millisecond
|
|
7
|
+
const monotonicUlid = monotonicFactory(() => Math.random());
|
|
8
|
+
export function serializeChunk(chunk) {
|
|
9
|
+
const eofByte = Buffer.from([chunk.eof ? 1 : 0]);
|
|
10
|
+
return Buffer.concat([eofByte, chunk.chunk]);
|
|
11
|
+
}
|
|
12
|
+
export function deserializeChunk(serialized) {
|
|
13
|
+
const eof = serialized[0] === 1;
|
|
14
|
+
const chunk = serialized.subarray(1);
|
|
15
|
+
return { eof, chunk };
|
|
16
|
+
}
|
|
17
|
+
export function createStreamer(basedir) {
|
|
18
|
+
const streamEmitter = new EventEmitter();
|
|
19
|
+
return {
|
|
20
|
+
async writeToStream(name, chunk) {
|
|
21
|
+
const chunkId = `strm_${monotonicUlid()}`;
|
|
22
|
+
if (typeof chunk === 'string') {
|
|
23
|
+
chunk = new TextEncoder().encode(chunk);
|
|
24
|
+
}
|
|
25
|
+
const serialized = serializeChunk({
|
|
26
|
+
chunk: Buffer.from(chunk),
|
|
27
|
+
eof: false,
|
|
28
|
+
});
|
|
29
|
+
const chunkPath = path.join(basedir, 'streams', 'chunks', `${name}-${chunkId}.json`);
|
|
30
|
+
await write(chunkPath, serialized);
|
|
31
|
+
// Emit real-time event
|
|
32
|
+
const chunkData = typeof chunk === 'string'
|
|
33
|
+
? new TextEncoder().encode(chunk)
|
|
34
|
+
: chunk instanceof Buffer
|
|
35
|
+
? new Uint8Array(chunk)
|
|
36
|
+
: chunk;
|
|
37
|
+
streamEmitter.emit(`chunk:${name}`, {
|
|
38
|
+
streamName: name,
|
|
39
|
+
chunkData,
|
|
40
|
+
chunkId,
|
|
41
|
+
});
|
|
42
|
+
},
|
|
43
|
+
async closeStream(name) {
|
|
44
|
+
const chunkId = `strm_${monotonicUlid()}`;
|
|
45
|
+
const chunkPath = path.join(basedir, 'streams', 'chunks', `${name}-${chunkId}.json`);
|
|
46
|
+
await write(chunkPath, serializeChunk({ chunk: Buffer.from([]), eof: true }));
|
|
47
|
+
streamEmitter.emit(`close:${name}`, { streamName: name });
|
|
48
|
+
},
|
|
49
|
+
async readFromStream(name, startIndex = 0) {
|
|
50
|
+
const chunksDir = path.join(basedir, 'streams', 'chunks');
|
|
51
|
+
let removeListeners = () => { };
|
|
52
|
+
return new ReadableStream({
|
|
53
|
+
async start(controller) {
|
|
54
|
+
// Track chunks delivered via events to prevent duplicates and maintain order.
|
|
55
|
+
const deliveredChunkIds = new Set();
|
|
56
|
+
// Buffer for chunks that arrive via events during disk reading
|
|
57
|
+
const bufferedEventChunks = [];
|
|
58
|
+
let isReadingFromDisk = true;
|
|
59
|
+
const chunkListener = (event) => {
|
|
60
|
+
deliveredChunkIds.add(event.chunkId);
|
|
61
|
+
if (isReadingFromDisk) {
|
|
62
|
+
// Buffer chunks that arrive during disk reading to maintain order
|
|
63
|
+
bufferedEventChunks.push({
|
|
64
|
+
chunkId: event.chunkId,
|
|
65
|
+
chunkData: event.chunkData,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
// After disk reading is complete, deliver chunks immediately
|
|
70
|
+
controller.enqueue(event.chunkData);
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
const closeListener = () => {
|
|
74
|
+
// Remove listeners before closing
|
|
75
|
+
streamEmitter.off(`chunk:${name}`, chunkListener);
|
|
76
|
+
streamEmitter.off(`close:${name}`, closeListener);
|
|
77
|
+
controller.close();
|
|
78
|
+
};
|
|
79
|
+
removeListeners = closeListener;
|
|
80
|
+
// Set up listeners FIRST to avoid missing events
|
|
81
|
+
streamEmitter.on(`chunk:${name}`, chunkListener);
|
|
82
|
+
streamEmitter.on(`close:${name}`, closeListener);
|
|
83
|
+
// Now load existing chunks from disk
|
|
84
|
+
const files = await listJSONFiles(chunksDir);
|
|
85
|
+
const chunkFiles = files
|
|
86
|
+
.filter((file) => file.startsWith(`${name}-`))
|
|
87
|
+
.sort(); // ULID lexicographic sort = chronological order
|
|
88
|
+
// Process existing chunks, skipping any already delivered via events
|
|
89
|
+
let isComplete = false;
|
|
90
|
+
for (let i = startIndex; i < chunkFiles.length; i++) {
|
|
91
|
+
const file = chunkFiles[i];
|
|
92
|
+
// Extract chunk ID from filename: "streamName-chunkId"
|
|
93
|
+
const chunkId = file.substring(name.length + 1);
|
|
94
|
+
// Skip if already delivered via event
|
|
95
|
+
if (deliveredChunkIds.has(chunkId)) {
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
const chunk = deserializeChunk(await readBuffer(path.join(chunksDir, `${file}.json`)));
|
|
99
|
+
if (chunk?.eof === true) {
|
|
100
|
+
isComplete = true;
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
if (chunk.chunk.byteLength) {
|
|
104
|
+
controller.enqueue(chunk.chunk);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
// Finished reading from disk - now deliver buffered event chunks in chronological order
|
|
108
|
+
isReadingFromDisk = false;
|
|
109
|
+
// Sort buffered chunks by ULID (chronological order)
|
|
110
|
+
bufferedEventChunks.sort((a, b) => a.chunkId.localeCompare(b.chunkId));
|
|
111
|
+
for (const buffered of bufferedEventChunks) {
|
|
112
|
+
controller.enqueue(buffered.chunkData);
|
|
113
|
+
}
|
|
114
|
+
if (isComplete) {
|
|
115
|
+
removeListeners();
|
|
116
|
+
controller.close();
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
},
|
|
120
|
+
cancel() {
|
|
121
|
+
removeListeners();
|
|
122
|
+
},
|
|
123
|
+
});
|
|
124
|
+
},
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
//# sourceMappingURL=streamer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streamer.js","sourceRoot":"","sources":["../src/streamer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,gBAAgB,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAE3D,2EAA2E;AAC3E,kDAAkD;AAClD,MAAM,aAAa,GAAG,gBAAgB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAc5D,MAAM,UAAU,cAAc,CAAC,KAAY;IACzC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACrC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,MAAM,aAAa,GAAG,IAAI,YAAY,EAalC,CAAC;IAEL,OAAO;QACL,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK;YAC7B,MAAM,OAAO,GAAG,QAAQ,aAAa,EAAE,EAAE,CAAC;YAE1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC1C,CAAC;YACD,MAAM,UAAU,GAAG,cAAc,CAAC;gBAChC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;gBACzB,GAAG,EAAE,KAAK;aACX,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CACzB,OAAO,EACP,SAAS,EACT,QAAQ,EACR,GAAG,IAAI,IAAI,OAAO,OAAO,CAC1B,CAAC;YAEF,MAAM,KAAK,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YAEnC,uBAAuB;YACvB,MAAM,SAAS,GACb,OAAO,KAAK,KAAK,QAAQ;gBACvB,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;gBACjC,CAAC,CAAC,KAAK,YAAY,MAAM;oBACvB,CAAC,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC;oBACvB,CAAC,CAAC,KAAK,CAAC;YAEd,aAAa,CAAC,IAAI,CAAC,SAAS,IAAI,EAAW,EAAE;gBAC3C,UAAU,EAAE,IAAI;gBAChB,SAAS;gBACT,OAAO;aACR,CAAC,CAAC;QACL,CAAC;QAED,KAAK,CAAC,WAAW,CAAC,IAAI;YACpB,MAAM,OAAO,GAAG,QAAQ,aAAa,EAAE,EAAE,CAAC;YAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CACzB,OAAO,EACP,SAAS,EACT,QAAQ,EACR,GAAG,IAAI,IAAI,OAAO,OAAO,CAC1B,CAAC;YAEF,MAAM,KAAK,CACT,SAAS,EACT,cAAc,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CACtD,CAAC;YAEF,aAAa,CAAC,IAAI,CAAC,SAAS,IAAI,EAAW,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,UAAU,GAAG,CAAC;YACvC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;YAC1D,IAAI,eAAe,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;YAE/B,OAAO,IAAI,cAAc,CAAa;gBACpC,KAAK,CAAC,KAAK,CAAC,UAAU;oBACpB,8EAA8E;oBAC9E,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;oBAC5C,+DAA+D;oBAC/D,MAAM,mBAAmB,GAGpB,EAAE,CAAC;oBACR,IAAI,iBAAiB,GAAG,IAAI,CAAC;oBAE7B,MAAM,aAAa,GAAG,CAAC,KAItB,EAAE,EAAE;wBACH,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAErC,IAAI,iBAAiB,EAAE,CAAC;4BACtB,kEAAkE;4BAClE,mBAAmB,CAAC,IAAI,CAAC;gCACvB,OAAO,EAAE,KAAK,CAAC,OAAO;gCACtB,SAAS,EAAE,KAAK,CAAC,SAAS;6BAC3B,CAAC,CAAC;wBACL,CAAC;6BAAM,CAAC;4BACN,6DAA6D;4BAC7D,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;wBACtC,CAAC;oBACH,CAAC,CAAC;oBAEF,MAAM,aAAa,GAAG,GAAG,EAAE;wBACzB,kCAAkC;wBAClC,aAAa,CAAC,GAAG,CAAC,SAAS,IAAI,EAAW,EAAE,aAAa,CAAC,CAAC;wBAC3D,aAAa,CAAC,GAAG,CAAC,SAAS,IAAI,EAAW,EAAE,aAAa,CAAC,CAAC;wBAC3D,UAAU,CAAC,KAAK,EAAE,CAAC;oBACrB,CAAC,CAAC;oBACF,eAAe,GAAG,aAAa,CAAC;oBAEhC,iDAAiD;oBACjD,aAAa,CAAC,EAAE,CAAC,SAAS,IAAI,EAAW,EAAE,aAAa,CAAC,CAAC;oBAC1D,aAAa,CAAC,EAAE,CAAC,SAAS,IAAI,EAAW,EAAE,aAAa,CAAC,CAAC;oBAE1D,qCAAqC;oBACrC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;oBAC7C,MAAM,UAAU,GAAG,KAAK;yBACrB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC;yBAC7C,IAAI,EAAE,CAAC,CAAC,gDAAgD;oBAE3D,qEAAqE;oBACrE,IAAI,UAAU,GAAG,KAAK,CAAC;oBACvB,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;wBACpD,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;wBAC3B,uDAAuD;wBACvD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBAEhD,sCAAsC;wBACtC,IAAI,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;4BACnC,SAAS;wBACX,CAAC;wBAED,MAAM,KAAK,GAAG,gBAAgB,CAC5B,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC,CACvD,CAAC;wBACF,IAAI,KAAK,EAAE,GAAG,KAAK,IAAI,EAAE,CAAC;4BACxB,UAAU,GAAG,IAAI,CAAC;4BAClB,MAAM;wBACR,CAAC;wBACD,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;4BAC3B,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBAClC,CAAC;oBACH,CAAC;oBAED,wFAAwF;oBACxF,iBAAiB,GAAG,KAAK,CAAC;oBAE1B,qDAAqD;oBACrD,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAChC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CACnC,CAAC;oBACF,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;wBAC3C,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;oBACzC,CAAC;oBAED,IAAI,UAAU,EAAE,CAAC;wBACf,eAAe,EAAE,CAAC;wBAClB,UAAU,CAAC,KAAK,EAAE,CAAC;wBACnB,OAAO;oBACT,CAAC;gBACH,CAAC;gBAED,MAAM;oBACJ,eAAe,EAAE,CAAC;gBACpB,CAAC;aACF,CAAC,CAAC;QACL,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/util.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Creates a lazily-evaluated, memoized version of the provided function.
|
|
3
|
+
*
|
|
4
|
+
* The returned object exposes a `value` getter that calls `fn` only once,
|
|
5
|
+
* caches its result, and returns the cached value on subsequent accesses.
|
|
6
|
+
*
|
|
7
|
+
* @typeParam T - The return type of the provided function.
|
|
8
|
+
* @param fn - The function to be called once and whose result will be cached.
|
|
9
|
+
* @returns An object with a `value` property that returns the memoized result of `fn`.
|
|
10
|
+
*/
|
|
11
|
+
export declare function once<T>(fn: () => T): {
|
|
12
|
+
readonly value: T;
|
|
13
|
+
};
|
|
14
|
+
//# sourceMappingURL=util.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,wBAAgB,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC;;EASlC"}
|
package/dist/util.js
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Creates a lazily-evaluated, memoized version of the provided function.
|
|
3
|
+
*
|
|
4
|
+
* The returned object exposes a `value` getter that calls `fn` only once,
|
|
5
|
+
* caches its result, and returns the cached value on subsequent accesses.
|
|
6
|
+
*
|
|
7
|
+
* @typeParam T - The return type of the provided function.
|
|
8
|
+
* @param fn - The function to be called once and whose result will be cached.
|
|
9
|
+
* @returns An object with a `value` property that returns the memoized result of `fn`.
|
|
10
|
+
*/
|
|
11
|
+
export function once(fn) {
|
|
12
|
+
const result = {
|
|
13
|
+
get value() {
|
|
14
|
+
const value = fn();
|
|
15
|
+
Object.defineProperty(result, 'value', { value });
|
|
16
|
+
return value;
|
|
17
|
+
},
|
|
18
|
+
};
|
|
19
|
+
return result;
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=util.js.map
|
package/dist/util.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,MAAM,UAAU,IAAI,CAAI,EAAW;IACjC,MAAM,MAAM,GAAG;QACb,IAAI,KAAK;YACP,MAAM,KAAK,GAAG,EAAE,EAAE,CAAC;YACnB,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAClD,OAAO,KAAK,CAAC;QACf,CAAC;KACF,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@workflow/world-local",
|
|
3
|
+
"version": "4.0.1-beta.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"files": [
|
|
7
|
+
"dist"
|
|
8
|
+
],
|
|
9
|
+
"publishConfig": {
|
|
10
|
+
"access": "public"
|
|
11
|
+
},
|
|
12
|
+
"exports": {
|
|
13
|
+
".": {
|
|
14
|
+
"types": "./dist/index.d.ts",
|
|
15
|
+
"default": "./dist/index.js"
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"dependencies": {
|
|
19
|
+
"@vercel/queue": "0.0.0-alpha.23",
|
|
20
|
+
"ulid": "^3.0.1",
|
|
21
|
+
"zod": "4.1.11",
|
|
22
|
+
"@workflow/world": "4.0.1-beta.0"
|
|
23
|
+
},
|
|
24
|
+
"devDependencies": {
|
|
25
|
+
"@opentelemetry/api": "^1.9.0",
|
|
26
|
+
"@types/ms": "^0.7.34",
|
|
27
|
+
"@types/node": "24.6.2",
|
|
28
|
+
"ms": "2.1.3",
|
|
29
|
+
"vitest": "^3.2.4",
|
|
30
|
+
"@workflow/errors": "4.0.1-beta.0",
|
|
31
|
+
"@workflow/tsconfig": "4.0.1-beta.0"
|
|
32
|
+
},
|
|
33
|
+
"peerDependencies": {
|
|
34
|
+
"@opentelemetry/api": "1"
|
|
35
|
+
},
|
|
36
|
+
"peerDependenciesMeta": {
|
|
37
|
+
"@opentelemetry/api": {
|
|
38
|
+
"optional": true
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
"scripts": {
|
|
42
|
+
"build": "tsc",
|
|
43
|
+
"dev": "tsc --watch",
|
|
44
|
+
"clean": "tsc --build --clean",
|
|
45
|
+
"test": "vitest run src",
|
|
46
|
+
"typecheck": "tsc --noEmit"
|
|
47
|
+
}
|
|
48
|
+
}
|