@ekairos/structure 1.21.54-beta.0 → 1.21.57-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clearDataset.tool.d.ts +0 -1
- package/dist/clearDataset.tool.js +0 -1
- package/dist/completeObject.tool.d.ts +0 -1
- package/dist/completeObject.tool.js +0 -1
- package/dist/completeRows.tool.d.ts +0 -1
- package/dist/completeRows.tool.js +0 -1
- package/dist/dataset/steps.d.ts +0 -1
- package/dist/dataset/steps.js +48 -12
- package/dist/datasetFiles.d.ts +0 -1
- package/dist/datasetFiles.js +0 -1
- package/dist/datasetReader.d.ts +16 -0
- package/dist/datasetReader.js +25 -0
- package/dist/domain.d.ts +0 -1
- package/dist/domain.js +0 -1
- package/dist/executeCommand.tool.d.ts +0 -1
- package/dist/executeCommand.tool.js +0 -1
- package/dist/file/steps.d.ts +0 -1
- package/dist/file/steps.js +3 -4
- package/dist/generateSchema.tool.d.ts +0 -1
- package/dist/generateSchema.tool.js +0 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/prompts.d.ts +0 -1
- package/dist/prompts.js +0 -1
- package/dist/rowsOutputPaging.d.ts +57 -0
- package/dist/rowsOutputPaging.js +148 -0
- package/dist/rowsOutputPaging.steps.d.ts +37 -0
- package/dist/rowsOutputPaging.steps.js +125 -0
- package/dist/rowsPagination.steps.d.ts +59 -0
- package/dist/rowsPagination.steps.js +190 -0
- package/dist/runtime.d.ts +1 -2
- package/dist/runtime.js +1 -2
- package/dist/sandbox/steps.d.ts +0 -1
- package/dist/sandbox/steps.js +10 -7
- package/dist/schema.d.ts +0 -1
- package/dist/schema.js +3 -2
- package/dist/service.d.ts +0 -1
- package/dist/service.js +43 -34
- package/dist/steps/commitFromEvents.step.d.ts +0 -1
- package/dist/steps/commitFromEvents.step.js +2 -3
- package/dist/steps/persistObjectFromStory.step.d.ts +0 -1
- package/dist/steps/persistObjectFromStory.step.js +3 -4
- package/dist/structure.d.ts +29 -4
- package/dist/structure.js +35 -5
- package/package.json +14 -4
- package/dist/clearDataset.tool.d.ts.map +0 -1
- package/dist/clearDataset.tool.js.map +0 -1
- package/dist/completeObject.tool.d.ts.map +0 -1
- package/dist/completeObject.tool.js.map +0 -1
- package/dist/completeRows.tool.d.ts.map +0 -1
- package/dist/completeRows.tool.js.map +0 -1
- package/dist/dataset/steps.d.ts.map +0 -1
- package/dist/dataset/steps.js.map +0 -1
- package/dist/datasetFiles.d.ts.map +0 -1
- package/dist/datasetFiles.js.map +0 -1
- package/dist/domain.d.ts.map +0 -1
- package/dist/domain.js.map +0 -1
- package/dist/executeCommand.tool.d.ts.map +0 -1
- package/dist/executeCommand.tool.js.map +0 -1
- package/dist/file/steps.d.ts.map +0 -1
- package/dist/file/steps.js.map +0 -1
- package/dist/generateSchema.tool.d.ts.map +0 -1
- package/dist/generateSchema.tool.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/prompts.d.ts.map +0 -1
- package/dist/prompts.js.map +0 -1
- package/dist/runtime.d.ts.map +0 -1
- package/dist/runtime.js.map +0 -1
- package/dist/sandbox/steps.d.ts.map +0 -1
- package/dist/sandbox/steps.js.map +0 -1
- package/dist/schema.d.ts.map +0 -1
- package/dist/schema.js.map +0 -1
- package/dist/service.d.ts.map +0 -1
- package/dist/service.js.map +0 -1
- package/dist/steps/commitFromEvents.step.d.ts.map +0 -1
- package/dist/steps/commitFromEvents.step.js.map +0 -1
- package/dist/steps/persistObjectFromStory.step.d.ts.map +0 -1
- package/dist/steps/persistObjectFromStory.step.js.map +0 -1
- package/dist/structure.d.ts.map +0 -1
- package/dist/structure.js.map +0 -1
- package/dist/types/runtime.d.ts +0 -56
- package/dist/types/runtime.d.ts.map +0 -1
- package/dist/types/runtime.js +0 -2
- package/dist/types/runtime.js.map +0 -1
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { getDatasetOutputPath, getDatasetWorkstation } from "./datasetFiles";
|
|
2
|
+
import { createDatasetSandboxStep, runDatasetSandboxCommandStep } from "./sandbox/steps";
|
|
3
|
+
import { getStoryRuntime } from "./runtime";
|
|
4
|
+
/**
|
|
5
|
+
* Step 1/2:
|
|
6
|
+
* Download the rows output.jsonl from Instant storage into a sandbox file.
|
|
7
|
+
*
|
|
8
|
+
* This isolates network flakiness (e.g. undici `TypeError: terminated`) into a single step
|
|
9
|
+
* and makes subsequent reads purely sandbox-local.
|
|
10
|
+
*/
|
|
11
|
+
export async function structureDownloadRowsOutputToSandboxStep(params) {
|
|
12
|
+
"use step";
|
|
13
|
+
const runtime = params.runtime ?? "python3.13";
|
|
14
|
+
const timeoutMs = params.timeoutMs ?? 10 * 60 * 1000;
|
|
15
|
+
const { sandboxId } = await createDatasetSandboxStep({
|
|
16
|
+
env: params.env,
|
|
17
|
+
runtime,
|
|
18
|
+
timeoutMs,
|
|
19
|
+
purpose: "structure.rows-output.reader",
|
|
20
|
+
params: { structureId: params.structureId },
|
|
21
|
+
});
|
|
22
|
+
const workstation = getDatasetWorkstation(params.structureId);
|
|
23
|
+
const localPath = getDatasetOutputPath(params.structureId);
|
|
24
|
+
await runDatasetSandboxCommandStep({
|
|
25
|
+
env: params.env,
|
|
26
|
+
sandboxId,
|
|
27
|
+
cmd: "mkdir",
|
|
28
|
+
args: ["-p", workstation],
|
|
29
|
+
});
|
|
30
|
+
const storyRuntime = await getStoryRuntime(params.env);
|
|
31
|
+
const db = storyRuntime.db;
|
|
32
|
+
const contextKey = `structure:${params.structureId}`;
|
|
33
|
+
const query = (await db.query({
|
|
34
|
+
context_contexts: {
|
|
35
|
+
$: { where: { key: contextKey }, limit: 1 },
|
|
36
|
+
structure_output_file: {},
|
|
37
|
+
},
|
|
38
|
+
}));
|
|
39
|
+
const ctx = query.context_contexts?.[0];
|
|
40
|
+
const linked = Array.isArray(ctx?.structure_output_file) ? ctx.structure_output_file[0] : ctx.structure_output_file;
|
|
41
|
+
const url = linked?.url;
|
|
42
|
+
if (!url) {
|
|
43
|
+
throw new Error("Rows output file not found");
|
|
44
|
+
}
|
|
45
|
+
const py = [
|
|
46
|
+
"import sys, urllib.request",
|
|
47
|
+
"url = sys.argv[1]",
|
|
48
|
+
"out_path = sys.argv[2]",
|
|
49
|
+
"with urllib.request.urlopen(url) as r:",
|
|
50
|
+
" data = r.read()",
|
|
51
|
+
"with open(out_path, 'wb') as f:",
|
|
52
|
+
" f.write(data)",
|
|
53
|
+
"print('ok', len(data))",
|
|
54
|
+
].join("\n");
|
|
55
|
+
const res = await runDatasetSandboxCommandStep({
|
|
56
|
+
env: params.env,
|
|
57
|
+
sandboxId,
|
|
58
|
+
cmd: "python",
|
|
59
|
+
args: ["-c", py, String(url), localPath],
|
|
60
|
+
});
|
|
61
|
+
if (res.exitCode !== 0) {
|
|
62
|
+
throw new Error(res.stderr || "Failed to download rows output to sandbox");
|
|
63
|
+
}
|
|
64
|
+
return { sandboxId, localPath };
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Step 2/2:
|
|
68
|
+
* Read the next chunk of ROW records from the sandbox-local output.jsonl, bounded by `limit`.
|
|
69
|
+
*
|
|
70
|
+
* Pagination state is passed explicitly via `{ byteOffset, rowOffset }` and returned as next offsets.
|
|
71
|
+
*/
|
|
72
|
+
export async function structureReadRowsOutputChunkStep(params) {
|
|
73
|
+
"use step";
|
|
74
|
+
const py = [
|
|
75
|
+
"import sys, json",
|
|
76
|
+
"path = sys.argv[1]",
|
|
77
|
+
"byte_offset = int(sys.argv[2])",
|
|
78
|
+
"row_offset = int(sys.argv[3])",
|
|
79
|
+
"limit = int(sys.argv[4])",
|
|
80
|
+
"rows = []",
|
|
81
|
+
"next_byte = byte_offset",
|
|
82
|
+
"next_row = row_offset",
|
|
83
|
+
"with open(path, 'rb') as f:",
|
|
84
|
+
" f.seek(byte_offset)",
|
|
85
|
+
" while len(rows) < limit:",
|
|
86
|
+
" line = f.readline()",
|
|
87
|
+
" if not line:",
|
|
88
|
+
" break",
|
|
89
|
+
" next_byte = f.tell()",
|
|
90
|
+
" try:",
|
|
91
|
+
" obj = json.loads(line.decode('utf-8'))",
|
|
92
|
+
" except Exception:",
|
|
93
|
+
" continue",
|
|
94
|
+
" if obj.get('type') != 'row':",
|
|
95
|
+
" continue",
|
|
96
|
+
" rows.append(obj.get('data'))",
|
|
97
|
+
" next_row += 1",
|
|
98
|
+
"done = len(rows) < limit",
|
|
99
|
+
"print(json.dumps({",
|
|
100
|
+
" 'rows': rows,",
|
|
101
|
+
" 'nextByteOffset': next_byte,",
|
|
102
|
+
" 'nextRowOffset': next_row,",
|
|
103
|
+
" 'done': done,",
|
|
104
|
+
"}))",
|
|
105
|
+
].join("\n");
|
|
106
|
+
const res = await runDatasetSandboxCommandStep({
|
|
107
|
+
env: params.env,
|
|
108
|
+
sandboxId: params.sandboxId,
|
|
109
|
+
cmd: "python",
|
|
110
|
+
args: [
|
|
111
|
+
"-c",
|
|
112
|
+
py,
|
|
113
|
+
params.localPath,
|
|
114
|
+
String(params.byteOffset ?? 0),
|
|
115
|
+
String(params.rowOffset ?? 0),
|
|
116
|
+
String(params.limit),
|
|
117
|
+
],
|
|
118
|
+
});
|
|
119
|
+
if (res.exitCode !== 0) {
|
|
120
|
+
throw new Error(res.stderr || "Failed to read rows chunk from sandbox");
|
|
121
|
+
}
|
|
122
|
+
const out = String(res.stdout ?? "").trim();
|
|
123
|
+
const parsed = JSON.parse(out);
|
|
124
|
+
return parsed;
|
|
125
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { type DatasetSandboxId } from "./sandbox/steps";
|
|
2
|
+
/**
|
|
3
|
+
* Step 1: Download Structure rows output file (output.jsonl) into a sandbox.
|
|
4
|
+
*
|
|
5
|
+
* This enables pagination by reading chunks from the local sandbox filesystem.
|
|
6
|
+
*/
|
|
7
|
+
export declare function downloadStructureRowsOutputToSandboxStep(params: {
|
|
8
|
+
env: any;
|
|
9
|
+
sandboxId: DatasetSandboxId;
|
|
10
|
+
structureId: string;
|
|
11
|
+
}): Promise<{
|
|
12
|
+
filePath: string;
|
|
13
|
+
}>;
|
|
14
|
+
/**
|
|
15
|
+
* @deprecated Prefer `downloadStructureRowsOutputToSandboxStep` (kept for symmetry with chunk naming).
|
|
16
|
+
*
|
|
17
|
+
* Note: The name includes `RowsOutputJsonl` to be explicit about the file format.
|
|
18
|
+
*/
|
|
19
|
+
export declare function structureDownloadRowsOutputJsonlToSandboxStep(params: {
|
|
20
|
+
env: any;
|
|
21
|
+
sandboxId: DatasetSandboxId;
|
|
22
|
+
structureId: string;
|
|
23
|
+
}): Promise<{
|
|
24
|
+
filePath: string;
|
|
25
|
+
}>;
|
|
26
|
+
/**
|
|
27
|
+
* Step 2: Read a chunk/page of JSONL records from the downloaded sandbox file.
|
|
28
|
+
*
|
|
29
|
+
* Naming (consistent):
|
|
30
|
+
* - `structureReadRowsOutputJsonlStep` reads the whole file (base64)
|
|
31
|
+
* - `structureReadRowsOutputJsonlChunkStep` reads a paginated chunk from sandbox
|
|
32
|
+
*
|
|
33
|
+
* Offset/limit are line-based (0-indexed).
|
|
34
|
+
*/
|
|
35
|
+
export declare function structureReadRowsOutputJsonlChunkStep(params: {
|
|
36
|
+
env: any;
|
|
37
|
+
sandboxId: DatasetSandboxId;
|
|
38
|
+
structureId: string;
|
|
39
|
+
offset: number;
|
|
40
|
+
limit: number;
|
|
41
|
+
}): Promise<{
|
|
42
|
+
records: any[];
|
|
43
|
+
nextOffset: number;
|
|
44
|
+
done: boolean;
|
|
45
|
+
}>;
|
|
46
|
+
/**
|
|
47
|
+
* @deprecated Use `structureReadRowsOutputJsonlChunkStep` instead (naming consistency).
|
|
48
|
+
*/
|
|
49
|
+
export declare function readStructureRowsChunkFromSandboxStep(params: {
|
|
50
|
+
env: any;
|
|
51
|
+
sandboxId: DatasetSandboxId;
|
|
52
|
+
structureId: string;
|
|
53
|
+
offset: number;
|
|
54
|
+
limit: number;
|
|
55
|
+
}): Promise<{
|
|
56
|
+
records: any[];
|
|
57
|
+
nextOffset: number;
|
|
58
|
+
done: boolean;
|
|
59
|
+
}>;
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { getDatasetWorkstation } from "./datasetFiles";
|
|
2
|
+
import { runDatasetSandboxCommandStep, writeDatasetSandboxTextFileStep, } from "./sandbox/steps";
|
|
3
|
+
async function getRowsOutputUrl(params) {
|
|
4
|
+
const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
|
|
5
|
+
const runtime = (await resolveStoryRuntime(params.env));
|
|
6
|
+
const db = runtime.db;
|
|
7
|
+
const contextKey = `structure:${params.structureId}`;
|
|
8
|
+
const query = (await db.query({
|
|
9
|
+
context_contexts: {
|
|
10
|
+
$: { where: { key: contextKey }, limit: 1 },
|
|
11
|
+
structure_output_file: {},
|
|
12
|
+
},
|
|
13
|
+
}));
|
|
14
|
+
const ctx = query.context_contexts?.[0];
|
|
15
|
+
const linked = Array.isArray(ctx?.structure_output_file) ? ctx.structure_output_file[0] : ctx?.structure_output_file;
|
|
16
|
+
const url = linked?.url;
|
|
17
|
+
if (!url) {
|
|
18
|
+
throw new Error("Rows output file not found");
|
|
19
|
+
}
|
|
20
|
+
return String(url);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Step 1: Download Structure rows output file (output.jsonl) into a sandbox.
|
|
24
|
+
*
|
|
25
|
+
* This enables pagination by reading chunks from the local sandbox filesystem.
|
|
26
|
+
*/
|
|
27
|
+
export async function downloadStructureRowsOutputToSandboxStep(params) {
|
|
28
|
+
"use step";
|
|
29
|
+
const workstation = getDatasetWorkstation(params.structureId);
|
|
30
|
+
const filePath = `${workstation}/rows_output.jsonl`;
|
|
31
|
+
const scriptPath = `${workstation}/download_rows_output.py`;
|
|
32
|
+
// Ensure directory exists
|
|
33
|
+
await runDatasetSandboxCommandStep({
|
|
34
|
+
env: params.env,
|
|
35
|
+
sandboxId: params.sandboxId,
|
|
36
|
+
cmd: "mkdir",
|
|
37
|
+
args: ["-p", workstation],
|
|
38
|
+
});
|
|
39
|
+
const url = await getRowsOutputUrl({ env: params.env, structureId: params.structureId });
|
|
40
|
+
// Write a deterministic script to the sandbox and run it (no external deps).
|
|
41
|
+
await writeDatasetSandboxTextFileStep({
|
|
42
|
+
env: params.env,
|
|
43
|
+
sandboxId: params.sandboxId,
|
|
44
|
+
path: scriptPath,
|
|
45
|
+
text: [
|
|
46
|
+
"import argparse",
|
|
47
|
+
"import urllib.request",
|
|
48
|
+
"",
|
|
49
|
+
"def main():",
|
|
50
|
+
" p = argparse.ArgumentParser()",
|
|
51
|
+
" p.add_argument('--url', required=True)",
|
|
52
|
+
" p.add_argument('--out', required=True)",
|
|
53
|
+
" args = p.parse_args()",
|
|
54
|
+
"",
|
|
55
|
+
" # Download to local sandbox file",
|
|
56
|
+
" with urllib.request.urlopen(args.url, timeout=60) as r:",
|
|
57
|
+
" data = r.read()",
|
|
58
|
+
" with open(args.out, 'wb') as f:",
|
|
59
|
+
" f.write(data)",
|
|
60
|
+
"",
|
|
61
|
+
" print('ok')",
|
|
62
|
+
" print('bytes', len(data))",
|
|
63
|
+
"",
|
|
64
|
+
"if __name__ == '__main__':",
|
|
65
|
+
" main()",
|
|
66
|
+
"",
|
|
67
|
+
].join("\n"),
|
|
68
|
+
});
|
|
69
|
+
const res = await runDatasetSandboxCommandStep({
|
|
70
|
+
env: params.env,
|
|
71
|
+
sandboxId: params.sandboxId,
|
|
72
|
+
cmd: "python",
|
|
73
|
+
args: [scriptPath, "--url", url, "--out", filePath],
|
|
74
|
+
});
|
|
75
|
+
if (res.exitCode !== 0) {
|
|
76
|
+
throw new Error(res.stderr || "Failed to download rows output into sandbox");
|
|
77
|
+
}
|
|
78
|
+
return { filePath };
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* @deprecated Prefer `downloadStructureRowsOutputToSandboxStep` (kept for symmetry with chunk naming).
|
|
82
|
+
*
|
|
83
|
+
* Note: The name includes `RowsOutputJsonl` to be explicit about the file format.
|
|
84
|
+
*/
|
|
85
|
+
export async function structureDownloadRowsOutputJsonlToSandboxStep(params) {
|
|
86
|
+
"use step";
|
|
87
|
+
return await downloadStructureRowsOutputToSandboxStep(params);
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Step 2: Read a chunk/page of JSONL records from the downloaded sandbox file.
|
|
91
|
+
*
|
|
92
|
+
* Naming (consistent):
|
|
93
|
+
* - `structureReadRowsOutputJsonlStep` reads the whole file (base64)
|
|
94
|
+
* - `structureReadRowsOutputJsonlChunkStep` reads a paginated chunk from sandbox
|
|
95
|
+
*
|
|
96
|
+
* Offset/limit are line-based (0-indexed).
|
|
97
|
+
*/
|
|
98
|
+
export async function structureReadRowsOutputJsonlChunkStep(params) {
|
|
99
|
+
"use step";
|
|
100
|
+
const workstation = getDatasetWorkstation(params.structureId);
|
|
101
|
+
const filePath = `${workstation}/rows_output.jsonl`;
|
|
102
|
+
const scriptPath = `${workstation}/read_rows_chunk.py`;
|
|
103
|
+
await writeDatasetSandboxTextFileStep({
|
|
104
|
+
env: params.env,
|
|
105
|
+
sandboxId: params.sandboxId,
|
|
106
|
+
path: scriptPath,
|
|
107
|
+
text: [
|
|
108
|
+
"import argparse",
|
|
109
|
+
"import json",
|
|
110
|
+
"",
|
|
111
|
+
"def main():",
|
|
112
|
+
" p = argparse.ArgumentParser()",
|
|
113
|
+
" p.add_argument('--path', required=True)",
|
|
114
|
+
" p.add_argument('--offset', type=int, required=True)",
|
|
115
|
+
" p.add_argument('--limit', type=int, required=True)",
|
|
116
|
+
" args = p.parse_args()",
|
|
117
|
+
"",
|
|
118
|
+
" records = []",
|
|
119
|
+
" current = 0",
|
|
120
|
+
" taken = 0",
|
|
121
|
+
" done = True",
|
|
122
|
+
"",
|
|
123
|
+
" with open(args.path, 'r', encoding='utf-8', errors='replace') as f:",
|
|
124
|
+
" for line in f:",
|
|
125
|
+
" if current < args.offset:",
|
|
126
|
+
" current += 1",
|
|
127
|
+
" continue",
|
|
128
|
+
" if taken >= args.limit:",
|
|
129
|
+
" done = False",
|
|
130
|
+
" break",
|
|
131
|
+
" line = line.strip()",
|
|
132
|
+
" if not line:",
|
|
133
|
+
" current += 1",
|
|
134
|
+
" continue",
|
|
135
|
+
" try:",
|
|
136
|
+
" records.append(json.loads(line))",
|
|
137
|
+
" except Exception:",
|
|
138
|
+
" # Skip invalid JSON line",
|
|
139
|
+
" pass",
|
|
140
|
+
" taken += 1",
|
|
141
|
+
" current += 1",
|
|
142
|
+
"",
|
|
143
|
+
" out = {",
|
|
144
|
+
" 'records': records,",
|
|
145
|
+
" 'nextOffset': args.offset + taken,",
|
|
146
|
+
" 'done': done,",
|
|
147
|
+
" }",
|
|
148
|
+
" print(json.dumps(out, ensure_ascii=False))",
|
|
149
|
+
"",
|
|
150
|
+
"if __name__ == '__main__':",
|
|
151
|
+
" main()",
|
|
152
|
+
"",
|
|
153
|
+
].join("\n"),
|
|
154
|
+
});
|
|
155
|
+
const res = await runDatasetSandboxCommandStep({
|
|
156
|
+
env: params.env,
|
|
157
|
+
sandboxId: params.sandboxId,
|
|
158
|
+
cmd: "python",
|
|
159
|
+
args: [
|
|
160
|
+
scriptPath,
|
|
161
|
+
"--path",
|
|
162
|
+
filePath,
|
|
163
|
+
"--offset",
|
|
164
|
+
String(Math.max(0, params.offset ?? 0)),
|
|
165
|
+
"--limit",
|
|
166
|
+
String(Math.max(1, params.limit ?? 1)),
|
|
167
|
+
],
|
|
168
|
+
});
|
|
169
|
+
if (res.exitCode !== 0) {
|
|
170
|
+
throw new Error(res.stderr || "Failed to read rows chunk from sandbox");
|
|
171
|
+
}
|
|
172
|
+
const text = (res.stdout ?? "").trim();
|
|
173
|
+
if (!text) {
|
|
174
|
+
return { records: [], nextOffset: params.offset, done: true };
|
|
175
|
+
}
|
|
176
|
+
// The script prints a single JSON object.
|
|
177
|
+
const parsed = JSON.parse(text);
|
|
178
|
+
return {
|
|
179
|
+
records: Array.isArray(parsed?.records) ? parsed.records : [],
|
|
180
|
+
nextOffset: Number(parsed?.nextOffset ?? params.offset),
|
|
181
|
+
done: Boolean(parsed?.done),
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* @deprecated Use `structureReadRowsOutputJsonlChunkStep` instead (naming consistency).
|
|
186
|
+
*/
|
|
187
|
+
export async function readStructureRowsChunkFromSandboxStep(params) {
|
|
188
|
+
"use step";
|
|
189
|
+
return await structureReadRowsOutputJsonlChunkStep(params);
|
|
190
|
+
}
|
package/dist/runtime.d.ts
CHANGED
|
@@ -2,9 +2,8 @@
|
|
|
2
2
|
* Internal helper to resolve Story runtime from within workflow steps.
|
|
3
3
|
*
|
|
4
4
|
* Why dynamic import?
|
|
5
|
-
* - Some bundlers (notably Turbopack step bundles) can drop/hoist static imports in "use
|
|
5
|
+
* - Some bundlers (notably Turbopack step bundles) can drop/hoist static imports in "use-step" modules,
|
|
6
6
|
* causing `ReferenceError: resolveStoryRuntime is not defined`.
|
|
7
7
|
* - Using a dynamic import keeps the symbol resolution local to the step runtime.
|
|
8
8
|
*/
|
|
9
9
|
export declare function getStoryRuntime(env: any): Promise<import("@ekairos/story/runtime").StoryRuntime>;
|
|
10
|
-
//# sourceMappingURL=runtime.d.ts.map
|
package/dist/runtime.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Internal helper to resolve Story runtime from within workflow steps.
|
|
3
3
|
*
|
|
4
4
|
* Why dynamic import?
|
|
5
|
-
* - Some bundlers (notably Turbopack step bundles) can drop/hoist static imports in "use
|
|
5
|
+
* - Some bundlers (notably Turbopack step bundles) can drop/hoist static imports in "use-step" modules,
|
|
6
6
|
* causing `ReferenceError: resolveStoryRuntime is not defined`.
|
|
7
7
|
* - Using a dynamic import keeps the symbol resolution local to the step runtime.
|
|
8
8
|
*/
|
|
@@ -10,4 +10,3 @@ export async function getStoryRuntime(env) {
|
|
|
10
10
|
const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
|
|
11
11
|
return await resolveStoryRuntime(env);
|
|
12
12
|
}
|
|
13
|
-
//# sourceMappingURL=runtime.js.map
|
package/dist/sandbox/steps.d.ts
CHANGED
package/dist/sandbox/steps.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { getStoryRuntime } from "../runtime";
|
|
2
1
|
export async function createDatasetSandboxStep(params) {
|
|
3
2
|
"use step";
|
|
4
3
|
console.log("[ekairos/structure] sandbox.step createSandbox begin");
|
|
5
4
|
console.log("[ekairos/structure] sandbox.step createSandbox runtime", params.runtime);
|
|
6
|
-
const
|
|
5
|
+
const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
|
|
6
|
+
const db = (await resolveStoryRuntime(params.env)).db;
|
|
7
7
|
const { SandboxService } = (await import("@ekairos/sandbox"));
|
|
8
8
|
const service = new SandboxService(db);
|
|
9
9
|
const created = await service.createSandbox(params);
|
|
@@ -18,7 +18,8 @@ export async function runDatasetSandboxCommandStep(params) {
|
|
|
18
18
|
console.log("[ekairos/structure] sandbox.step runCommand begin");
|
|
19
19
|
console.log("[ekairos/structure] sandbox.step runCommand sandboxId", params.sandboxId);
|
|
20
20
|
console.log("[ekairos/structure] sandbox.step runCommand cmd", params.cmd);
|
|
21
|
-
const
|
|
21
|
+
const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
|
|
22
|
+
const db = (await resolveStoryRuntime(params.env)).db;
|
|
22
23
|
const { SandboxService } = (await import("@ekairos/sandbox"));
|
|
23
24
|
const service = new SandboxService(db);
|
|
24
25
|
const result = await service.runCommand(params.sandboxId, params.cmd, params.args ?? []);
|
|
@@ -38,7 +39,8 @@ export async function writeDatasetSandboxFilesStep(params) {
|
|
|
38
39
|
console.log("[ekairos/structure] sandbox.step writeFiles begin");
|
|
39
40
|
console.log("[ekairos/structure] sandbox.step writeFiles sandboxId", params.sandboxId);
|
|
40
41
|
console.log("[ekairos/structure] sandbox.step writeFiles paths", params.files.map((f) => f.path));
|
|
41
|
-
const
|
|
42
|
+
const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
|
|
43
|
+
const db = (await resolveStoryRuntime(params.env)).db;
|
|
42
44
|
const { SandboxService } = (await import("@ekairos/sandbox"));
|
|
43
45
|
const service = new SandboxService(db);
|
|
44
46
|
const result = await service.writeFiles(params.sandboxId, params.files);
|
|
@@ -67,7 +69,8 @@ export async function readDatasetSandboxFileStep(params) {
|
|
|
67
69
|
console.log("[ekairos/structure] sandbox.step readFile begin");
|
|
68
70
|
console.log("[ekairos/structure] sandbox.step readFile sandboxId", params.sandboxId);
|
|
69
71
|
console.log("[ekairos/structure] sandbox.step readFile path", params.path);
|
|
70
|
-
const
|
|
72
|
+
const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
|
|
73
|
+
const db = (await resolveStoryRuntime(params.env)).db;
|
|
71
74
|
const { SandboxService } = (await import("@ekairos/sandbox"));
|
|
72
75
|
const service = new SandboxService(db);
|
|
73
76
|
const result = await service.readFile(params.sandboxId, params.path);
|
|
@@ -93,7 +96,8 @@ export async function stopDatasetSandboxStep(params) {
|
|
|
93
96
|
"use step";
|
|
94
97
|
console.log("[ekairos/structure] sandbox.step stopSandbox begin");
|
|
95
98
|
console.log("[ekairos/structure] sandbox.step stopSandbox sandboxId", params.sandboxId);
|
|
96
|
-
const
|
|
99
|
+
const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
|
|
100
|
+
const db = (await resolveStoryRuntime(params.env)).db;
|
|
97
101
|
const { SandboxService } = (await import("@ekairos/sandbox"));
|
|
98
102
|
const service = new SandboxService(db);
|
|
99
103
|
const result = await service.stopSandbox(params.sandboxId);
|
|
@@ -101,4 +105,3 @@ export async function stopDatasetSandboxStep(params) {
|
|
|
101
105
|
throw new Error(result.error);
|
|
102
106
|
console.log("[ekairos/structure] sandbox.step stopSandbox ok");
|
|
103
107
|
}
|
|
104
|
-
//# sourceMappingURL=steps.js.map
|
package/dist/schema.d.ts
CHANGED
package/dist/schema.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { i } from "@instantdb/core";
|
|
2
2
|
import { domain } from "@ekairos/domain";
|
|
3
|
-
import { storyDomain } from "@ekairos/story";
|
|
3
|
+
import { storyDomain } from "@ekairos/story/schema";
|
|
4
|
+
import { sandboxDomain } from "@ekairos/sandbox/schema";
|
|
4
5
|
const entities = {
|
|
5
6
|
// Keep $files compatible with Instant's base file fields used by structure flows.
|
|
6
7
|
$files: i.entity({
|
|
@@ -29,5 +30,5 @@ const links = {
|
|
|
29
30
|
const rooms = {};
|
|
30
31
|
export const structureDomain = domain("structure")
|
|
31
32
|
.includes(storyDomain)
|
|
33
|
+
.includes(sandboxDomain)
|
|
32
34
|
.schema({ entities, links, rooms });
|
|
33
|
-
//# sourceMappingURL=schema.js.map
|
package/dist/service.d.ts
CHANGED
package/dist/service.js
CHANGED
|
@@ -55,42 +55,22 @@ export class DatasetService {
|
|
|
55
55
|
if (!url)
|
|
56
56
|
return { ok: false, error: "Rows output file not found" };
|
|
57
57
|
async function* createGenerator(fileUrl) {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
const
|
|
67
|
-
if (
|
|
68
|
-
|
|
69
|
-
buffer += decoder.decode(value, { stream: true });
|
|
70
|
-
let newlineIndex = buffer.indexOf("\n");
|
|
71
|
-
while (newlineIndex !== -1) {
|
|
72
|
-
const line = buffer.slice(0, newlineIndex);
|
|
73
|
-
buffer = buffer.slice(newlineIndex + 1);
|
|
74
|
-
const trimmed = line.trim();
|
|
75
|
-
if (trimmed) {
|
|
76
|
-
try {
|
|
77
|
-
const record = JSON.parse(trimmed);
|
|
78
|
-
yield record;
|
|
79
|
-
}
|
|
80
|
-
catch {
|
|
81
|
-
// skip invalid
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
newlineIndex = buffer.indexOf("\n");
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
const remaining = buffer.trim();
|
|
88
|
-
if (remaining) {
|
|
58
|
+
// NOTE:
|
|
59
|
+
// We intentionally download the file fully (with retry/timeout) before yielding.
|
|
60
|
+
// This avoids partial processing + duplicates when network streams abort mid-way
|
|
61
|
+
// (e.g. undici `TypeError: terminated`).
|
|
62
|
+
const text = await fetchTextWithRetry(fileUrl, { attempts: 4, timeoutMs: 90000 });
|
|
63
|
+
// Parse JSONL (one JSON object per line)
|
|
64
|
+
const lines = text.split("\n");
|
|
65
|
+
for (const line of lines) {
|
|
66
|
+
const trimmed = line.trim();
|
|
67
|
+
if (!trimmed)
|
|
68
|
+
continue;
|
|
89
69
|
try {
|
|
90
|
-
yield JSON.parse(
|
|
70
|
+
yield JSON.parse(trimmed);
|
|
91
71
|
}
|
|
92
72
|
catch {
|
|
93
|
-
// skip invalid
|
|
73
|
+
// skip invalid line
|
|
94
74
|
}
|
|
95
75
|
}
|
|
96
76
|
}
|
|
@@ -176,4 +156,33 @@ function createUuidV4() {
|
|
|
176
156
|
return v.toString(16);
|
|
177
157
|
});
|
|
178
158
|
}
|
|
179
|
-
|
|
159
|
+
async function sleep(ms) {
|
|
160
|
+
return new Promise((r) => setTimeout(r, ms));
|
|
161
|
+
}
|
|
162
|
+
async function fetchTextWithRetry(url, opts) {
|
|
163
|
+
let lastError = null;
|
|
164
|
+
for (let attempt = 1; attempt <= opts.attempts; attempt++) {
|
|
165
|
+
const controller = new AbortController();
|
|
166
|
+
const timer = setTimeout(() => controller.abort(), opts.timeoutMs);
|
|
167
|
+
try {
|
|
168
|
+
const res = await fetch(url, { signal: controller.signal });
|
|
169
|
+
if (!res.ok) {
|
|
170
|
+
throw new Error(`Failed to download rows output file (HTTP ${res.status})`);
|
|
171
|
+
}
|
|
172
|
+
return await res.text();
|
|
173
|
+
}
|
|
174
|
+
catch (e) {
|
|
175
|
+
lastError = e;
|
|
176
|
+
// Backoff: 250ms, 500ms, 1000ms...
|
|
177
|
+
if (attempt < opts.attempts) {
|
|
178
|
+
await sleep(250 * Math.pow(2, attempt - 1));
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
finally {
|
|
183
|
+
clearTimeout(timer);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
const message = lastError instanceof Error ? lastError.message : String(lastError);
|
|
187
|
+
throw new Error(message || "Failed to download rows output file");
|
|
188
|
+
}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { getStoryRuntime } from "../runtime";
|
|
2
1
|
function isToolCompletePart(value) {
|
|
3
2
|
if (!value || typeof value !== "object")
|
|
4
3
|
return false;
|
|
@@ -23,7 +22,8 @@ export async function structureCommitFromEventsStep(params) {
|
|
|
23
22
|
"use step";
|
|
24
23
|
const contextKey = `structure:${params.structureId}`;
|
|
25
24
|
try {
|
|
26
|
-
const
|
|
25
|
+
const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
|
|
26
|
+
const runtime = (await resolveStoryRuntime(params.env));
|
|
27
27
|
const store = runtime.store;
|
|
28
28
|
const db = runtime.db;
|
|
29
29
|
const events = await store.getEvents({ key: contextKey });
|
|
@@ -79,4 +79,3 @@ export async function structureCommitFromEventsStep(params) {
|
|
|
79
79
|
return { ok: false, error: message };
|
|
80
80
|
}
|
|
81
81
|
}
|
|
82
|
-
//# sourceMappingURL=commitFromEvents.step.js.map
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import { getStoryRuntime } from "../runtime";
|
|
2
|
-
import { structurePatchContextContentStep, structureGetContextStep } from "../dataset/steps";
|
|
3
1
|
function extractJsonObject(text) {
|
|
4
2
|
const trimmed = text.trim();
|
|
5
3
|
if (!trimmed)
|
|
@@ -38,13 +36,15 @@ function extractJsonObject(text) {
|
|
|
38
36
|
}
|
|
39
37
|
export async function persistObjectResultFromStoryStep(params) {
|
|
40
38
|
"use step";
|
|
41
|
-
const
|
|
39
|
+
const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
|
|
40
|
+
const runtime = (await resolveStoryRuntime(params.env));
|
|
42
41
|
const store = runtime.store;
|
|
43
42
|
console.log("[ekairos/structure] persistObjectFromStory.step execute begin");
|
|
44
43
|
console.log("[ekairos/structure] persistObjectFromStory.step execute datasetId", params.datasetId);
|
|
45
44
|
const contextKey = `structure:${params.datasetId}`;
|
|
46
45
|
const events = await store.getEvents({ key: contextKey });
|
|
47
46
|
console.log("[ekairos/structure] persistObjectFromStory.step execute eventsCount", Array.isArray(events) ? events.length : -1);
|
|
47
|
+
const { structurePatchContextContentStep, structureGetContextStep } = await import("../dataset/steps");
|
|
48
48
|
const ctxResult = await structureGetContextStep({ env: params.env, contextKey });
|
|
49
49
|
const existingContent = ctxResult.ok ? (ctxResult.data?.content ?? {}) : {};
|
|
50
50
|
for (let i = events.length - 1; i >= 0; i--) {
|
|
@@ -87,4 +87,3 @@ export async function persistObjectResultFromStoryStep(params) {
|
|
|
87
87
|
console.log("[ekairos/structure] persistObjectFromStory.step execute noJsonFound");
|
|
88
88
|
return { ok: false };
|
|
89
89
|
}
|
|
90
|
-
//# sourceMappingURL=persistObjectFromStory.step.js.map
|