@ekairos/dataset 1.22.45-beta.development.0 → 1.22.47-beta.development.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builder/instructions.d.ts +1 -2
- package/dist/builder/instructions.js +4 -11
- package/dist/builder/materialize.d.ts +1 -2
- package/dist/builder/materialize.js +30 -36
- package/dist/builder/persistence.d.ts +1 -2
- package/dist/builder/persistence.js +17 -26
- package/dist/builder/schemaInference.d.ts +1 -2
- package/dist/builder/schemaInference.js +4 -12
- package/dist/builder/sourceRows.d.ts +0 -1
- package/dist/builder/sourceRows.js +3 -9
- package/dist/builder/types.d.ts +1 -2
- package/dist/builder/types.js +1 -3
- package/dist/clearDataset.tool.d.ts +0 -1
- package/dist/clearDataset.tool.js +12 -16
- package/dist/completeDataset.tool.d.ts +0 -1
- package/dist/completeDataset.tool.js +18 -25
- package/dist/dataset/steps.d.ts +5 -6
- package/dist/dataset/steps.js +25 -36
- package/dist/dataset.d.ts +2 -3
- package/dist/dataset.js +13 -17
- package/dist/datasetFiles.d.ts +0 -1
- package/dist/datasetFiles.js +5 -12
- package/dist/domain.d.ts +1 -2
- package/dist/domain.js +1 -6
- package/dist/eventsReactRuntime.d.ts +0 -1
- package/dist/eventsReactRuntime.js +9 -13
- package/dist/executeCommand.tool.d.ts +0 -1
- package/dist/executeCommand.tool.js +12 -16
- package/dist/file/file-dataset.agent.d.ts +1 -2
- package/dist/file/file-dataset.agent.js +34 -38
- package/dist/file/filepreview.d.ts +0 -1
- package/dist/file/filepreview.js +12 -16
- package/dist/file/generateSchema.tool.d.ts +0 -1
- package/dist/file/generateSchema.tool.js +10 -14
- package/dist/file/index.d.ts +1 -2
- package/dist/file/index.js +1 -18
- package/dist/file/prompts.d.ts +1 -2
- package/dist/file/prompts.js +13 -17
- package/dist/file/steps.d.ts +0 -1
- package/dist/file/steps.js +3 -7
- package/dist/index.d.ts +6 -7
- package/dist/index.js +6 -23
- package/dist/materializeDataset.tool.d.ts +1 -2
- package/dist/materializeDataset.tool.js +40 -45
- package/dist/query/index.d.ts +1 -2
- package/dist/query/index.js +1 -18
- package/dist/query/queryDomain.d.ts +1 -2
- package/dist/query/queryDomain.js +2 -39
- package/dist/query/queryDomain.step.d.ts +0 -1
- package/dist/query/queryDomain.step.js +8 -12
- package/dist/sandbox/steps.d.ts +0 -1
- package/dist/sandbox/steps.js +30 -74
- package/dist/sandbox.steps.d.ts +1 -2
- package/dist/sandbox.steps.js +1 -18
- package/dist/schema.d.ts +0 -1
- package/dist/schema.js +22 -26
- package/dist/service.d.ts +1 -2
- package/dist/service.js +5 -10
- package/dist/skill.d.ts +0 -1
- package/dist/skill.js +12 -17
- package/dist/transform/filepreview.d.ts +0 -1
- package/dist/transform/filepreview.js +6 -10
- package/dist/transform/index.d.ts +2 -3
- package/dist/transform/index.js +2 -8
- package/dist/transform/prompts.d.ts +0 -1
- package/dist/transform/prompts.js +12 -16
- package/dist/transform/transform-dataset.agent.d.ts +1 -2
- package/dist/transform/transform-dataset.agent.js +31 -35
- package/dist/transform/transformDataset.d.ts +0 -1
- package/dist/transform/transformDataset.js +7 -11
- package/package.json +12 -7
- package/dist/builder/instructions.d.ts.map +0 -1
- package/dist/builder/instructions.js.map +0 -1
- package/dist/builder/materialize.d.ts.map +0 -1
- package/dist/builder/materialize.js.map +0 -1
- package/dist/builder/persistence.d.ts.map +0 -1
- package/dist/builder/persistence.js.map +0 -1
- package/dist/builder/schemaInference.d.ts.map +0 -1
- package/dist/builder/schemaInference.js.map +0 -1
- package/dist/builder/sourceRows.d.ts.map +0 -1
- package/dist/builder/sourceRows.js.map +0 -1
- package/dist/builder/types.d.ts.map +0 -1
- package/dist/builder/types.js.map +0 -1
- package/dist/clearDataset.tool.d.ts.map +0 -1
- package/dist/clearDataset.tool.js.map +0 -1
- package/dist/completeDataset.tool.d.ts.map +0 -1
- package/dist/completeDataset.tool.js.map +0 -1
- package/dist/dataset/steps.d.ts.map +0 -1
- package/dist/dataset/steps.js.map +0 -1
- package/dist/dataset.d.ts.map +0 -1
- package/dist/dataset.js.map +0 -1
- package/dist/datasetFiles.d.ts.map +0 -1
- package/dist/datasetFiles.js.map +0 -1
- package/dist/domain.d.ts.map +0 -1
- package/dist/domain.js.map +0 -1
- package/dist/eventsReactRuntime.d.ts.map +0 -1
- package/dist/eventsReactRuntime.js.map +0 -1
- package/dist/executeCommand.tool.d.ts.map +0 -1
- package/dist/executeCommand.tool.js.map +0 -1
- package/dist/file/file-dataset.agent.d.ts.map +0 -1
- package/dist/file/file-dataset.agent.js.map +0 -1
- package/dist/file/filepreview.d.ts.map +0 -1
- package/dist/file/filepreview.js.map +0 -1
- package/dist/file/generateSchema.tool.d.ts.map +0 -1
- package/dist/file/generateSchema.tool.js.map +0 -1
- package/dist/file/index.d.ts.map +0 -1
- package/dist/file/index.js.map +0 -1
- package/dist/file/prompts.d.ts.map +0 -1
- package/dist/file/prompts.js.map +0 -1
- package/dist/file/steps.d.ts.map +0 -1
- package/dist/file/steps.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/materializeDataset.tool.d.ts.map +0 -1
- package/dist/materializeDataset.tool.js.map +0 -1
- package/dist/query/index.d.ts.map +0 -1
- package/dist/query/index.js.map +0 -1
- package/dist/query/queryDomain.d.ts.map +0 -1
- package/dist/query/queryDomain.js.map +0 -1
- package/dist/query/queryDomain.step.d.ts.map +0 -1
- package/dist/query/queryDomain.step.js.map +0 -1
- package/dist/sandbox/steps.d.ts.map +0 -1
- package/dist/sandbox/steps.js.map +0 -1
- package/dist/sandbox.steps.d.ts.map +0 -1
- package/dist/sandbox.steps.js.map +0 -1
- package/dist/schema.d.ts.map +0 -1
- package/dist/schema.js.map +0 -1
- package/dist/service.d.ts.map +0 -1
- package/dist/service.js.map +0 -1
- package/dist/skill.d.ts.map +0 -1
- package/dist/skill.js.map +0 -1
- package/dist/transform/filepreview.d.ts.map +0 -1
- package/dist/transform/filepreview.js.map +0 -1
- package/dist/transform/index.d.ts.map +0 -1
- package/dist/transform/index.js.map +0 -1
- package/dist/transform/prompts.d.ts.map +0 -1
- package/dist/transform/prompts.js.map +0 -1
- package/dist/transform/transform-dataset.agent.d.ts.map +0 -1
- package/dist/transform/transform-dataset.agent.js.map +0 -1
- package/dist/transform/transformDataset.d.ts.map +0 -1
- package/dist/transform/transformDataset.js.map +0 -1
package/dist/sandbox/steps.js
CHANGED
|
@@ -1,63 +1,20 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
-
};
|
|
38
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.createDatasetSandboxStep = createDatasetSandboxStep;
|
|
40
|
-
exports.runDatasetSandboxCommandStep = runDatasetSandboxCommandStep;
|
|
41
|
-
exports.writeDatasetSandboxFilesStep = writeDatasetSandboxFilesStep;
|
|
42
|
-
exports.readDatasetSandboxFileStep = readDatasetSandboxFileStep;
|
|
43
|
-
exports.stopDatasetSandboxStep = stopDatasetSandboxStep;
|
|
44
|
-
const node_child_process_1 = require("node:child_process");
|
|
45
|
-
const node_fs_1 = require("node:fs");
|
|
46
|
-
const node_path_1 = __importDefault(require("node:path"));
|
|
47
|
-
const node_util_1 = require("node:util");
|
|
48
|
-
const runtime_1 = require("@ekairos/events/runtime");
|
|
49
|
-
const execFileAsync = (0, node_util_1.promisify)(node_child_process_1.execFile);
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { promises as fs } from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { promisify } from "node:util";
|
|
5
|
+
import { getContextRuntime } from "@ekairos/events/runtime";
|
|
6
|
+
const execFileAsync = promisify(execFile);
|
|
50
7
|
const localSandboxRoots = new Map();
|
|
51
8
|
function isLocalDatasetSandboxMode() {
|
|
52
9
|
return String(process.env.DATASET_TEST_LOCAL_SANDBOX ?? "").trim() === "1";
|
|
53
10
|
}
|
|
54
11
|
function getLocalSandboxRoot(sandboxId) {
|
|
55
12
|
return (localSandboxRoots.get(sandboxId) ||
|
|
56
|
-
|
|
13
|
+
path.resolve(process.cwd(), "test-results", "dataset-sandboxes", sandboxId));
|
|
57
14
|
}
|
|
58
15
|
async function ensureLocalSandboxRoot(sandboxId) {
|
|
59
16
|
const root = getLocalSandboxRoot(sandboxId);
|
|
60
|
-
await
|
|
17
|
+
await fs.mkdir(root, { recursive: true });
|
|
61
18
|
localSandboxRoots.set(sandboxId, root);
|
|
62
19
|
return root;
|
|
63
20
|
}
|
|
@@ -67,17 +24,17 @@ async function runLocalSandboxCommand(params) {
|
|
|
67
24
|
const args = params.args ?? [];
|
|
68
25
|
if (cmd === "mkdir") {
|
|
69
26
|
const target = args[args.length - 1];
|
|
70
|
-
await
|
|
27
|
+
await fs.mkdir(String(target ?? ""), { recursive: true });
|
|
71
28
|
return { exitCode: 0, stdout: "", stderr: "" };
|
|
72
29
|
}
|
|
73
30
|
if (cmd === "rm") {
|
|
74
31
|
const target = args[args.length - 1];
|
|
75
|
-
await
|
|
32
|
+
await fs.rm(String(target ?? ""), { force: true, recursive: false });
|
|
76
33
|
return { exitCode: 0, stdout: "", stderr: "" };
|
|
77
34
|
}
|
|
78
35
|
if (cmd === "test" && args[0] === "-f") {
|
|
79
36
|
try {
|
|
80
|
-
await
|
|
37
|
+
await fs.access(String(args[1] ?? ""));
|
|
81
38
|
return { exitCode: 0, stdout: "", stderr: "" };
|
|
82
39
|
}
|
|
83
40
|
catch {
|
|
@@ -99,22 +56,22 @@ async function runLocalSandboxCommand(params) {
|
|
|
99
56
|
stderr: String(result.stderr ?? ""),
|
|
100
57
|
};
|
|
101
58
|
}
|
|
102
|
-
async function createDatasetSandboxStep(params) {
|
|
59
|
+
export async function createDatasetSandboxStep(params) {
|
|
103
60
|
"use step";
|
|
104
61
|
if (isLocalDatasetSandboxMode()) {
|
|
105
62
|
const sandboxId = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
106
63
|
await ensureLocalSandboxRoot(sandboxId);
|
|
107
64
|
return { sandboxId };
|
|
108
65
|
}
|
|
109
|
-
const db = (await
|
|
110
|
-
const { SandboxService } = (await
|
|
66
|
+
const db = (await getContextRuntime(params.env)).db;
|
|
67
|
+
const { SandboxService } = (await import("@ekairos/sandbox"));
|
|
111
68
|
const service = new SandboxService(db);
|
|
112
69
|
const created = await service.createSandbox(params);
|
|
113
70
|
if (!created.ok)
|
|
114
71
|
throw new Error(created.error);
|
|
115
72
|
return { sandboxId: created.data.sandboxId };
|
|
116
73
|
}
|
|
117
|
-
async function runDatasetSandboxCommandStep(params) {
|
|
74
|
+
export async function runDatasetSandboxCommandStep(params) {
|
|
118
75
|
"use step";
|
|
119
76
|
if (isLocalDatasetSandboxMode()) {
|
|
120
77
|
return await runLocalSandboxCommand({
|
|
@@ -123,8 +80,8 @@ async function runDatasetSandboxCommandStep(params) {
|
|
|
123
80
|
args: params.args,
|
|
124
81
|
});
|
|
125
82
|
}
|
|
126
|
-
const db = (await
|
|
127
|
-
const { SandboxService } = (await
|
|
83
|
+
const db = (await getContextRuntime(params.env)).db;
|
|
84
|
+
const { SandboxService } = (await import("@ekairos/sandbox"));
|
|
128
85
|
const service = new SandboxService(db);
|
|
129
86
|
const result = await service.runCommand(params.sandboxId, params.cmd, params.args ?? []);
|
|
130
87
|
if (!result.ok)
|
|
@@ -135,49 +92,48 @@ async function runDatasetSandboxCommandStep(params) {
|
|
|
135
92
|
stderr: result.data.error ?? "",
|
|
136
93
|
};
|
|
137
94
|
}
|
|
138
|
-
async function writeDatasetSandboxFilesStep(params) {
|
|
95
|
+
export async function writeDatasetSandboxFilesStep(params) {
|
|
139
96
|
"use step";
|
|
140
97
|
if (isLocalDatasetSandboxMode()) {
|
|
141
98
|
for (const file of params.files) {
|
|
142
|
-
await
|
|
143
|
-
await
|
|
99
|
+
await fs.mkdir(path.dirname(file.path), { recursive: true });
|
|
100
|
+
await fs.writeFile(file.path, Buffer.from(file.contentBase64, "base64"));
|
|
144
101
|
}
|
|
145
102
|
return;
|
|
146
103
|
}
|
|
147
|
-
const db = (await
|
|
148
|
-
const { SandboxService } = (await
|
|
104
|
+
const db = (await getContextRuntime(params.env)).db;
|
|
105
|
+
const { SandboxService } = (await import("@ekairos/sandbox"));
|
|
149
106
|
const service = new SandboxService(db);
|
|
150
107
|
const result = await service.writeFiles(params.sandboxId, params.files);
|
|
151
108
|
if (!result.ok)
|
|
152
109
|
throw new Error(result.error);
|
|
153
110
|
}
|
|
154
|
-
async function readDatasetSandboxFileStep(params) {
|
|
111
|
+
export async function readDatasetSandboxFileStep(params) {
|
|
155
112
|
"use step";
|
|
156
113
|
if (isLocalDatasetSandboxMode()) {
|
|
157
|
-
const content = await
|
|
114
|
+
const content = await fs.readFile(params.path);
|
|
158
115
|
return { contentBase64: Buffer.from(content).toString("base64") };
|
|
159
116
|
}
|
|
160
|
-
const db = (await
|
|
161
|
-
const { SandboxService } = (await
|
|
117
|
+
const db = (await getContextRuntime(params.env)).db;
|
|
118
|
+
const { SandboxService } = (await import("@ekairos/sandbox"));
|
|
162
119
|
const service = new SandboxService(db);
|
|
163
120
|
const result = await service.readFile(params.sandboxId, params.path);
|
|
164
121
|
if (!result.ok)
|
|
165
122
|
throw new Error(result.error);
|
|
166
123
|
return result.data;
|
|
167
124
|
}
|
|
168
|
-
async function stopDatasetSandboxStep(params) {
|
|
125
|
+
export async function stopDatasetSandboxStep(params) {
|
|
169
126
|
"use step";
|
|
170
127
|
if (isLocalDatasetSandboxMode()) {
|
|
171
128
|
const root = getLocalSandboxRoot(params.sandboxId);
|
|
172
|
-
await
|
|
129
|
+
await fs.rm(root, { recursive: true, force: true });
|
|
173
130
|
localSandboxRoots.delete(params.sandboxId);
|
|
174
131
|
return;
|
|
175
132
|
}
|
|
176
|
-
const db = (await
|
|
177
|
-
const { SandboxService } = (await
|
|
133
|
+
const db = (await getContextRuntime(params.env)).db;
|
|
134
|
+
const { SandboxService } = (await import("@ekairos/sandbox"));
|
|
178
135
|
const service = new SandboxService(db);
|
|
179
136
|
const result = await service.stopSandbox(params.sandboxId);
|
|
180
137
|
if (!result.ok)
|
|
181
138
|
throw new Error(result.error);
|
|
182
139
|
}
|
|
183
|
-
//# sourceMappingURL=steps.js.map
|
package/dist/sandbox.steps.d.ts
CHANGED
|
@@ -1,2 +1 @@
|
|
|
1
|
-
export * from "./sandbox/steps";
|
|
2
|
-
//# sourceMappingURL=sandbox.steps.d.ts.map
|
|
1
|
+
export * from "./sandbox/steps.js";
|
package/dist/sandbox.steps.js
CHANGED
|
@@ -1,18 +1 @@
|
|
|
1
|
-
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./sandbox/steps"), exports);
|
|
18
|
-
//# sourceMappingURL=sandbox.steps.js.map
|
|
1
|
+
export * from "./sandbox/steps.js";
|
package/dist/schema.d.ts
CHANGED
package/dist/schema.js
CHANGED
|
@@ -1,29 +1,26 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.datasetDomain = void 0;
|
|
4
|
-
const core_1 = require("@instantdb/core");
|
|
5
|
-
const domain_1 = require("@ekairos/domain");
|
|
1
|
+
import { i } from "@instantdb/core";
|
|
2
|
+
import { domain } from "@ekairos/domain";
|
|
6
3
|
const entities = {
|
|
7
|
-
dataset_datasets:
|
|
8
|
-
datasetId:
|
|
9
|
-
sandboxId:
|
|
10
|
-
status:
|
|
11
|
-
createdAt:
|
|
12
|
-
updatedAt:
|
|
13
|
-
organizationId:
|
|
14
|
-
title:
|
|
15
|
-
sources:
|
|
16
|
-
sourceKinds:
|
|
17
|
-
instructions:
|
|
18
|
-
analysis:
|
|
19
|
-
schema:
|
|
20
|
-
calculatedTotalRows:
|
|
21
|
-
actualGeneratedRowCount:
|
|
4
|
+
dataset_datasets: i.entity({
|
|
5
|
+
datasetId: i.string().unique().indexed(),
|
|
6
|
+
sandboxId: i.string().optional().indexed(),
|
|
7
|
+
status: i.string().optional().indexed(),
|
|
8
|
+
createdAt: i.number().optional().indexed(),
|
|
9
|
+
updatedAt: i.number().optional(),
|
|
10
|
+
organizationId: i.string().optional().indexed(),
|
|
11
|
+
title: i.string().optional(),
|
|
12
|
+
sources: i.json().optional(),
|
|
13
|
+
sourceKinds: i.json().optional(),
|
|
14
|
+
instructions: i.string().optional(),
|
|
15
|
+
analysis: i.json().optional(),
|
|
16
|
+
schema: i.json().optional(),
|
|
17
|
+
calculatedTotalRows: i.number().optional(),
|
|
18
|
+
actualGeneratedRowCount: i.number().optional(),
|
|
22
19
|
}),
|
|
23
|
-
dataset_records:
|
|
24
|
-
rowContent:
|
|
25
|
-
order:
|
|
26
|
-
createdAt:
|
|
20
|
+
dataset_records: i.entity({
|
|
21
|
+
rowContent: i.json(),
|
|
22
|
+
order: i.number().indexed(),
|
|
23
|
+
createdAt: i.number(),
|
|
27
24
|
}),
|
|
28
25
|
};
|
|
29
26
|
const links = {
|
|
@@ -37,9 +34,8 @@ const links = {
|
|
|
37
34
|
},
|
|
38
35
|
};
|
|
39
36
|
const rooms = {};
|
|
40
|
-
|
|
37
|
+
export const datasetDomain = domain("dataset").withSchema({
|
|
41
38
|
entities,
|
|
42
39
|
links,
|
|
43
40
|
rooms,
|
|
44
41
|
});
|
|
45
|
-
//# sourceMappingURL=schema.js.map
|
package/dist/service.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { InstantAdminDatabase } from "@instantdb/admin";
|
|
2
2
|
import { SchemaOf } from "@ekairos/domain";
|
|
3
|
-
import { datasetDomain } from "./schema";
|
|
3
|
+
import { datasetDomain } from "./schema.js";
|
|
4
4
|
export type ServiceResult<T = any> = {
|
|
5
5
|
ok: true;
|
|
6
6
|
data: T;
|
|
@@ -87,4 +87,3 @@ export declare class DatasetService {
|
|
|
87
87
|
readOne(datasetId: string): Promise<ServiceResult<any | null>>;
|
|
88
88
|
}
|
|
89
89
|
export {};
|
|
90
|
-
//# sourceMappingURL=service.d.ts.map
|
package/dist/service.js
CHANGED
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.DatasetService = void 0;
|
|
4
|
-
const admin_1 = require("@instantdb/admin");
|
|
5
|
-
class DatasetService {
|
|
1
|
+
import { id as newId } from "@instantdb/admin";
|
|
2
|
+
export class DatasetService {
|
|
6
3
|
constructor(db) {
|
|
7
4
|
this.db = db;
|
|
8
5
|
}
|
|
@@ -30,9 +27,9 @@ class DatasetService {
|
|
|
30
27
|
}
|
|
31
28
|
async createDataset(params) {
|
|
32
29
|
try {
|
|
33
|
-
const datasetId = params.id ?? (
|
|
30
|
+
const datasetId = params.id ?? newId();
|
|
34
31
|
const existing = await this.resolveDatasetEntityId(datasetId);
|
|
35
|
-
const entityId = existing.ok ? existing.data : (
|
|
32
|
+
const entityId = existing.ok ? existing.data : newId();
|
|
36
33
|
const mutations = [];
|
|
37
34
|
mutations.push(this.db.tx.dataset_datasets[entityId].update({
|
|
38
35
|
datasetId,
|
|
@@ -76,7 +73,7 @@ class DatasetService {
|
|
|
76
73
|
return resolved;
|
|
77
74
|
const mutations = [];
|
|
78
75
|
for (const record of params.records) {
|
|
79
|
-
const recordId = (
|
|
76
|
+
const recordId = newId();
|
|
80
77
|
mutations.push(this.db.tx.dataset_records[recordId].update({
|
|
81
78
|
rowContent: record.rowContent,
|
|
82
79
|
order: record.order,
|
|
@@ -507,5 +504,3 @@ class DatasetService {
|
|
|
507
504
|
}
|
|
508
505
|
}
|
|
509
506
|
}
|
|
510
|
-
exports.DatasetService = DatasetService;
|
|
511
|
-
//# sourceMappingURL=service.js.map
|
package/dist/skill.d.ts
CHANGED
package/dist/skill.js
CHANGED
|
@@ -1,48 +1,44 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.buildDatasetSkillPackage = buildDatasetSkillPackage;
|
|
7
|
-
const node_fs_1 = require("node:fs");
|
|
8
|
-
const node_path_1 = __importDefault(require("node:path"));
|
|
1
|
+
import { readdirSync, readFileSync, statSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
9
4
|
function walkFiles(rootDir, currentDir = rootDir) {
|
|
10
|
-
const entries =
|
|
5
|
+
const entries = readdirSync(currentDir, { withFileTypes: true });
|
|
11
6
|
const files = [];
|
|
12
7
|
for (const entry of entries) {
|
|
13
|
-
const absPath =
|
|
8
|
+
const absPath = path.join(currentDir, entry.name);
|
|
14
9
|
if (entry.isDirectory()) {
|
|
15
10
|
files.push(...walkFiles(rootDir, absPath));
|
|
16
11
|
continue;
|
|
17
12
|
}
|
|
18
13
|
if (entry.isFile()) {
|
|
19
|
-
files.push(
|
|
14
|
+
files.push(path.relative(rootDir, absPath).replace(/\\/g, "/"));
|
|
20
15
|
}
|
|
21
16
|
}
|
|
22
17
|
return files;
|
|
23
18
|
}
|
|
24
19
|
function resolveDatasetSkillRoot() {
|
|
25
|
-
const
|
|
20
|
+
const currentDir = path.dirname(fileURLToPath(import.meta.url));
|
|
21
|
+
const fromDist = path.resolve(currentDir, "..", "skill");
|
|
26
22
|
if (statExists(fromDist))
|
|
27
23
|
return fromDist;
|
|
28
|
-
const fromSrc =
|
|
24
|
+
const fromSrc = path.resolve(currentDir, "..", "..", "skill");
|
|
29
25
|
if (statExists(fromSrc))
|
|
30
26
|
return fromSrc;
|
|
31
27
|
throw new Error("dataset_skill_root_not_found");
|
|
32
28
|
}
|
|
33
29
|
function statExists(candidate) {
|
|
34
30
|
try {
|
|
35
|
-
return
|
|
31
|
+
return statSync(candidate).isDirectory();
|
|
36
32
|
}
|
|
37
33
|
catch {
|
|
38
34
|
return false;
|
|
39
35
|
}
|
|
40
36
|
}
|
|
41
|
-
function buildDatasetSkillPackage() {
|
|
37
|
+
export function buildDatasetSkillPackage() {
|
|
42
38
|
const skillRoot = resolveDatasetSkillRoot();
|
|
43
39
|
const files = walkFiles(skillRoot).map((relativePath) => ({
|
|
44
40
|
path: relativePath,
|
|
45
|
-
contentBase64:
|
|
41
|
+
contentBase64: readFileSync(path.join(skillRoot, relativePath)).toString("base64"),
|
|
46
42
|
}));
|
|
47
43
|
return {
|
|
48
44
|
name: "dataset",
|
|
@@ -50,4 +46,3 @@ function buildDatasetSkillPackage() {
|
|
|
50
46
|
files,
|
|
51
47
|
};
|
|
52
48
|
}
|
|
53
|
-
//# sourceMappingURL=skill.js.map
|
|
@@ -20,4 +20,3 @@ interface PreviewOptions {
|
|
|
20
20
|
}
|
|
21
21
|
export declare function generateSourcePreview(env: any, sandboxId: string, sourcePath: string, datasetId: string, options?: PreviewOptions): Promise<TransformSourcePreviewContext>;
|
|
22
22
|
export {};
|
|
23
|
-
//# sourceMappingURL=filepreview.d.ts.map
|
|
@@ -1,13 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.generateSourcePreview = generateSourcePreview;
|
|
4
|
-
const datasetFiles_1 = require("../datasetFiles");
|
|
5
|
-
const steps_1 = require("../sandbox/steps");
|
|
1
|
+
import { getDatasetWorkstation } from "../datasetFiles.js";
|
|
2
|
+
import { runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep } from "../sandbox/steps.js";
|
|
6
3
|
const DEFAULT_HEAD_LINES = 50;
|
|
7
4
|
async function runPythonSnippet(env, sandboxId, datasetId, scriptName, code, args, description) {
|
|
8
|
-
const workstation =
|
|
5
|
+
const workstation = getDatasetWorkstation(datasetId);
|
|
9
6
|
const scriptPath = `${workstation}/${scriptName}.py`;
|
|
10
|
-
await
|
|
7
|
+
await writeDatasetSandboxFilesStep({
|
|
11
8
|
env,
|
|
12
9
|
sandboxId,
|
|
13
10
|
files: [
|
|
@@ -17,7 +14,7 @@ async function runPythonSnippet(env, sandboxId, datasetId, scriptName, code, arg
|
|
|
17
14
|
},
|
|
18
15
|
],
|
|
19
16
|
});
|
|
20
|
-
const result = await
|
|
17
|
+
const result = await runDatasetSandboxCommandStep({
|
|
21
18
|
env,
|
|
22
19
|
sandboxId,
|
|
23
20
|
cmd: "python",
|
|
@@ -33,7 +30,7 @@ async function runPythonSnippet(env, sandboxId, datasetId, scriptName, code, arg
|
|
|
33
30
|
stderr,
|
|
34
31
|
};
|
|
35
32
|
}
|
|
36
|
-
async function generateSourcePreview(env, sandboxId, sourcePath, datasetId, options = {}) {
|
|
33
|
+
export async function generateSourcePreview(env, sandboxId, sourcePath, datasetId, options = {}) {
|
|
37
34
|
const context = {
|
|
38
35
|
totalRows: 0,
|
|
39
36
|
};
|
|
@@ -96,4 +93,3 @@ except Exception as e:
|
|
|
96
93
|
context.head = head;
|
|
97
94
|
return context;
|
|
98
95
|
}
|
|
99
|
-
//# sourceMappingURL=filepreview.js.map
|
|
@@ -1,3 +1,2 @@
|
|
|
1
|
-
export { createTransformDatasetStory, type TransformDatasetAgentParams, type TransformDatasetContext, } from "./transform-dataset.agent";
|
|
2
|
-
export { transformDataset, type TransformDatasetInput, type TransformDatasetResult, } from "./transformDataset";
|
|
3
|
-
//# sourceMappingURL=index.d.ts.map
|
|
1
|
+
export { createTransformDatasetStory, type TransformDatasetAgentParams, type TransformDatasetContext, } from "./transform-dataset.agent.js";
|
|
2
|
+
export { transformDataset, type TransformDatasetInput, type TransformDatasetResult, } from "./transformDataset.js";
|
package/dist/transform/index.js
CHANGED
|
@@ -1,8 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.transformDataset = exports.createTransformDatasetStory = void 0;
|
|
4
|
-
var transform_dataset_agent_1 = require("./transform-dataset.agent");
|
|
5
|
-
Object.defineProperty(exports, "createTransformDatasetStory", { enumerable: true, get: function () { return transform_dataset_agent_1.createTransformDatasetStory; } });
|
|
6
|
-
var transformDataset_1 = require("./transformDataset");
|
|
7
|
-
Object.defineProperty(exports, "transformDataset", { enumerable: true, get: function () { return transformDataset_1.transformDataset; } });
|
|
8
|
-
//# sourceMappingURL=index.js.map
|
|
1
|
+
export { createTransformDatasetStory, } from "./transform-dataset.agent.js";
|
|
2
|
+
export { transformDataset, } from "./transformDataset.js";
|
|
@@ -1,31 +1,28 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.buildTransformDatasetPrompt = buildTransformDatasetPrompt;
|
|
4
|
-
const xmlbuilder2_1 = require("xmlbuilder2");
|
|
1
|
+
import { create } from "xmlbuilder2";
|
|
5
2
|
function buildRole() {
|
|
6
|
-
let xml =
|
|
3
|
+
let xml = create()
|
|
7
4
|
.ele("Role")
|
|
8
5
|
.txt("You are a dataset transformer. Your goal is to read one or more existing JSONL datasets and produce a NEW JSONL dataset whose records conform exactly to the provided output schema.")
|
|
9
6
|
.up();
|
|
10
7
|
return xml.end({ prettyPrint: true, headless: true });
|
|
11
8
|
}
|
|
12
9
|
function buildGoal() {
|
|
13
|
-
let xml =
|
|
10
|
+
let xml = create()
|
|
14
11
|
.ele("Goal")
|
|
15
12
|
.txt("Transform the source dataset(s) (JSONL with {type:'row', data:{...}} per line) into a new dataset strictly matching the output schema. Save to output.jsonl in the dataset workstation. Each line must remain a single JSON object representing one record. You may need to combine, filter, or reshape data from multiple source datasets.")
|
|
16
13
|
.up();
|
|
17
14
|
return xml.end({ prettyPrint: true, headless: true });
|
|
18
15
|
}
|
|
19
16
|
function buildContextSection(context) {
|
|
20
|
-
let xml =
|
|
17
|
+
let xml = create()
|
|
21
18
|
.ele("Context")
|
|
22
19
|
.ele("DatasetId").txt(context.datasetId).up();
|
|
23
|
-
let sourcesXml =
|
|
20
|
+
let sourcesXml = create().ele("SourceDatasets");
|
|
24
21
|
for (const sourceId of context.sourceDatasetIds) {
|
|
25
22
|
sourcesXml = sourcesXml.ele("SourceDatasetId").txt(sourceId).up();
|
|
26
23
|
}
|
|
27
24
|
xml = xml.import(sourcesXml.first());
|
|
28
|
-
let sandboxXml =
|
|
25
|
+
let sandboxXml = create().ele("Sandbox");
|
|
29
26
|
for (const sourcePathInfo of context.sandboxConfig.sourcePaths) {
|
|
30
27
|
sandboxXml = sandboxXml.ele("SourceFile")
|
|
31
28
|
.ele("DatasetId").txt(sourcePathInfo.datasetId).up()
|
|
@@ -35,10 +32,10 @@ function buildContextSection(context) {
|
|
|
35
32
|
sandboxXml = sandboxXml.ele("OutputPath").txt(context.sandboxConfig.outputPath).up();
|
|
36
33
|
xml = xml.import(sandboxXml.first());
|
|
37
34
|
if (context.sourcePreviews && context.sourcePreviews.length > 0) {
|
|
38
|
-
let previewsXml =
|
|
35
|
+
let previewsXml = create().ele("SourcePreviews");
|
|
39
36
|
for (const sourcePreviewInfo of context.sourcePreviews) {
|
|
40
37
|
const sp = sourcePreviewInfo.preview;
|
|
41
|
-
let px =
|
|
38
|
+
let px = create().ele("SourcePreview")
|
|
42
39
|
.ele("DatasetId").txt(sourcePreviewInfo.datasetId).up()
|
|
43
40
|
.ele("TotalRows").txt(String(sp.totalRows)).up();
|
|
44
41
|
if (sp.metadata) {
|
|
@@ -71,7 +68,7 @@ function buildContextSection(context) {
|
|
|
71
68
|
xml = xml.import(previewsXml.first());
|
|
72
69
|
}
|
|
73
70
|
if (Array.isArray(context.errors) && context.errors.length > 0) {
|
|
74
|
-
let ex =
|
|
71
|
+
let ex = create().ele("PreviousErrors");
|
|
75
72
|
for (const e of context.errors) {
|
|
76
73
|
ex = ex.ele("Error").txt(e).up();
|
|
77
74
|
}
|
|
@@ -81,7 +78,7 @@ function buildContextSection(context) {
|
|
|
81
78
|
return xml.end({ prettyPrint: true, headless: true });
|
|
82
79
|
}
|
|
83
80
|
function buildOutputSchemaSection(context) {
|
|
84
|
-
let xml =
|
|
81
|
+
let xml = create()
|
|
85
82
|
.ele("OutputSchema")
|
|
86
83
|
.ele("JsonSchema").txt(JSON.stringify(context.outputSchema?.schema ?? context.outputSchema ?? {}, null, 2)).up()
|
|
87
84
|
.up();
|
|
@@ -92,7 +89,7 @@ function buildInstructions(context) {
|
|
|
92
89
|
const multipleSourcesNote = context.sourceDatasetIds.length > 1
|
|
93
90
|
? "You have multiple source datasets available. You may need to read, join, filter, or combine data from them to produce the output."
|
|
94
91
|
: "";
|
|
95
|
-
let xml =
|
|
92
|
+
let xml = create()
|
|
96
93
|
.ele("Instructions")
|
|
97
94
|
.ele("Workflow")
|
|
98
95
|
.ele("Step", { number: "1", name: "Inspect Source" })
|
|
@@ -121,7 +118,7 @@ function buildInstructions(context) {
|
|
|
121
118
|
.up();
|
|
122
119
|
return xml.end({ prettyPrint: true, headless: true });
|
|
123
120
|
}
|
|
124
|
-
function buildTransformDatasetPrompt(context) {
|
|
121
|
+
export function buildTransformDatasetPrompt(context) {
|
|
125
122
|
const sections = [];
|
|
126
123
|
sections.push(buildRole());
|
|
127
124
|
sections.push("");
|
|
@@ -134,4 +131,3 @@ function buildTransformDatasetPrompt(context) {
|
|
|
134
131
|
sections.push(buildInstructions(context));
|
|
135
132
|
return sections.join("\n");
|
|
136
133
|
}
|
|
137
|
-
//# sourceMappingURL=prompts.js.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type ContextReactor } from "@ekairos/events";
|
|
2
|
-
import { TransformSourcePreviewContext } from "./filepreview";
|
|
2
|
+
import { TransformSourcePreviewContext } from "./filepreview.js";
|
|
3
3
|
export type TransformDatasetContext = {
|
|
4
4
|
datasetId: string;
|
|
5
5
|
sourceDatasetIds: string[];
|
|
@@ -56,4 +56,3 @@ export declare function createTransformDatasetStory<Env extends {
|
|
|
56
56
|
}>;
|
|
57
57
|
story: any;
|
|
58
58
|
};
|
|
59
|
-
//# sourceMappingURL=transform-dataset.agent.d.ts.map
|