@ekairos/dataset 1.22.40-beta.development.0 → 1.22.40

package/dist/dataset.js

@@ -1,55 +1,63 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.dataset = dataset;
-const admin_1 = require("@instantdb/admin");
-const instructions_1 = require("./builder/instructions");
-const materialize_1 = require("./builder/materialize");
-const persistence_1 = require("./builder/persistence");
-function dataset(runtime, options = {}) {
+import { buildObjectOutputInstructions } from "./builder/instructions.js";
+import { resolveDatasetResourceContext } from "./builder/context.js";
+import { createDatasetId } from "./id.js";
+import { completeDatasetStep, materializeDerivedDataset, materializeSingleFileLikeResource, } from "./builder/materialize.js";
+import { materializeQueryResource } from "./builder/materializeQuery.js";
+import { createDatasetBuildResult, finalizeBuildResult, } from "./builder/persistence.js";
+export function dataset(runtime, options = {}) {
     const datasetId = normalizeDatasetId(options.datasetId);
     const typedRuntime = runtime;
     const state = {
         runtime: typedRuntime,
         env: typedRuntime.env,
-        sources: [],
+        resources: [],
         output: "rows",
         inferSchema: false,
+        durable: options.durable,
         first: false,
     };
     const api = {
         datasetId,
-        fromFile(source) {
-            state.sources.push({ kind: "file", ...source });
+        fromFile(resource) {
+            state.resources.push({ kind: "file", ...resource });
             return api;
         },
-        fromText(source) {
-            state.sources.push({ kind: "text", ...source });
+        fromText(resource) {
+            state.resources.push({ kind: "text", ...resource });
             return api;
         },
-        fromDataset(source) {
-            state.sources.push({ kind: "dataset", ...source });
+        fromDataset(resource) {
+            state.resources.push({ kind: "dataset", ...resource });
             return api;
         },
-        from(...sources) {
-            for (const source of sources) {
-                if ("kind" in source) {
-                    state.sources.push(source);
+        fromContext(context) {
+            state.resources.push({ kind: "context", ...context });
+            return api;
+        },
+        from(...resources) {
+            for (const resource of resources) {
+                if ("kind" in resource) {
+                    state.resources.push(resource);
                     continue;
                 }
-                if ("fileId" in source) {
-                    state.sources.push({ kind: "file", ...source });
+                if ("fileId" in resource) {
+                    state.resources.push({ kind: "file", ...resource });
                     continue;
                 }
-                if ("datasetId" in source) {
-                    state.sources.push({ kind: "dataset", ...source });
+                if ("datasetId" in resource) {
+                    state.resources.push({ kind: "dataset", ...resource });
                     continue;
                 }
-                state.sources.push({ kind: "text", ...source });
+                if ("id" in resource || "key" in resource) {
+                    state.resources.push({ kind: "context", ...resource });
+                    continue;
+                }
+                state.resources.push({ kind: "text", ...resource });
             }
             return api;
         },
-        fromQuery(domain, source) {
-            state.sources.push({ kind: "query", domain, ...source });
+        fromQuery(domain, resource) {
+            state.resources.push({ kind: "query", domain, ...resource });
             return api;
         },
         title(title) {
@@ -97,57 +105,72 @@ function dataset(runtime, options = {}) {
             return api;
         },
         async build(options) {
-            if (state.sources.length === 0) {
-                throw new Error("dataset_sources_required");
+            if (state.resources.length === 0) {
+                throw new Error("dataset_resources_required");
             }
             const targetDatasetId = options?.datasetId
                 ? normalizeDatasetId(options.datasetId)
                 : datasetId;
-            const effectiveState = state.output === "object"
+            const stateWithBuildOptions = {
+                ...state,
+                durable: options?.durable ?? state.durable,
+            };
+            const context = await resolveDatasetResourceContext(typedRuntime, targetDatasetId, stateWithBuildOptions.resources);
+            stateWithBuildOptions.resources = context.resources;
+            stateWithBuildOptions.contextId = context.contextId;
+            stateWithBuildOptions.contextResources = context.contextResources;
+            const effectiveState = stateWithBuildOptions.output === "object"
                 ? {
-                    ...state,
+                    ...stateWithBuildOptions,
                     first: true,
-                    instructions: (0, instructions_1.buildObjectOutputInstructions)(state.instructions),
+                    instructions: buildObjectOutputInstructions(stateWithBuildOptions.instructions),
                 }
-                : state;
-            const onlySource = effectiveState.sources[0];
-            const isSingleSource = effectiveState.sources.length === 1;
+                : stateWithBuildOptions;
+            const onlyResource = effectiveState.resources[0];
+            const isSingleResource = effectiveState.resources.length === 1;
             const hasInstructions = Boolean(String(effectiveState.instructions ?? "").trim());
-            if (isSingleSource && onlySource.kind === "query" && !hasInstructions) {
-                await (0, materialize_1.materializeQuerySource)(effectiveState.runtime, onlySource, {
+            if (isSingleResource && onlyResource.kind === "query" && !hasInstructions) {
+                await materializeQueryResource(effectiveState.runtime, onlyResource, {
                     datasetId: targetDatasetId,
                     sandboxId: effectiveState.sandboxId,
                     schema: effectiveState.outputSchema,
-                    title: effectiveState.title ?? onlySource.title,
+                    title: effectiveState.title ?? onlyResource.title,
                     instructions: effectiveState.instructions,
                     first: effectiveState.first,
+                    contextId: effectiveState.contextId ?? "",
                 });
-                return finalizeOutputResult(await (0, persistence_1.finalizeBuildResult)(effectiveState.runtime, targetDatasetId, effectiveState.first), effectiveState.output);
+                return finalizeOutputResult(await finalizeBuildResult(effectiveState.runtime, targetDatasetId, effectiveState.first), effectiveState.output);
             }
-            if (isSingleSource && (onlySource.kind === "file" || onlySource.kind === "text")) {
-                if (!effectiveState.sandboxId) {
-                    throw new Error("dataset_sandbox_required");
-                }
+            if (isSingleResource && (onlyResource.kind === "file" || onlyResource.kind === "text")) {
                 if (!effectiveState.reactor) {
                     throw new Error("dataset_reactor_required");
                 }
-                await (0, materialize_1.materializeSingleFileLikeSource)(effectiveState, onlySource, targetDatasetId);
-                return finalizeOutputResult(await (0, persistence_1.finalizeBuildResult)(effectiveState.runtime, targetDatasetId, effectiveState.first), effectiveState.output);
-            }
-            if (!effectiveState.sandboxId) {
-                throw new Error("dataset_sandbox_required");
+                await materializeSingleFileLikeResource(effectiveState, onlyResource, targetDatasetId);
+                const completed = await completeDatasetStep({
+                    runtime: effectiveState.runtime,
+                    datasetId: targetDatasetId,
+                    schema: effectiveState.outputSchema,
+                    first: effectiveState.first,
+                });
+                return finalizeOutputResult(createDatasetBuildResult(effectiveState.runtime, completed), effectiveState.output);
             }
             if (!effectiveState.reactor) {
                 throw new Error("dataset_reactor_required");
             }
-            await (0, materialize_1.materializeDerivedDataset)(effectiveState, targetDatasetId);
-            return finalizeOutputResult(await (0, persistence_1.finalizeBuildResult)(effectiveState.runtime, targetDatasetId, effectiveState.first), effectiveState.output);
+            await materializeDerivedDataset(effectiveState, targetDatasetId);
+            const completed = await completeDatasetStep({
+                runtime: effectiveState.runtime,
+                datasetId: targetDatasetId,
+                schema: effectiveState.outputSchema,
+                first: effectiveState.first,
+            });
+            return finalizeOutputResult(createDatasetBuildResult(effectiveState.runtime, completed), effectiveState.output);
         },
     };
     return api;
 }
 function normalizeDatasetId(datasetId) {
-    const normalized = String(datasetId ?? (0, admin_1.id)()).trim();
+    const normalized = String(datasetId ?? createDatasetId()).trim();
     if (!normalized) {
         throw new Error("dataset_id_required");
     }
@@ -161,4 +184,3 @@ function finalizeOutputResult(result, output) {
         object: result.firstRow ?? null,
     };
 }
-//# sourceMappingURL=dataset.js.map

package/dist/datasetFiles.d.ts

@@ -1,5 +1,9 @@
 export declare const DATASET_OUTPUT_FILE_NAME = "output.jsonl";
 export declare function getDatasetWorkdirBase(): string;
 export declare function getDatasetWorkstation(datasetId: string): string;
+export declare function getDatasetResourcesDir(datasetId: string): string;
+export declare function getDatasetScriptsDir(datasetId: string): string;
+export declare function getDatasetArtifactsDir(datasetId: string): string;
+export declare function getDatasetLogsDir(datasetId: string): string;
+export declare function getDatasetStandardDirs(datasetId: string): string[];
 export declare function getDatasetOutputPath(datasetId: string): string;
-//# sourceMappingURL=datasetFiles.d.ts.map

package/dist/datasetFiles.js

@@ -1,33 +1,35 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DATASET_OUTPUT_FILE_NAME = void 0;
-exports.getDatasetWorkdirBase = getDatasetWorkdirBase;
-exports.getDatasetWorkstation = getDatasetWorkstation;
-exports.getDatasetOutputPath = getDatasetOutputPath;
-exports.DATASET_OUTPUT_FILE_NAME = "output.jsonl";
-const DEFAULT_VERCEL_WORKDIR_BASE = "/vercel/sandbox/datasets";
-const DEFAULT_DAYTONA_WORKDIR_BASE = "/home/daytona/.ekairos/datasets";
-const DEFAULT_SPRITES_WORKDIR_BASE = "/workspace/.ekairos/datasets";
+export const DATASET_OUTPUT_FILE_NAME = "output.jsonl";
+const DATASET_WORKDIR_BASE = "/tmp/ekairos/dataset";
 function trimTrailingSlash(value) {
     return value.endsWith("/") ? value.slice(0, -1) : value;
 }
-function getDatasetWorkdirBase() {
-    const explicit = String(process.env.DATASET_SANDBOX_WORKDIR_BASE ?? "").trim();
-    if (explicit)
-        return trimTrailingSlash(explicit);
-    const provider = String(process.env.SANDBOX_PROVIDER ?? "").trim().toLowerCase();
-    if (provider === "daytona")
-        return DEFAULT_DAYTONA_WORKDIR_BASE;
-    if (provider === "vercel")
-        return DEFAULT_VERCEL_WORKDIR_BASE;
-    if (provider === "sprites")
-        return DEFAULT_SPRITES_WORKDIR_BASE;
-    return DEFAULT_VERCEL_WORKDIR_BASE;
-}
-function getDatasetWorkstation(datasetId) {
+export function getDatasetWorkdirBase() {
+    return trimTrailingSlash(DATASET_WORKDIR_BASE);
+}
+export function getDatasetWorkstation(datasetId) {
     return `${getDatasetWorkdirBase()}/${datasetId}`;
 }
-function getDatasetOutputPath(datasetId) {
-    return `${getDatasetWorkstation(datasetId)}/${exports.DATASET_OUTPUT_FILE_NAME}`;
+export function getDatasetResourcesDir(datasetId) {
+    return `${getDatasetWorkstation(datasetId)}/resources`;
+}
+export function getDatasetScriptsDir(datasetId) {
+    return `${getDatasetWorkstation(datasetId)}/scripts`;
+}
+export function getDatasetArtifactsDir(datasetId) {
+    return `${getDatasetWorkstation(datasetId)}/artifacts`;
+}
+export function getDatasetLogsDir(datasetId) {
+    return `${getDatasetWorkstation(datasetId)}/logs`;
+}
+export function getDatasetStandardDirs(datasetId) {
+    return [
+        getDatasetWorkstation(datasetId),
+        getDatasetResourcesDir(datasetId),
+        getDatasetScriptsDir(datasetId),
+        getDatasetArtifactsDir(datasetId),
+        getDatasetLogsDir(datasetId),
+    ];
+}
+export function getDatasetOutputPath(datasetId) {
+    return `${getDatasetWorkstation(datasetId)}/${DATASET_OUTPUT_FILE_NAME}`;
 }
-//# sourceMappingURL=datasetFiles.js.map

package/dist/defineNotation.tool.d.ts

@@ -0,0 +1,49 @@
+interface DefineNotationToolParams {
+    datasetId: string;
+    runtime: any;
+}
+/**
+ * defineNotation — author or REFINE the formal DEFINITION of the dataset.
+ *
+ * A dataset has two co-equal faces: its formal definition (the notation —
+ * the proposition that defines the set, in LaTeX) and its materialization
+ * (the rows + the code that produces them). They sit at the SAME level: the
+ * definition is not a side note about the data, it IS the dataset stated
+ * intensionally. The same notation is the PLAN (you state it first and the
+ * materialization realizes it) and, finalized, the RESULT (it describes what
+ * you produced).
+ *
+ * Call it FIRST with the initial definition derived from the resources, and
+ * AGAIN whenever the analysis discovers new sets, variables, constraints or
+ * corrections — every call keeps the prior version in history. Mark the last
+ * call with final=true so the definition describes the produced dataset.
+ * Predicates may be formal/semantic (trusted); the few that are arithmetic
+ * MAY carry optional advisory evidence.
+ */
+export declare function createDefineNotationTool({ datasetId, runtime }: DefineNotationToolParams): import("ai").Tool<{
+    latex: string;
+    symbols: {
+        name: string;
+        kind: "function" | "set" | "variable" | "constant" | "predicate";
+        description: string;
+        latex?: string | undefined;
+    }[];
+    predicates: {
+        id: string;
+        description: string;
+        latex: string;
+        checkJson?: string | undefined;
+    }[];
+    reason: string;
+    final?: boolean | undefined;
+}, {
+    success: boolean;
+    error: string;
+} | {
+    warning?: string | undefined;
+    success: boolean;
+    version: number;
+    status: import("./notation.js").DatasetNotationStatus;
+    error?: undefined;
+}>;
+export {};

package/dist/defineNotation.tool.js

@@ -0,0 +1,154 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { DatasetService } from "./service.js";
+import { datasetDomain } from "./schema.js";
+import { reviseDatasetNotation, } from "./notation.js";
+const symbolSchema = z.object({
+    name: z.string().describe("Plain identifier, e.g. 'D', 'Orders', 'w'"),
+    latex: z
+        .string()
+        .optional()
+        .describe("LaTeX for the symbol, e.g. '\\\\mathcal{D}' (defaults to the name)"),
+    kind: z.enum(["set", "variable", "function", "constant", "predicate"]),
+    description: z.string().describe("What this symbol denotes in the data"),
+});
+const predicateSchema = z.object({
+    id: z.string().describe("Stable id, e.g. 'p1', 'cardinality'"),
+    description: z.string().describe("The claim in plain language"),
+    latex: z
+        .string()
+        .describe("The claim in LaTeX, e.g. '\\\\forall r \\\\in D: r.amount > 0'"),
+    checkJson: z
+        .string()
+        .optional()
+        .describe([
+        "OPTIONAL arithmetic form of the claim as a JSON string, used only for",
+        "advisory evidence over the produced rows (not a verdict). Shapes:",
+        '{"kind":"row_count","op":"=","value":124}',
+        '{"kind":"field_type","field":"amount","type":"number","allowNull":true}',
+        '{"kind":"field_range","field":"amount","min":0}',
+        '{"kind":"field_in","field":"status","values":["paid","void"]}',
+        '{"kind":"field_nonnull","field":"orderId"}',
+        '{"kind":"field_matches","field":"sku","pattern":"^[A-Z0-9-]+$"}',
+        '{"kind":"unique","fields":["orderId"]}',
+        '{"kind":"aggregate","fn":"sum","field":"amount","op":">=","value":0}',
+        'Propositional composition: {"kind":"and"|"or","checks":[...]},',
+        '{"kind":"not","check":...}, {"kind":"implies","if":...,"then":...}.',
+        "Fields support dot-paths into nested records (company.taxId).",
+        "Omit for formal/semantic claims (the normal case) — they are trusted.",
+    ].join(" ")),
+});
+async function getDatasetService(runtime) {
+    const scoped = await runtime.use(datasetDomain);
+    return new DatasetService(scoped.db);
+}
+/**
+ * defineNotation — author or REFINE the formal DEFINITION of the dataset.
+ *
+ * A dataset has two co-equal faces: its formal definition (the notation —
+ * the proposition that defines the set, in LaTeX) and its materialization
+ * (the rows + the code that produces them). They sit at the SAME level: the
+ * definition is not a side note about the data, it IS the dataset stated
+ * intensionally. The same notation is the PLAN (you state it first and the
+ * materialization realizes it) and, finalized, the RESULT (it describes what
+ * you produced).
+ *
+ * Call it FIRST with the initial definition derived from the resources, and
+ * AGAIN whenever the analysis discovers new sets, variables, constraints or
+ * corrections — every call keeps the prior version in history. Mark the last
+ * call with final=true so the definition describes the produced dataset.
+ * Predicates may be formal/semantic (trusted); the few that are arithmetic
+ * MAY carry optional advisory evidence.
+ */
+export function createDefineNotationTool({ datasetId, runtime }) {
+    return tool({
+        description: [
+            "Author or refine the formal DEFINITION of the dataset: the dataset as a",
+            "set in LaTeX (set-builder, relational algebra, quantified or even",
+            "semantic predicates) plus the symbols it binds. This definition and the",
+            "materialization (rows + code) are TWO CO-EQUAL FACES of the dataset —",
+            "the definition is the dataset stated intensionally, not a comment on it.",
+            "It is your PLAN (state it before writing any code; the materialization",
+            "realizes it) and, once final, the RESULT (it describes what you",
+            "produced). The definition is a logical proposition, possibly derived —",
+            "it need not be mechanically provable; we trust the formality. State it",
+            "first, refine it on every discovery, and set final=true on the last",
+            "call. For the few predicates that are arithmetic you MAY attach a",
+            "checkJson for optional advisory evidence (non-blocking, never a verdict).",
+        ].join(" "),
+        inputSchema: z.object({
+            latex: z
+                .string()
+                .describe("Main definition of the dataset as a set, in LaTeX. Example: 'D = \\\\{(w,r,t) \\\\mid t = \\\\sum_{o \\\\in Orders} o.amount,\\\\; o.status = paid\\\\}'"),
+            symbols: z.array(symbolSchema).describe("Symbols bound by the definition"),
+            predicates: z
+                .array(predicateSchema)
+                .describe("Claims the set satisfies; attach a checkJson only when arithmetic"),
+            reason: z
+                .string()
+                .describe("What this revision states or what discovery triggered it (or 'initial definition')"),
+            final: z
+                .boolean()
+                .optional()
+                .describe("true when this definition describes the dataset you are about to complete (the RESULT)"),
+        }),
+        execute: async ({ latex, symbols, predicates, reason, final }) => {
+            try {
+                const service = await getDatasetService(runtime);
+                const existing = await service.getDatasetById(datasetId);
+                const previous = (existing.ok ? existing.data?.notation : null);
+                const parsedPredicates = [];
+                const checkErrors = [];
+                for (const predicate of predicates) {
+                    let check;
+                    if (predicate.checkJson) {
+                        try {
+                            check = JSON.parse(predicate.checkJson);
+                            if (!check || typeof check !== "object" || !("kind" in check)) {
+                                throw new Error("check must be an object with a 'kind'");
+                            }
+                        }
+                        catch (error) {
+                            checkErrors.push(`predicate ${predicate.id}: invalid checkJson (${String(error).slice(0, 80)})`);
+                            check = undefined;
+                        }
+                    }
+                    parsedPredicates.push({
+                        id: predicate.id,
+                        description: predicate.description,
+                        latex: predicate.latex,
+                        ...(check ? { check } : {}),
+                    });
+                }
+                const notation = reviseDatasetNotation(previous, {
+                    latex,
+                    symbols: symbols,
+                    predicates: parsedPredicates,
+                    reason,
+                    final,
+                });
+                const update = await service.updateDatasetNotation({ datasetId, notation });
+                if (!update.ok) {
+                    return { success: false, error: update.error };
+                }
+                console.log(`[Dataset ${datasetId}] definition v${notation.version} (${notation.status}): ${reason}`);
+                return {
+                    success: true,
+                    version: notation.version,
+                    status: notation.status,
+                    ...(checkErrors.length
+                        ? {
+                            warning: `some checks were dropped: ${checkErrors.join("; ")}`,
+                        }
+                        : {}),
+                };
+            }
+            catch (error) {
+                return {
+                    success: false,
+                    error: error instanceof Error ? error.message : String(error),
+                };
+            }
+        },
+    });
+}

package/dist/domain.d.ts

@@ -1,2 +1 @@
-export { datasetDomain } from "./schema";
-//# sourceMappingURL=domain.d.ts.map
+export { datasetDomain } from "./schema.js";

package/dist/domain.js

@@ -1,6 +1 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.datasetDomain = void 0;
-var schema_1 = require("./schema");
-Object.defineProperty(exports, "datasetDomain", { enumerable: true, get: function () { return schema_1.datasetDomain; } });
-//# sourceMappingURL=domain.js.map
+export { datasetDomain } from "./schema.js";

package/dist/executeCommand.tool.d.ts

@@ -1,35 +1,7 @@
 interface ExecuteCommandToolParams {
     datasetId: string;
     sandboxId: string;
-    env?: any;
+    runtime: any;
 }
-export declare function createExecuteCommandTool({ datasetId, sandboxId, env }: ExecuteCommandToolParams): import("ai").Tool<{
-    pythonCode: string;
-    scriptName: string;
-}, {
-    success: boolean;
-    exitCode: number;
-    stdout: string;
-    stderr: string;
-    scriptPath: string;
-    error: string;
-    stdoutTruncated: boolean;
-    stderrTruncated: boolean;
-    stdoutOriginalLength: number;
-    stderrOriginalLength: number;
-    message?: undefined;
-} | {
-    success: boolean;
-    exitCode: number;
-    stdout: string;
-    stderr: string;
-    scriptPath: string;
-    message: string;
-    stdoutTruncated: boolean;
-    stderrTruncated: boolean;
-    stdoutOriginalLength: number;
-    stderrOriginalLength: number;
-    error?: undefined;
-}>;
+export declare function createExecuteCommandTool({ datasetId, sandboxId, runtime }: ExecuteCommandToolParams): any;
 export {};
-//# sourceMappingURL=executeCommand.tool.d.ts.map