@ekairos/dataset 1.22.40-beta.development.0 → 1.22.40

package/dist/transform/filepreview.d.ts

@@ -1,4 +1,4 @@
-export type TransformSourcePreviewContext = {
+export type TransformInputPreviewContext = {
     totalRows: number;
     metadata?: {
         description: string;
@@ -18,6 +18,5 @@ export type TransformSourcePreviewContext = {
 interface PreviewOptions {
     headLines?: number;
 }
-export declare function generateSourcePreview(env: any, sandboxId: string, sourcePath: string, datasetId: string, options?: PreviewOptions): Promise<TransformSourcePreviewContext>;
+export declare function generateInputPreview(runtime: any, sandboxId: string, inputPath: string, datasetId: string, options?: PreviewOptions): Promise<TransformInputPreviewContext>;
 export {};
-//# sourceMappingURL=filepreview.d.ts.map

package/dist/transform/filepreview.js

@@ -1,39 +1,23 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateSourcePreview = generateSourcePreview;
-const datasetFiles_1 = require("../datasetFiles");
-const steps_1 = require("../sandbox/steps");
+import { runDatasetSandboxCommandStep } from "../sandbox/steps.js";
 const DEFAULT_HEAD_LINES = 50;
-async function runPythonSnippet(env, sandboxId, datasetId, scriptName, code, args, description) {
-    const workstation = (0, datasetFiles_1.getDatasetWorkstation)(datasetId);
-    const scriptPath = `${workstation}/${scriptName}.py`;
-    await (0, steps_1.writeDatasetSandboxFilesStep)({
-        env,
-        sandboxId,
-        files: [
-            {
-                path: scriptPath,
-                contentBase64: Buffer.from(code, "utf-8").toString("base64"),
-            },
-        ],
-    });
-    const result = await (0, steps_1.runDatasetSandboxCommandStep)({
-        env,
+async function runPythonSnippet(runtime, sandboxId, datasetId, scriptName, code, args, description) {
+    const result = await runDatasetSandboxCommandStep({
+        runtime,
         sandboxId,
         cmd: "python",
-        args: [scriptPath, ...args],
+        args: ["-c", code, ...args],
     });
     const stdout = result.stdout || "";
     const stderr = result.stderr || "";
     return {
         description,
         script: code,
-        command: `python ${scriptPath} ${args.join(" ")}`,
+        command: `python -c <${scriptName}.py> ${args.join(" ")}`,
         stdout,
         stderr,
     };
 }
-async function generateSourcePreview(env, sandboxId, sourcePath, datasetId, options = {}) {
+export async function generateInputPreview(runtime, sandboxId, inputPath, datasetId, options = {}) {
     const context = {
         totalRows: 0,
     };
@@ -57,7 +41,7 @@ try:
 except Exception as e:
     print(str(e))
 `;
-    const meta = await runPythonSnippet(env, sandboxId, datasetId, "jsonl_count", countScript, [sourcePath], "Counts number of JSONL records with type='row'");
+    const meta = await runPythonSnippet(runtime, sandboxId, datasetId, "jsonl_count", countScript, [inputPath], "Counts number of JSONL records with type='row'");
     context.metadata = meta;
     try {
         if (meta.stdout) {
@@ -92,8 +76,7 @@ try:
 except Exception as e:
     print(str(e))
 `;
-    const head = await runPythonSnippet(env, sandboxId, datasetId, "jsonl_head", headScript, [sourcePath, String(headLines)], `Reads the first ${headLines} JSONL row records`);
+    const head = await runPythonSnippet(runtime, sandboxId, datasetId, "jsonl_head", headScript, [inputPath, String(headLines)], `Reads the first ${headLines} JSONL row records`);
     context.head = head;
     return context;
 }
-//# sourceMappingURL=filepreview.js.map

package/dist/transform/index.d.ts

@@ -1,3 +1,2 @@
-export { createTransformDatasetStory, type TransformDatasetAgentParams, type TransformDatasetContext, } from "./transform-dataset.agent";
-export { transformDataset, type TransformDatasetInput, type TransformDatasetResult, } from "./transformDataset";
-//# sourceMappingURL=index.d.ts.map
+export { createTransformDatasetContext, registerTransformDatasetContext, type TransformDatasetAgentParams, type TransformDatasetContext, type TransformDatasetRunOptions, } from "./transform-dataset.agent.js";
+export { transformDataset, type TransformDatasetInput, type TransformDatasetResult, } from "./transformDataset.js";

package/dist/transform/index.js

@@ -1,8 +1,2 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformDataset = exports.createTransformDatasetStory = void 0;
-var transform_dataset_agent_1 = require("./transform-dataset.agent");
-Object.defineProperty(exports, "createTransformDatasetStory", { enumerable: true, get: function () { return transform_dataset_agent_1.createTransformDatasetStory; } });
-var transformDataset_1 = require("./transformDataset");
-Object.defineProperty(exports, "transformDataset", { enumerable: true, get: function () { return transformDataset_1.transformDataset; } });
-//# sourceMappingURL=index.js.map
+export { createTransformDatasetContext, registerTransformDatasetContext, } from "./transform-dataset.agent.js";
+export { transformDataset, } from "./transformDataset.js";

package/dist/transform/prompts.d.ts

@@ -1,35 +1,2 @@
-export type TransformPromptContext = {
-    datasetId: string;
-    sourceDatasetIds: string[];
-    outputSchema: any;
-    sandboxConfig: {
-        sourcePaths: Array<{
-            datasetId: string;
-            path: string;
-        }>;
-        outputPath: string;
-    };
-    sourcePreviews?: Array<{
-        datasetId: string;
-        preview: {
-            totalRows: number;
-            metadata?: {
-                description: string;
-                script: string;
-                command: string;
-                stdout: string;
-                stderr: string;
-            };
-            head?: {
-                description: string;
-                script: string;
-                command: string;
-                stdout: string;
-                stderr: string;
-            };
-        };
-    }>;
-    errors: string[];
-};
+import type { TransformPromptContext } from "./transform-dataset.types.js";
 export declare function buildTransformDatasetPrompt(context: TransformPromptContext): string;
-//# sourceMappingURL=prompts.d.ts.map

package/dist/transform/prompts.js

@@ -1,45 +1,48 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildTransformDatasetPrompt = buildTransformDatasetPrompt;
-const xmlbuilder2_1 = require("xmlbuilder2");
+import { create } from "xmlbuilder2";
 function buildRole() {
-    let xml = (0, xmlbuilder2_1.create)()
+    let xml = create()
         .ele("Role")
-        .txt("You are a dataset transformer. Your goal is to read one or more existing JSONL datasets and produce a NEW JSONL dataset whose records conform exactly to the provided output schema.")
+        .txt("You are a dataset transformer. Your goal is to read one or more input datasets/resources and produce a NEW dataset whose records conform exactly to the provided output schema.")
         .up();
     return xml.end({ prettyPrint: true, headless: true });
 }
 function buildGoal() {
-    let xml = (0, xmlbuilder2_1.create)()
+    let xml = create()
         .ele("Goal")
-        .txt("Transform the source dataset(s) (JSONL with {type:'row', data:{...}} per line) into a new dataset strictly matching the output schema. Save to output.jsonl in the dataset workstation. Each line must remain a single JSON object representing one record. You may need to combine, filter, or reshape data from multiple source datasets.")
+        .txt("Transform the input dataset(s) into a new dataset strictly matching the output schema. Use the lowest-cost direct completion tool that can produce the correct output. Use sandbox command execution only when commands are necessary to inspect, parse, aggregate, join, or compute over files/resources that cannot be handled directly from the visible context and previews.")
         .up();
     return xml.end({ prettyPrint: true, headless: true });
 }
 function buildContextSection(context) {
-    let xml = (0, xmlbuilder2_1.create)()
+    let xml = create()
         .ele("Context")
         .ele("DatasetId").txt(context.datasetId).up();
-    let sourcesXml = (0, xmlbuilder2_1.create)().ele("SourceDatasets");
-    for (const sourceId of context.sourceDatasetIds) {
-        sourcesXml = sourcesXml.ele("SourceDatasetId").txt(sourceId).up();
-    }
-    xml = xml.import(sourcesXml.first());
-    let sandboxXml = (0, xmlbuilder2_1.create)().ele("Sandbox");
-    for (const sourcePathInfo of context.sandboxConfig.sourcePaths) {
-        sandboxXml = sandboxXml.ele("SourceFile")
-            .ele("DatasetId").txt(sourcePathInfo.datasetId).up()
-            .ele("Path").txt(sourcePathInfo.path).up()
-            .up();
+    if (context.contextResources && context.contextResources.length > 0) {
+        let resourcesXml = create().ele("ContextResources");
+        for (const resource of context.contextResources) {
+            resourcesXml = resourcesXml
+                .ele("Resource")
+                .ele("Key").txt(String(resource.key)).up()
+                .ele("Type").txt(String(resource.type)).up()
+                .ele("Name").txt(String(resource.name)).up()
+                .ele("Description").txt(String(resource.description)).up()
+                .ele("DescriptorJson").txt(JSON.stringify(resource, null, 2)).up()
+                .up();
+        }
+        xml = xml.import(resourcesXml.first());
     }
+    let sandboxXml = create().ele("Sandbox");
+    sandboxXml = sandboxXml.ele("ContextResourcesPath").txt("/tmp/ekairos/contexts/{contextId}/resources").up();
+    sandboxXml = sandboxXml.ele("ResourcesManifest").txt("/tmp/ekairos/contexts/{contextId}/resources/manifest.json").up();
     sandboxXml = sandboxXml.ele("OutputPath").txt(context.sandboxConfig.outputPath).up();
+    sandboxXml = sandboxXml.ele("Note").txt("Context resources are materialized lazily only when executeCommand is called. Do not assume resource files exist unless you are using executeCommand. If executeCommand is used, read the manifest path from os.environ['EKAIROS_CONTEXT_RESOURCES_MANIFEST'] inside Python.").up();
     xml = xml.import(sandboxXml.first());
-    if (context.sourcePreviews && context.sourcePreviews.length > 0) {
-        let previewsXml = (0, xmlbuilder2_1.create)().ele("SourcePreviews");
-        for (const sourcePreviewInfo of context.sourcePreviews) {
-            const sp = sourcePreviewInfo.preview;
-            let px = (0, xmlbuilder2_1.create)().ele("SourcePreview")
-                .ele("DatasetId").txt(sourcePreviewInfo.datasetId).up()
+    if (context.inputPreviews && context.inputPreviews.length > 0) {
+        let previewsXml = create().ele("InputPreviews");
+        for (const inputPreviewInfo of context.inputPreviews) {
+            const sp = inputPreviewInfo.preview;
+            let px = create().ele("InputPreview")
+                .ele("DatasetId").txt(inputPreviewInfo.datasetId).up()
                 .ele("TotalRows").txt(String(sp.totalRows)).up();
             if (sp.metadata) {
                 const m = sp.metadata;
@@ -71,7 +74,7 @@ function buildContextSection(context) {
         xml = xml.import(previewsXml.first());
     }
     if (Array.isArray(context.errors) && context.errors.length > 0) {
-        let ex = (0, xmlbuilder2_1.create)().ele("PreviousErrors");
+        let ex = create().ele("PreviousErrors");
         for (const e of context.errors) {
             ex = ex.ele("Error").txt(e).up();
         }
@@ -81,7 +84,7 @@ function buildContextSection(context) {
     return xml.end({ prettyPrint: true, headless: true });
 }
 function buildOutputSchemaSection(context) {
-    let xml = (0, xmlbuilder2_1.create)()
+    let xml = create()
         .ele("OutputSchema")
         .ele("JsonSchema").txt(JSON.stringify(context.outputSchema?.schema ?? context.outputSchema ?? {}, null, 2)).up()
         .up();
@@ -89,39 +92,57 @@ function buildOutputSchemaSection(context) {
 }
 function buildInstructions(context) {
     const outputPath = context.sandboxConfig.outputPath;
-    const multipleSourcesNote = context.sourceDatasetIds.length > 1
-        ? "You have multiple source datasets available. You may need to read, join, filter, or combine data from them to produce the output."
+    const multipleInputsNote = (context.contextResources?.length ?? context.inputDatasetIds.length) > 1
+        ? "You have multiple context resources available. You may need to read, join, filter, or combine data from them to produce the output."
         : "";
-    let xml = (0, xmlbuilder2_1.create)()
+    let xml = create()
         .ele("Instructions")
         .ele("Workflow")
-        .ele("Step", { number: "1", name: "Inspect Source" })
-        .ele("Action").txt(`Review SourcePreviews to understand current record structures (data fields, shapes, edge cases). ${multipleSourcesNote}`).up()
+        .ele("Step", { number: "1", name: "Inspect Inputs" })
+        .ele("Action").txt(`Review ContextResources and any InputPreviews to understand current record structures, evidence, fields, shapes and edge cases. ${multipleInputsNote}`).up()
+        .ele("Note").txt("ContextResources DescriptorJson may include inline text, metadata, previewRows, or other visible evidence. Treat that visible content as already available context. Do not use executeCommand only to reread it.").up()
+        .up()
+        .ele("Step", { number: "2", name: "Define the Output Dataset (PLAN FIRST)" })
+        .ele("Action").txt("Call defineNotation with the formal definition of the OUTPUT dataset as a set derived from the input sets: e.g. D = \\pi_{fields}(\\sigma_{condition}(A \\bowtie B)) or set-builder with quantifiers, in LaTeX. Declare the input sets, bound variables and the predicates the output set satisfies.").up()
+        .ele("Note").txt("The definition and the materialization (the transform code + output rows) are TWO CO-EQUAL FACES of the dataset; author the definition FIRST as the PLAN: it states which sets you draw from, how they combine (join, filter, project, aggregate) and which invariants the output keeps (e.g. totals preserved). The definition is a formal proposition we trust — predicates may be semantic. Only for purely arithmetic invariants you MAY add a checkJson for optional advisory evidence. REFINE the definition whenever inspection of the inputs reveals new sets, variables or corrections, and call defineNotation with final=true just before completing — as the RESULT it describes the produced output; any arithmetic predicates then get advisory evidence (never a verdict).").up()
         .up()
-        .ele("Step", { number: "2", name: "Plan Mapping" })
-        .ele("Action").txt("Plan a deterministic mapping from source data fields to the output schema fields (normalize names, types, and formats).").up()
-        .ele("Note").txt("If fields are missing, set defaults; if types differ, coerce consistently. When working with multiple sources, decide how to combine or relate them.").up()
+        .ele("Step", { number: "3", name: "Plan Mapping" })
+        .ele("Action").txt("Plan a deterministic mapping from input data fields to the output schema fields (normalize names, types, and formats).").up()
+        .ele("Note").txt("If fields are missing, set defaults; if types differ, coerce consistently. When working with multiple inputs, decide how to combine or relate them. Output field names must remain exactly as declared by the output schema.").up()
         .up()
-        .ele("Step", { number: "3", name: "Transform" })
-        .ele("Action").txt("Use executeCommand to run a Python script that reads source JSONL file(s) and writes transformed records to output.jsonl. Keep line-per-record JSON objects with { 'type': 'row', 'data': { ... } }.").up()
-        .ele("Requirement").txt(`Write file to: ${outputPath}`).up()
+        .ele("Step", { number: "4", name: "Transform" })
+        .ele("Action").txt("For single-object output, use completeObject with the final object. For row output, use replaceRows with the final rows. Use executeCommand only when command execution is necessary, not merely convenient.").up()
+        .ele("Requirement").txt("Do not call completeObject until you have constructed the complete data object. completeObject requires data; a summary-only call is invalid and wastes a model iteration.").up()
+        .ele("Requirement").txt("Command execution is necessary only when the final output cannot be produced directly from the provided context, resource descriptors, or previews, and requires running code to inspect, parse, aggregate, join, or compute over files/resources.").up()
+        .ele("Requirement").txt("If the final output can be written directly from context already visible to you, do not use executeCommand. Do not use executeCommand just to format JSON, build an object, write output.jsonl, or make completion easier.").up()
+        .ele("Requirement").txt("Before using executeCommand, verify that direct completion is insufficient: you need file/resource contents not already visible in DescriptorJson or previews, deterministic computation over many rows, parsing/aggregation that is unreliable to do directly, or output too large/repetitive for direct completion. If none apply, command execution is not needed.").up()
+        .ele("Requirement").txt("When using executeCommand, provide commandDescription before the script runs. It must describe the inputs/resources used, operation performed, expected output, and why a command is the right tool.").up()
+        .ele("Requirement").txt("When executeCommand is used, context resources are materialized before the script runs at /tmp/ekairos/contexts/{contextId}/resources. The Python process receives EKAIROS_CONTEXT_RESOURCES_DIR and EKAIROS_CONTEXT_RESOURCES_MANIFEST environment variables. Read os.environ['EKAIROS_CONTEXT_RESOURCES_MANIFEST'] inside the script to discover exact files and metadata. Manifest entries expose files as resource['files'][index]['path'].").up()
+        .ele("Requirement").txt("If only some resources are needed for a command, pass resourceKeys with the specific ContextResources keys. Omit resourceKeys only when the script truly needs all resources.").up()
+        .ele("Requirement").txt(`If executeCommand is used, write file to: ${outputPath}`).up()
+        .ele("Requirement").txt("Every data object MUST use the exact property names from OutputSchema required/properties keys. Do not translate, localize, rename, or infer alternative field names.").up()
         .ele("Requirement").txt("Do not print large data to stdout; only progress and summaries.").up()
+        .ele("Requirement").txt("Do not install packages, download dependencies, or access the network from executeCommand. Use only the available runtime and standard library unless a dependency is already present.").up()
         .up()
-        .ele("Step", { number: "4", name: "Validate and Complete" })
-        .ele("Action").txt("Call completeDataset to validate against the output schema and mark as completed.").up()
+        .ele("Step", { number: "5", name: "Validate and Complete" })
+        .ele("Action").txt("Call defineNotation with final=true (the definition as RESULT, matching the produced output), then: when using completeObject or replaceRows, no separate completeDataset call is needed. When using executeCommand, call completeDataset to validate against the output schema and mark as completed.").up()
+        .ele("Behavior").txt("If any completion tool returns success:false, inspect validation details, repair the output, and call the appropriate completion tool again. Do not stop until a completion tool returns success:true.").up()
         .up()
         .up()
         .ele("Rules")
+        .ele("Rule").txt("The formal definition (defineNotation) and the materialization (transform code + output rows) are co-equal faces of the dataset: author the definition first as the PLAN, refine it on every discovery, finalize it as the RESULT before completing.").up()
         .ele("Rule").txt("Output must strictly match the output schema for each record in data.").up()
-        .ele("Rule").txt("Each line in output.jsonl must be a standalone JSON object with {type:'row', data:{...}}.").up()
+        .ele("Rule").txt("OutputSchema property names are authoritative. Field names are a technical contract; only field values may preserve input language.").up()
+        .ele("Rule").txt("Use the cheapest correct tool. completeObject and replaceRows are low-cost completion tools. executeCommand is a high-cost computation tool and requires an explicit commandDescription.").up()
+        .ele("Rule").txt("If using output.jsonl, each line must be a standalone JSON object with {type:'row', data:{...}}.").up()
         .ele("Rule").txt("Do not include headers, summaries, or metadata as records.").up()
-        .ele("Rule").txt("Be robust to malformed lines in source: skip or sanitize, but do not crash.").up()
+        .ele("Rule").txt("Be robust to malformed lines in input: skip or sanitize, but do not crash.").up()
         .up()
-        .ele("CurrentTask").txt("Transform source dataset(s) to match OutputSchema and write output.jsonl, then complete.").up()
+        .ele("CurrentTask").txt("Transform input dataset(s) to match OutputSchema and complete the dataset with the appropriate available tool.").up()
         .up();
     return xml.end({ prettyPrint: true, headless: true });
 }
-function buildTransformDatasetPrompt(context) {
+export function buildTransformDatasetPrompt(context) {
     const sections = [];
     sections.push(buildRole());
     sections.push("");
@@ -134,4 +155,3 @@ function buildTransformDatasetPrompt(context) {
     sections.push(buildInstructions(context));
     return sections.join("\n");
 }
-//# sourceMappingURL=prompts.js.map

package/dist/transform/transform-dataset.agent.d.ts

@@ -1,59 +1,34 @@
-import { type ContextReactor } from "@ekairos/events";
-import { TransformSourcePreviewContext } from "./filepreview";
-export type TransformDatasetContext = {
-    datasetId: string;
-    sourceDatasetIds: string[];
-    outputSchema: any;
-    sandboxConfig: {
-        sourcePaths: Array<{
-            datasetId: string;
-            path: string;
-        }>;
-        outputPath: string;
-    };
-    sourcePreviews?: Array<{
-        datasetId: string;
-        preview: TransformSourcePreviewContext;
-    }>;
-    errors: string[];
-    iterationCount: number;
-    instructions?: string;
-};
-export type TransformDatasetAgentParams = {
-    sourceDatasetIds: string[];
-    outputSchema: any;
-    instructions?: string;
-    datasetId?: string;
-    model?: string;
-    sandboxId?: string;
-    reactor?: ContextReactor<any, any>;
-};
-export type TransformDatasetResult = {
-    id: string;
-    status?: string;
-    title?: string;
-    schema?: any;
-    analysis?: any;
-    calculatedTotalRows?: number;
-    actualGeneratedRowCount?: number;
-    createdAt?: number;
-    updatedAt?: number;
-};
-export declare function createTransformDatasetStory<Env extends {
+import { type ContextReactor } from "@ekairos/reactor/context";
+import type { TransformDatasetRunOptions, TransformSandboxState, TransformInputPreviewContext } from "./transform-dataset.types.js";
+export type { TransformDatasetAgentParams, TransformDatasetContext, TransformDatasetResult, TransformDatasetRunOptions, TransformPromptContext, TransformSandboxState, } from "./transform-dataset.types.js";
+export declare function createTransformDatasetContext<Env extends {
     orgId: string;
 }>(params: {
-    sourceDatasetIds: string[];
+    inputDatasetIds: string[];
     outputSchema: any;
     instructions?: string;
     datasetId?: string;
     model?: string;
     sandboxId?: string;
     reactor?: ContextReactor<any, any>;
+    sandboxState?: TransformSandboxState;
+    inputPreviews?: Array<{
+        datasetId: string;
+        preview: TransformInputPreviewContext;
+    }>;
+    contextResources?: any[];
 }): {
     datasetId: string;
-    transform(env: Env, prompt?: string): Promise<{
+    transform(runtime: {
+        env: Env;
+    }, options?: TransformDatasetRunOptions): Promise<{
         datasetId: string;
     }>;
-    story: any;
+    context: any;
 };
-//# sourceMappingURL=transform-dataset.agent.d.ts.map
+export declare function registerTransformDatasetContext<Env extends {
+    orgId: string;
+}>(opts?: {
+    model?: string;
+    reactor?: ContextReactor<any, any>;
+}): void;