@ekairos/dataset 1.22.85-beta.development.0 → 1.22.87-beta.development.0

package/dist/transform/prompts.js

@@ -2,14 +2,14 @@ import { create } from "xmlbuilder2";
 function buildRole() {
     let xml = create()
         .ele("Role")
-        .txt("You are a dataset transformer. Your goal is to read one or more existing JSONL datasets and produce a NEW JSONL dataset whose records conform exactly to the provided output schema.")
+        .txt("You are a dataset transformer. Your goal is to read one or more input datasets/resources and produce a NEW dataset whose records conform exactly to the provided output schema.")
         .up();
     return xml.end({ prettyPrint: true, headless: true });
 }
 function buildGoal() {
     let xml = create()
         .ele("Goal")
-        .txt("Transform the input dataset(s) (JSONL with {type:'row', data:{...}} per line) into a new dataset strictly matching the output schema. Save to output.jsonl in the dataset workstation. Each line must remain a single JSON object representing one record. You may need to combine, filter, or reshape data from multiple input datasets.")
+        .txt("Transform the input dataset(s) into a new dataset strictly matching the output schema. Use the lowest-cost direct completion tool that can produce the correct output. Use sandbox command execution only when commands are necessary to inspect, parse, aggregate, join, or compute over files/resources that cannot be handled directly from the visible context and previews.")
         .up();
     return xml.end({ prettyPrint: true, headless: true });
 }
@@ -17,19 +17,25 @@ function buildContextSection(context) {
     let xml = create()
         .ele("Context")
         .ele("DatasetId").txt(context.datasetId).up();
-    let inputsXml = create().ele("InputDatasets");
-    for (const sourceId of context.inputDatasetIds) {
-        inputsXml = inputsXml.ele("InputDatasetId").txt(sourceId).up();
+    if (context.contextResources && context.contextResources.length > 0) {
+        let resourcesXml = create().ele("ContextResources");
+        for (const resource of context.contextResources) {
+            resourcesXml = resourcesXml
+                .ele("Resource")
+                .ele("Key").txt(String(resource.key)).up()
+                .ele("Type").txt(String(resource.type)).up()
+                .ele("Name").txt(String(resource.name)).up()
+                .ele("Description").txt(String(resource.description)).up()
+                .ele("DescriptorJson").txt(JSON.stringify(resource, null, 2)).up()
+                .up();
+        }
+        xml = xml.import(resourcesXml.first());
     }
-    xml = xml.import(inputsXml.first());
     let sandboxXml = create().ele("Sandbox");
-    for (const inputPathInfo of context.sandboxConfig.inputPaths) {
-        sandboxXml = sandboxXml.ele("InputFile")
-            .ele("DatasetId").txt(inputPathInfo.datasetId).up()
-            .ele("Path").txt(inputPathInfo.path).up()
-            .up();
-    }
+    sandboxXml = sandboxXml.ele("ContextResourcesPath").txt("/tmp/ekairos/contexts/{contextId}/resources").up();
+    sandboxXml = sandboxXml.ele("ResourcesManifest").txt("/tmp/ekairos/contexts/{contextId}/resources/manifest.json").up();
     sandboxXml = sandboxXml.ele("OutputPath").txt(context.sandboxConfig.outputPath).up();
+    sandboxXml = sandboxXml.ele("Note").txt("Context resources are materialized lazily only when executeCommand is called. Do not assume resource files exist unless you are using executeCommand. If executeCommand is used, read the manifest path from os.environ['EKAIROS_CONTEXT_RESOURCES_MANIFEST'] inside Python.").up();
     xml = xml.import(sandboxXml.first());
     if (context.inputPreviews && context.inputPreviews.length > 0) {
         let previewsXml = create().ele("InputPreviews");
@@ -86,38 +92,48 @@ function buildOutputSchemaSection(context) {
 }
 function buildInstructions(context) {
     const outputPath = context.sandboxConfig.outputPath;
-    const multipleInputsNote = context.inputDatasetIds.length > 1
-        ? "You have multiple input datasets available. You may need to read, join, filter, or combine data from them to produce the output."
+    const multipleInputsNote = (context.contextResources?.length ?? context.inputDatasetIds.length) > 1
+        ? "You have multiple context resources available. You may need to read, join, filter, or combine data from them to produce the output."
         : "";
     let xml = create()
         .ele("Instructions")
         .ele("Workflow")
         .ele("Step", { number: "1", name: "Inspect Inputs" })
-        .ele("Action").txt(`Review InputPreviews to understand current record structures (data fields, shapes, edge cases). ${multipleInputsNote}`).up()
+        .ele("Action").txt(`Review ContextResources and any InputPreviews to understand current record structures, evidence, fields, shapes and edge cases. ${multipleInputsNote}`).up()
+        .ele("Note").txt("ContextResources DescriptorJson may include inline text, metadata, previewRows, or other visible evidence. Treat that visible content as already available context. Do not use executeCommand only to reread it.").up()
         .up()
         .ele("Step", { number: "2", name: "Plan Mapping" })
         .ele("Action").txt("Plan a deterministic mapping from input data fields to the output schema fields (normalize names, types, and formats).").up()
         .ele("Note").txt("If fields are missing, set defaults; if types differ, coerce consistently. When working with multiple inputs, decide how to combine or relate them. Output field names must remain exactly as declared by the output schema.").up()
         .up()
         .ele("Step", { number: "3", name: "Transform" })
-        .ele("Action").txt("Use executeCommand to run a Python script that reads input JSONL file(s) and writes transformed records to output.jsonl. Keep line-per-record JSON objects with { 'type': 'row', 'data': { ... } }.").up()
-        .ele("Requirement").txt(`Write file to: ${outputPath}`).up()
+        .ele("Action").txt("For single-object output, use completeObject with the final object. For row output, use replaceRows with the final rows. Use executeCommand only when command execution is necessary, not merely convenient.").up()
+        .ele("Requirement").txt("Do not call completeObject until you have constructed the complete data object. completeObject requires data; a summary-only call is invalid and wastes a model iteration.").up()
+        .ele("Requirement").txt("Command execution is necessary only when the final output cannot be produced directly from the provided context, resource descriptors, or previews, and requires running code to inspect, parse, aggregate, join, or compute over files/resources.").up()
+        .ele("Requirement").txt("If the final output can be written directly from context already visible to you, do not use executeCommand. Do not use executeCommand just to format JSON, build an object, write output.jsonl, or make completion easier.").up()
+        .ele("Requirement").txt("Before using executeCommand, verify that direct completion is insufficient: you need file/resource contents not already visible in DescriptorJson or previews, deterministic computation over many rows, parsing/aggregation that is unreliable to do directly, or output too large/repetitive for direct completion. If none apply, command execution is not needed.").up()
+        .ele("Requirement").txt("When using executeCommand, provide commandDescription before the script runs. It must describe the inputs/resources used, operation performed, expected output, and why a command is the right tool.").up()
+        .ele("Requirement").txt("When executeCommand is used, context resources are materialized before the script runs at /tmp/ekairos/contexts/{contextId}/resources. The Python process receives EKAIROS_CONTEXT_RESOURCES_DIR and EKAIROS_CONTEXT_RESOURCES_MANIFEST environment variables. Read os.environ['EKAIROS_CONTEXT_RESOURCES_MANIFEST'] inside the script to discover exact files and metadata. Manifest entries expose files as resource['files'][index]['path'].").up()
+        .ele("Requirement").txt("If only some resources are needed for a command, pass resourceKeys with the specific ContextResources keys. Omit resourceKeys only when the script truly needs all resources.").up()
+        .ele("Requirement").txt(`If executeCommand is used, write file to: ${outputPath}`).up()
         .ele("Requirement").txt("Every data object MUST use the exact property names from OutputSchema required/properties keys. Do not translate, localize, rename, or infer alternative field names.").up()
         .ele("Requirement").txt("Do not print large data to stdout; only progress and summaries.").up()
+        .ele("Requirement").txt("Do not install packages, download dependencies, or access the network from executeCommand. Use only the available runtime and standard library unless a dependency is already present.").up()
         .up()
         .ele("Step", { number: "4", name: "Validate and Complete" })
-        .ele("Action").txt("Call completeDataset to validate against the output schema and mark as completed.").up()
-        .ele("Behavior").txt("If completeDataset returns success:false, inspect validation details, rewrite output.jsonl, and call completeDataset again. Do not stop until completeDataset returns success:true.").up()
+        .ele("Action").txt("When using completeObject or replaceRows, no separate completeDataset call is needed. When using executeCommand, call completeDataset to validate against the output schema and mark as completed.").up()
+        .ele("Behavior").txt("If any completion tool returns success:false, inspect validation details, repair the output, and call the appropriate completion tool again. Do not stop until a completion tool returns success:true.").up()
         .up()
         .up()
         .ele("Rules")
         .ele("Rule").txt("Output must strictly match the output schema for each record in data.").up()
         .ele("Rule").txt("OutputSchema property names are authoritative. Field names are a technical contract; only field values may preserve input language.").up()
-        .ele("Rule").txt("Each line in output.jsonl must be a standalone JSON object with {type:'row', data:{...}}.").up()
+        .ele("Rule").txt("Use the cheapest correct tool. completeObject and replaceRows are low-cost completion tools. executeCommand is a high-cost computation tool and requires an explicit commandDescription.").up()
+        .ele("Rule").txt("If using output.jsonl, each line must be a standalone JSON object with {type:'row', data:{...}}.").up()
         .ele("Rule").txt("Do not include headers, summaries, or metadata as records.").up()
         .ele("Rule").txt("Be robust to malformed lines in input: skip or sanitize, but do not crash.").up()
         .up()
-        .ele("CurrentTask").txt("Transform input dataset(s) to match OutputSchema and write output.jsonl, then complete.").up()
+        .ele("CurrentTask").txt("Transform input dataset(s) to match OutputSchema and complete the dataset with the appropriate available tool.").up()
         .up();
     return xml.end({ prettyPrint: true, headless: true });
 }

package/dist/transform/transform-dataset.agent.d.ts

@@ -16,6 +16,7 @@ export declare function createTransformDatasetContext<Env extends {
         datasetId: string;
         preview: TransformInputPreviewContext;
     }>;
+    contextResources?: any[];
 }): {
     datasetId: string;
     transform(runtime: {

package/dist/transform/transform-dataset.agent.js

@@ -4,7 +4,8 @@ import { createCompleteDatasetTool, didCompleteDatasetSucceed, getDatasetFatalFa
 import { datasetUpdateSchemaStep } from "../dataset/steps.js";
 import { getDatasetOutputPath } from "../datasetFiles.js";
 import { createExecuteCommandTool } from "../executeCommand.tool.js";
-import { buildTransformDatasetPromptStep, ensureTransformInputsInSandboxStep, generateTransformInputPreviewsStep, } from "./transform-dataset.steps.js";
+import { createCompleteObjectTool, createReplaceRowsTool, } from "../writeDatasetRows.tool.js";
+import { buildTransformDatasetPromptStep, } from "./transform-dataset.steps.js";
 import { createDatasetId } from "../id.js";
 async function awaitContextRun(run) {
     if (!run)
@@ -45,27 +46,9 @@ function createTransformDatasetContextDefinition(params) {
             throw new Error("dataset_sandbox_required");
         }
         const initialized = sandboxState.initialized && Array.isArray(sandboxState.inputPaths)
-            ? {
-                inputPaths: sandboxState.inputPaths,
-                outputPath: previous?.sandboxConfig?.outputPath ?? getDatasetOutputPath(datasetId),
-                state: sandboxState,
-            }
-            : await ensureTransformInputsInSandboxStep({
-                runtime,
-                sandboxId,
-                datasetId,
-                inputDatasetIds,
-                state: sandboxState,
-            });
-        let inputPreviews = previous?.inputPreviews ?? params.inputPreviews ?? undefined;
-        if (!inputPreviews) {
-            inputPreviews = await generateTransformInputPreviewsStep({
-                runtime,
-                sandboxId,
-                datasetId,
-                inputPaths: initialized.inputPaths,
-            });
-        }
+            ? sandboxState
+            : { initialized: false, inputPaths: [] };
+        const inputPreviews = previous?.inputPreviews ?? params.inputPreviews ?? [];
         await datasetUpdateSchemaStep({
             runtime,
             datasetId,
@@ -78,9 +61,10 @@ function createTransformDatasetContextDefinition(params) {
             outputSchema,
             sandboxConfig: {
                 inputPaths: initialized.inputPaths,
-                outputPath: initialized.outputPath,
+                outputPath: previous?.sandboxConfig?.outputPath ?? getDatasetOutputPath(datasetId),
             },
             inputPreviews: inputPreviews.length > 0 ? inputPreviews : undefined,
+            contextResources: previous?.contextResources ?? params.contextResources ?? [],
             errors: [],
         };
         const basePrompt = await buildTransformDatasetPromptStep({
@@ -104,14 +88,16 @@ function createTransformDatasetContextDefinition(params) {
             outputSchema,
             instructions,
             sandboxId,
-            sandboxState: initialized.state,
+            sandboxState: initialized,
+            contextResources: previous?.contextResources ?? params.contextResources ?? [],
             system,
             sandboxConfig: {
                 inputPaths: initialized.inputPaths,
-                outputPath: initialized.outputPath,
+                outputPath: previous?.sandboxConfig?.outputPath ?? getDatasetOutputPath(datasetId),
             },
         };
     })
+        .resources(({ content }) => Array.isArray(content?.contextResources) ? content.contextResources : [])
         .narrative(async (stored) => {
         return String(stored?.content?.system ?? "");
     })
@@ -123,6 +109,18 @@ function createTransformDatasetContextDefinition(params) {
         if (!sandboxId)
             throw new Error("dataset_sandbox_required");
         return {
+            completeObject: createCompleteObjectTool({
+                datasetId,
+                sandboxId,
+                runtime,
+                schema: stored?.content?.outputSchema,
+            }),
+            replaceRows: createReplaceRowsTool({
+                datasetId,
+                sandboxId,
+                runtime,
+                schema: stored?.content?.outputSchema,
+            }),
             executeCommand: createExecuteCommandTool({
                 datasetId,
                 sandboxId,
@@ -168,6 +166,7 @@ export function createTransformDatasetContext(params) {
         reactor: params.reactor,
         sandboxState: params.sandboxState,
         inputPreviews: params.inputPreviews,
+        contextResources: params.contextResources,
     });
     return {
         datasetId,
@@ -209,6 +208,7 @@ export function createTransformDatasetContext(params) {
                     sandboxId: params.sandboxId ?? "",
                     sandboxState: params.sandboxState ?? { initialized: false, inputPaths: [] },
                     inputPreviews: params.inputPreviews,
+                    contextResources: params.contextResources ?? [],
                 },
             });
             await awaitContextRun(shell.run);

package/dist/transform/transform-dataset.types.d.ts

@@ -1,4 +1,4 @@
-import type { ContextReactor } from "@ekairos/events";
+import type { ContextReactor, StoredContextResource } from "@ekairos/events";
 import type { TransformInputPreviewContext } from "./filepreview.js";
 export type { TransformInputPreviewContext } from "./filepreview.js";
 export type TransformSandboxState = {
@@ -23,6 +23,7 @@ export type TransformDatasetContext = {
         datasetId: string;
         preview: TransformInputPreviewContext;
     }>;
+    contextResources?: StoredContextResource[];
     errors: string[];
     iterationCount: number;
     instructions?: string;
@@ -40,6 +41,7 @@ export type TransformDatasetAgentParams = {
         datasetId: string;
         preview: TransformInputPreviewContext;
     }>;
+    contextResources?: StoredContextResource[];
 };
 export type TransformDatasetRunOptions = {
     prompt?: string;
@@ -88,5 +90,6 @@ export type TransformPromptContext = {
             };
         };
     }>;
+    contextResources?: StoredContextResource[];
     errors: string[];
 };

package/dist/writeDatasetRows.tool.d.ts

@@ -0,0 +1,188 @@
+import type { DatasetSchemaInput } from "./builder/types.js";
+type WriteRowsToolParams = {
+    datasetId: string;
+    sandboxId: string;
+    runtime: any;
+    schema?: DatasetSchemaInput | any;
+};
+export declare function persistDatasetRowsStep(params: WriteRowsToolParams & {
+    rows: Array<Record<string, unknown>>;
+    summary?: string;
+}): Promise<{
+    success: boolean;
+    status: string;
+    rowSource: string;
+    outputPath: null;
+    storagePath: null;
+    error: string;
+    message: string;
+    validRows?: undefined;
+    rowRecordCount?: undefined;
+    repairInstructions?: undefined;
+    savedRecords?: undefined;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    outputPath: null;
+    storagePath: null;
+    validRows: number;
+    rowRecordCount: number;
+    error: string;
+    message: string;
+    repairInstructions: string[];
+    savedRecords?: undefined;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    outputPath: null;
+    storagePath: null;
+    validRows: number;
+    rowRecordCount: number;
+    savedRecords: number;
+    error: string;
+    message: string;
+    repairInstructions?: undefined;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    outputPath: null;
+    storagePath: null;
+    records: number;
+    savedRecords: number;
+    summary: string | undefined;
+    error?: undefined;
+    message?: undefined;
+    validRows?: undefined;
+    rowRecordCount?: undefined;
+    repairInstructions?: undefined;
+}>;
+export declare function createReplaceRowsTool(params: WriteRowsToolParams): import("ai").Tool<{
+    rows: Array<Record<string, unknown>>;
+    summary?: string;
+}, {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    outputPath: null;
+    storagePath: null;
+    error: string;
+    message: string;
+    validRows?: undefined;
+    rowRecordCount?: undefined;
+    repairInstructions?: undefined;
+    savedRecords?: undefined;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    outputPath: null;
+    storagePath: null;
+    validRows: number;
+    rowRecordCount: number;
+    error: string;
+    message: string;
+    repairInstructions: string[];
+    savedRecords?: undefined;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    outputPath: null;
+    storagePath: null;
+    validRows: number;
+    rowRecordCount: number;
+    savedRecords: number;
+    error: string;
+    message: string;
+    repairInstructions?: undefined;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    outputPath: null;
+    storagePath: null;
+    records: number;
+    savedRecords: number;
+    summary: string | undefined;
+    error?: undefined;
+    message?: undefined;
+    validRows?: undefined;
+    rowRecordCount?: undefined;
+    repairInstructions?: undefined;
+}>;
+export declare function createCompleteObjectTool(params: WriteRowsToolParams): import("ai").Tool<{
+    data: Record<string, unknown>;
+    summary?: string;
+}, {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    outputPath: null;
+    storagePath: null;
+    error: string;
+    message: string;
+    validRows?: undefined;
+    rowRecordCount?: undefined;
+    repairInstructions?: undefined;
+    savedRecords?: undefined;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    outputPath: null;
+    storagePath: null;
+    validRows: number;
+    rowRecordCount: number;
+    error: string;
+    message: string;
+    repairInstructions: string[];
+    savedRecords?: undefined;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    outputPath: null;
+    storagePath: null;
+    validRows: number;
+    rowRecordCount: number;
+    savedRecords: number;
+    error: string;
+    message: string;
+    repairInstructions?: undefined;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    outputPath: null;
+    storagePath: null;
+    records: number;
+    savedRecords: number;
+    summary: string | undefined;
+    error?: undefined;
+    message?: undefined;
+    validRows?: undefined;
+    rowRecordCount?: undefined;
+    repairInstructions?: undefined;
+}>;
+export {};