@ekairos/dataset 1.22.78-beta.development.0 → 1.22.80-beta.development.0

package/dist/file/prompts.js

@@ -89,7 +89,8 @@ function buildErrorsSection(errors) {
         return null;
     }
     let xml = create()
-        .ele("PreviousErrors");
+        .ele("PreviousErrors")
+        .ele("Instruction").txt("Treat these as repair feedback from the previous validation attempt. Rewrite output.jsonl from the schema contract; do not patch source column names into schema keys piecemeal.").up();
     for (const error of errors) {
         xml = xml.ele("Error").txt(error).up();
     }
@@ -114,16 +115,110 @@ function buildContextSection(context) {
     xml = xml.up();
     return xml.end({ prettyPrint: true, headless: true });
 }
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : null;
+}
+function getSchemaObject(context) {
+    return asRecord(context.schema?.schema);
+}
+function joinSchemaPath(basePath, key) {
+    return basePath === "$" ? `$.${key}` : `${basePath}.${key}`;
+}
+function collectSchemaContract(schema, path = "$", contract = {
+    requiredPaths: [],
+    propertyPaths: [],
+    enumConstraints: [],
+    closedObjectPaths: [],
+}) {
+    const record = asRecord(schema);
+    if (!record) {
+        return contract;
+    }
+    if (Array.isArray(record.enum)) {
+        contract.enumConstraints.push({
+            path,
+            values: record.enum.map((value) => JSON.stringify(value)),
+        });
+    }
+    const properties = asRecord(record.properties);
+    if (properties) {
+        if (record.additionalProperties === false) {
+            contract.closedObjectPaths.push(path);
+        }
+        const required = Array.isArray(record.required)
+            ? record.required.filter((value) => typeof value === "string")
+            : [];
+        for (const key of required) {
+            contract.requiredPaths.push(joinSchemaPath(path, key));
+        }
+        for (const [key, childSchema] of Object.entries(properties)) {
+            const childPath = joinSchemaPath(path, key);
+            contract.propertyPaths.push(childPath);
+            collectSchemaContract(childSchema, childPath, contract);
+        }
+    }
+    if (record.items) {
+        collectSchemaContract(record.items, `${path}[]`, contract);
+    }
+    for (const keyword of ["oneOf", "anyOf", "allOf"]) {
+        if (Array.isArray(record[keyword])) {
+            for (const childSchema of record[keyword]) {
+                collectSchemaContract(childSchema, path, contract);
+            }
+        }
+    }
+    return contract;
+}
+function appendLimitedList(xml, elementName, itemName, values, maxItems) {
+    let node = xml.ele(elementName);
+    for (const value of values.slice(0, maxItems)) {
+        node = node.ele(itemName).txt(value).up();
+    }
+    if (values.length > maxItems) {
+        node = node.ele("Truncated").txt(String(values.length - maxItems)).up();
+    }
+    return node.up();
+}
 function buildSchemaSection(context) {
-    if (!context.schema) {
+    const schema = getSchemaObject(context);
+    if (!context.schema || !schema) {
         return "";
     }
+    const contract = collectSchemaContract(schema);
     let xml = create()
         .com("Schema section: This defines the structure of ONE RECORD (row). Each line in the JSONL output must conform to this schema.")
         .ele("Schema")
         .ele("Title").txt(context.schema.title || "").up()
-        .ele("Description").txt(context.schema.description || "").up()
-        .ele("JsonSchema").txt(JSON.stringify(context.schema.schema, null, 2)).up()
+        .ele("Description").txt(context.schema.description || "").up();
+    xml = xml
+        .ele("SchemaContract")
+        .ele("Purpose").txt("Compact output contract derived from JSON Schema. Use this before writing output.jsonl.").up()
+        .ele("Rule").txt("Use only schema property keys in data objects. Source headers are input labels, not output keys.").up()
+        .ele("Rule").txt("Required paths are required everywhere, including nested objects and array items.").up()
+        .ele("Rule").txt("Enum fields must use exactly one of the listed literal values. Normalize source labels to the closest valid enum literal; never emit a value outside the enum.").up();
+    xml = appendLimitedList(xml, "RequiredPaths", "Path", contract.requiredPaths, 120);
+    xml = appendLimitedList(xml, "PropertyPaths", "Path", contract.propertyPaths, 160);
+    let enumsXml = xml.ele("EnumConstraints");
+    for (const constraint of contract.enumConstraints.slice(0, 80)) {
+        let enumXml = enumsXml.ele("Enum", { path: constraint.path });
+        for (const value of constraint.values.slice(0, 80)) {
+            enumXml = enumXml.ele("Value").txt(value).up();
+        }
+        if (constraint.values.length > 80) {
+            enumXml = enumXml.ele("Truncated").txt(String(constraint.values.length - 80)).up();
+        }
+        enumsXml = enumXml.up();
+    }
+    if (contract.enumConstraints.length > 80) {
+        enumsXml = enumsXml.ele("Truncated").txt(String(contract.enumConstraints.length - 80)).up();
+    }
+    xml = enumsXml.up();
+    xml = appendLimitedList(xml, "ClosedObjectPaths", "Path", contract.closedObjectPaths, 80);
+    xml = xml
+        .up()
+        .ele("JsonSchema").txt(JSON.stringify(schema, null, 2)).up()
         .up();
     return xml.end({ prettyPrint: true, headless: true });
 }
@@ -148,6 +243,9 @@ function buildInstructions(context) {
             .ele("Requirements")
             .ele("Requirement").txt("Every output row must conform exactly to the provided schema").up()
             .ele("Requirement").txt("Every data object MUST use the exact property names from the provided JSON Schema required/properties keys").up()
+            .ele("Requirement").txt("Build a schema-first mapping from source columns to schema fields before writing output.jsonl. Do not use raw source headers as JSON keys unless they are exactly schema keys").up()
+            .ele("Requirement").txt("For nested required fields, populate the required child keys inside each nested object or array item; top-level validity is not enough").up()
+            .ele("Requirement").txt("For enum fields, emit exactly one allowed enum literal from SchemaContract; normalize labels or abbreviations into allowed literals").up()
             .ele("Requirement").txt("Do not translate, localize, rename, camelize differently, or infer alternative field names. Field names are a technical contract; only field values may preserve the source language").up()
             .ele("Requirement").txt("Do not call generateSchema when a schema is already provided").up()
             .up()
@@ -173,6 +271,7 @@ function buildInstructions(context) {
         .ele("Requirement").txt("Parse ALL data rows/records from the file (exclude header sections and metadata)").up()
         .ele("Requirement").txt("Output JSONL format: each line is {\"type\": \"row\", \"data\": {...record...}}").up()
         .ele("Requirement").txt("When a schema is provided, each data object must contain the exact required schema keys and must not use translated or synonymous keys").up()
+        .ele("Requirement").txt("When validation returns zero valid rows, treat the previous output as structurally wrong and rewrite output.jsonl from the SchemaContract, not by applying small patches").up()
         .ele("Requirement").txt("Extract ONLY data records; skip any header lines, summary sections, or file metadata").up()
         .ele("Requirement").txt(`Save output to: ${outputPath}`).up()
         .ele("Requirement").txt("Use descriptive scriptName in snake_case (e.g., 'parse_csv_to_jsonl')").up()
@@ -207,6 +306,11 @@ export function buildFileDatasetPrompt(context) {
     sections.push("");
     sections.push(buildContextSection(context));
     sections.push("");
+    const schemaSection = buildSchemaSection(context);
+    if (schemaSection) {
+        sections.push(schemaSection);
+        sections.push("");
+    }
     sections.push(buildInstructions(context));
     return sections.join("\n");
 }

package/dist/transform/filepreview.js

@@ -1,9 +1,8 @@
-import { getDatasetWorkstation } from "../datasetFiles.js";
+import { getDatasetScriptsDir } from "../datasetFiles.js";
 import { runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep } from "../sandbox/steps.js";
 const DEFAULT_HEAD_LINES = 50;
 async function runPythonSnippet(runtime, sandboxId, datasetId, scriptName, code, args, description) {
-    const workstation = getDatasetWorkstation(datasetId);
-    const scriptPath = `${workstation}/${scriptName}.py`;
+    const scriptPath = `${getDatasetScriptsDir(datasetId)}/${scriptName}.py`;
     await writeDatasetSandboxFilesStep({
         runtime,
         sandboxId,

package/dist/transform/transform-dataset.agent.d.ts

@@ -1,5 +1,5 @@
 import { type ContextReactor } from "@ekairos/events";
-import type { TransformDatasetRunOptions } from "./transform-dataset.types.js";
+import type { TransformDatasetRunOptions, TransformSandboxState, TransformSourcePreviewContext } from "./transform-dataset.types.js";
 export type { TransformDatasetAgentParams, TransformDatasetContext, TransformDatasetResult, TransformDatasetRunOptions, TransformPromptContext, TransformSandboxState, } from "./transform-dataset.types.js";
 export declare function createTransformDatasetContext<Env extends {
     orgId: string;
@@ -11,6 +11,11 @@ export declare function createTransformDatasetContext<Env extends {
     model?: string;
     sandboxId?: string;
     reactor?: ContextReactor<any, any>;
+    sandboxState?: TransformSandboxState;
+    sourcePreviews?: Array<{
+        datasetId: string;
+        preview: TransformSourcePreviewContext;
+    }>;
 }): {
     datasetId: string;
     transform(runtime: {

package/dist/transform/transform-dataset.agent.js

@@ -2,6 +2,7 @@ import { createContext, INPUT_TEXT_ITEM_TYPE, WEB_CHANNEL, } from "@ekairos/even
 import { createClearDatasetTool } from "../clearDataset.tool.js";
 import { createCompleteDatasetTool, didCompleteDatasetSucceed, getDatasetFatalFailure, } from "../completeDataset.tool.js";
 import { datasetUpdateSchemaStep } from "../dataset/steps.js";
+import { getDatasetOutputPath } from "../datasetFiles.js";
 import { createExecuteCommandTool } from "../executeCommand.tool.js";
 import { buildTransformDatasetPromptStep, ensureTransformSourcesInSandboxStep, generateTransformSourcePreviewsStep, } from "./transform-dataset.steps.js";
 import { createDatasetId } from "../id.js";
@@ -20,7 +21,8 @@ function createTransformDatasetContextDefinition(params) {
     let contextBuilder = createContext("dataset.transform")
         .context(async (stored, _env, runtime) => {
         const previous = stored?.content ?? {};
-        const sandboxState = previous?.sandboxState ?? { initialized: false, sourcePaths: [] };
+        const sandboxState = previous?.sandboxState ??
+            params.sandboxState ?? { initialized: false, sourcePaths: [] };
         const datasetId = previous?.datasetId ?? fallbackDatasetId ?? "";
         const sourceDatasetIds = Array.isArray(previous?.sourceDatasetIds)
             ? previous.sourceDatasetIds
@@ -42,19 +44,28 @@ function createTransformDatasetContextDefinition(params) {
         if (!sandboxId) {
             throw new Error("dataset_sandbox_required");
         }
-        const initialized = await ensureTransformSourcesInSandboxStep({
-            runtime,
-            sandboxId,
-            datasetId,
-            sourceDatasetIds,
-            state: sandboxState,
-        });
-        const sourcePreviews = await generateTransformSourcePreviewsStep({
-            runtime,
-            sandboxId,
-            datasetId,
-            sourcePaths: initialized.sourcePaths,
-        });
+        const initialized = sandboxState.initialized && Array.isArray(sandboxState.sourcePaths)
+            ? {
+                sourcePaths: sandboxState.sourcePaths,
+                outputPath: previous?.sandboxConfig?.outputPath ?? getDatasetOutputPath(datasetId),
+                state: sandboxState,
+            }
+            : await ensureTransformSourcesInSandboxStep({
+                runtime,
+                sandboxId,
+                datasetId,
+                sourceDatasetIds,
+                state: sandboxState,
+            });
+        let sourcePreviews = previous?.sourcePreviews ?? params.sourcePreviews ?? undefined;
+        if (!sourcePreviews) {
+            sourcePreviews = await generateTransformSourcePreviewsStep({
+                runtime,
+                sandboxId,
+                datasetId,
+                sourcePaths: initialized.sourcePaths,
+            });
+        }
         await datasetUpdateSchemaStep({
             runtime,
             datasetId,
@@ -155,6 +166,8 @@ export function createTransformDatasetContext(params) {
         model: params.model,
         sandboxId: params.sandboxId,
         reactor: params.reactor,
+        sandboxState: params.sandboxState,
+        sourcePreviews: params.sourcePreviews,
     });
     return {
         datasetId,
@@ -189,12 +202,14 @@ export function createTransformDatasetContext(params) {
                     maxModelSteps: 5,
                 },
                 __initialContent: {
+                    ...(options.initialContent ?? {}),
                     datasetId,
                     sourceDatasetIds: params.sourceDatasetIds,
                     outputSchema: params.outputSchema,
                     instructions: params.instructions,
                     sandboxId: params.sandboxId ?? "",
-                    sandboxState: { initialized: false, sourcePaths: [] },
+                    sandboxState: params.sandboxState ?? { initialized: false, sourcePaths: [] },
+                    sourcePreviews: params.sourcePreviews,
                 },
             });
             await awaitContextRun(shell.run);

package/dist/transform/transform-dataset.steps.js

@@ -1,4 +1,4 @@
-import { getDatasetOutputPath, getDatasetWorkstation } from "../datasetFiles.js";
+import { getDatasetOutputPath, getDatasetSourcesDir, getDatasetStandardDirs, } from "../datasetFiles.js";
 import { datasetReadOutputJsonlStep } from "../dataset/steps.js";
 import { runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep } from "../sandbox/steps.js";
 import { generateSourcePreview } from "./filepreview.js";
@@ -12,16 +12,15 @@ export async function ensureTransformSourcesInSandboxStep(params) {
             state: params.state,
         };
     }
-    const workstation = getDatasetWorkstation(params.datasetId);
     await runDatasetSandboxCommandStep({
         runtime: params.runtime,
         sandboxId: params.sandboxId,
         cmd: "mkdir",
-        args: ["-p", workstation],
+        args: ["-p", ...getDatasetStandardDirs(params.datasetId)],
     });
     const sourcePaths = [];
     for (const sourceDatasetId of params.sourceDatasetIds) {
-        const sourcePath = `${workstation}/source_${sourceDatasetId}.jsonl`;
+        const sourcePath = `${getDatasetSourcesDir(params.datasetId)}/source_${sourceDatasetId}.jsonl`;
         const source = await datasetReadOutputJsonlStep({
             runtime: params.runtime,
             datasetId: sourceDatasetId,

package/dist/transform/transform-dataset.types.d.ts

@@ -35,10 +35,16 @@ export type TransformDatasetAgentParams = {
     model?: string;
     sandboxId?: string;
     reactor?: ContextReactor<any, any>;
+    sandboxState?: TransformSandboxState;
+    sourcePreviews?: Array<{
+        datasetId: string;
+        preview: TransformSourcePreviewContext;
+    }>;
 };
 export type TransformDatasetRunOptions = {
     prompt?: string;
     durable?: boolean;
+    initialContent?: Record<string, any>;
 };
 export type TransformDatasetResult = {
     id: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ekairos/dataset",
-  "version": "1.22.78-beta.development.0",
+  "version": "1.22.80-beta.development.0",
   "description": "Pulzar Dataset Tools",
   "type": "module",
   "main": "dist/index.js",
@@ -65,9 +65,9 @@
     "test:ai-sdk:instant": "vitest run -c vitest.codex.config.mts src/tests/materializeDataset.ai-sdk.instant.test.ts"
   },
   "dependencies": {
-    "@ekairos/domain": "^1.22.78-beta.development.0",
-    "@ekairos/events": "^1.22.78-beta.development.0",
-    "@ekairos/sandbox": "^1.22.78-beta.development.0",
+    "@ekairos/domain": "^1.22.80-beta.development.0",
+    "@ekairos/events": "^1.22.80-beta.development.0",
+    "@ekairos/sandbox": "^1.22.80-beta.development.0",
     "@instantdb/admin": "0.22.158",
     "@instantdb/core": "0.22.142",
     "ai": "^5.0.44",