@ekairos/dataset 1.22.83-beta.development.0 → 1.22.85-beta.development.0

package/dist/builder/materialize.js

@@ -1,17 +1,16 @@
 import { createFileParseContext } from "../file/file-dataset.agent.js";
 import { readInstantFileStep } from "../file/steps.js";
 import { createTransformDatasetContext } from "../transform/transform-dataset.agent.js";
-import { ensureTransformSourcesInSandboxStep, generateTransformSourcePreviewsStep, } from "../transform/transform-dataset.steps.js";
+import { ensureTransformInputsInSandboxStep, generateTransformInputPreviewsStep, } from "../transform/transform-dataset.steps.js";
 import { datasetGetByIdStep, datasetInferAndUpdateSchemaStep, datasetPreviewRowsStep, datasetReadOneStep, } from "../dataset/steps.js";
-import { getDatasetOutputPath, getDatasetScriptsDir, getDatasetSourcesDir, getDatasetStandardDirs, } from "../datasetFiles.js";
+import { getDatasetOutputPath, getDatasetScriptsDir, getDatasetResourcesDir, getDatasetStandardDirs, } from "../datasetFiles.js";
 import { registerDatasetAgentMaterializers } from "./agentMaterializers.js";
-import { buildFileDefaultInstructions, buildRawSourceInstructions, buildTransformInstructions, } from "./instructions.js";
-import { createOrUpdateDatasetMetadata, materializeRowsToDataset, uploadInlineTextSource, } from "./persistence.js";
-import { getDomainDescriptor } from "./sourceRows.js";
-import { materializeQuerySource } from "./materializeQuery.js";
+import { buildFileDefaultInstructions, buildRawResourceInstructions, buildTransformInstructions, } from "./instructions.js";
+import { createOrUpdateDatasetMetadata, materializeRowsToDataset, uploadInlineTextResource, } from "./persistence.js";
+import { materializeQueryResource } from "./materializeQuery.js";
 import { readDatasetSandboxTextFileStep, runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep, writeDatasetSandboxTextFilesStep, } from "../sandbox/steps.js";
-function makeIntermediateDatasetId(targetDatasetId, sourceKind, index) {
-    return `${targetDatasetId}__${sourceKind}_${index}`;
+function makeIntermediateDatasetId(targetDatasetId, resourceKind, index) {
+    return `${targetDatasetId}__${resourceKind}_${index}`;
 }
 function normalizeParsedTextRows(value) {
     if (Array.isArray(value)) {
@@ -21,10 +20,10 @@ function normalizeParsedTextRows(value) {
         return [value];
     return [{ value }];
 }
-function materializeRawTextRows(source) {
-    const text = String(source.text ?? "");
-    const mimeType = String(source.mimeType ?? "").toLowerCase();
-    const name = String(source.name ?? "").toLowerCase();
+function materializeRawTextRows(resource) {
+    const text = String(resource.text ?? "");
+    const mimeType = String(resource.mimeType ?? "").toLowerCase();
+    const name = String(resource.name ?? "").toLowerCase();
     const shouldParseJson = mimeType.includes("json") || name.endsWith(".json") || name.endsWith(".jsonl");
     if (shouldParseJson) {
         try {
@@ -67,10 +66,14 @@ function isPdfContentDisposition(value) {
     const text = String(value ?? "").toLowerCase();
     return text.includes("application/pdf") || text.includes(".pdf");
 }
-function sanitizePdfFileName(value, fallback) {
+function sanitizeResourceFileName(value, fallback) {
     const name = String(value ?? "").trim() || fallback;
     const cleaned = name.replace(/[\\/:"*?<>|]+/g, "_").replace(/\s+/g, "_").slice(0, 120);
-    return cleaned.toLowerCase().endsWith(".pdf") ? cleaned : `${cleaned || fallback}.pdf`;
+    return cleaned || fallback;
+}
+function sanitizePdfFileName(value, fallback) {
+    const cleaned = sanitizeResourceFileName(value, fallback);
+    return cleaned.toLowerCase().endsWith(".pdf") ? cleaned : `${cleaned}.pdf`;
 }
 function pdfTextRowsSchema() {
     return {
@@ -98,14 +101,14 @@ function parseJsonlDataRows(content) {
         .map((record) => record?.data)
         .filter((row) => row && typeof row === "object" && !Array.isArray(row));
 }
-async function tryMaterializeRawPdfFileSource(state, source, targetDatasetId) {
-    const file = await readInstantFileStep({ runtime: state.runtime, fileId: source.fileId });
+async function tryMaterializeRawPdfFileResource(state, resource, targetDatasetId) {
+    const file = await readInstantFileStep({ runtime: state.runtime, fileId: resource.fileId });
     if (!isPdfContentDisposition(file.contentDisposition))
         return null;
     const sandboxId = resolveDatasetSandboxId(state, targetDatasetId);
     const outputPath = getDatasetOutputPath(targetDatasetId);
-    const fileName = sanitizePdfFileName(parseContentDispositionFileName(file.contentDisposition), `${source.fileId}.pdf`);
-    const sourcePath = `${getDatasetSourcesDir(targetDatasetId)}/${fileName}`;
+    const fileName = sanitizePdfFileName(parseContentDispositionFileName(file.contentDisposition), `${resource.fileId}.pdf`);
+    const resourcePath = `${getDatasetResourcesDir(targetDatasetId)}/${fileName}`;
     const scriptPath = `${getDatasetScriptsDir(targetDatasetId)}/extract_pdf_text.py`;
     await runDatasetSandboxCommandStep({
         runtime: state.runtime,
@@ -116,7 +119,7 @@ async function tryMaterializeRawPdfFileSource(state, source, targetDatasetId) {
     await writeDatasetSandboxFilesStep({
         runtime: state.runtime,
         sandboxId,
-        files: [{ path: sourcePath, contentBase64: file.contentBase64 }],
+        files: [{ path: resourcePath, contentBase64: file.contentBase64 }],
     });
     const install = await runDatasetSandboxCommandStep({
         runtime: state.runtime,
@@ -139,11 +142,11 @@ async function tryMaterializeRawPdfFileSource(state, source, targetDatasetId) {
                     "import sys",
                     "from pypdf import PdfReader",
                     "",
-                    "source_path = Path(sys.argv[1])",
+                    "resource_path = Path(sys.argv[1])",
                     "output_path = Path(sys.argv[2])",
                     "file_id = sys.argv[3]",
                     "file_name = sys.argv[4]",
-                    "reader = PdfReader(str(source_path))",
+                    "reader = PdfReader(str(resource_path))",
                     "rows = 0",
                     "with output_path.open('w', encoding='utf-8') as out:",
                     "    for index, page in enumerate(reader.pages, start=1):",
@@ -173,7 +176,7 @@ async function tryMaterializeRawPdfFileSource(state, source, targetDatasetId) {
         runtime: state.runtime,
         sandboxId,
         cmd: "python",
-        args: [scriptPath, sourcePath, outputPath, source.fileId, fileName],
+        args: [scriptPath, resourcePath, outputPath, resource.fileId, fileName],
     });
     if (extraction.exitCode !== 0) {
         throw new Error(`dataset_pdf_text_extraction_failed:${extraction.stderr || extraction.stdout}`);
@@ -192,36 +195,45 @@ async function tryMaterializeRawPdfFileSource(state, source, targetDatasetId) {
         sandboxId,
         title: state.title ?? fileName,
         instructions: state.instructions,
-        sources: [{ kind: "file", fileId: source.fileId, description: source.description }],
-        sourceKinds: ["file"],
+        contextId: state.contextId ?? "",
         rows,
         schema: pdfTextRowsSchema(),
         first: state.first,
     });
     return targetDatasetId;
 }
-async function materializeRawTextSource(state, source, targetDatasetId) {
-    const rows = materializeRawTextRows(source);
+async function materializeRawTextResource(state, resource, targetDatasetId) {
+    const rows = materializeRawTextRows(resource);
     await materializeRowsToDataset(state.runtime, {
         datasetId: targetDatasetId,
         sandboxId: state.sandboxId,
-        title: state.title ?? source.name ?? targetDatasetId,
+        title: state.title ?? resource.name ?? targetDatasetId,
         instructions: state.instructions,
-        sources: [
-            {
-                kind: "text",
-                mimeType: source.mimeType,
-                name: source.name,
-                description: source.description,
-            },
-        ],
-        sourceKinds: ["text"],
+        contextId: state.contextId ?? "",
         rows,
         schema: state.outputSchema,
         first: state.first,
     });
     return targetDatasetId;
 }
+async function writePreparedFileResourceToSandbox(params) {
+    const file = await readInstantFileStep({ runtime: params.runtime, fileId: params.fileId });
+    const contentDispositionName = parseContentDispositionFileName(file.contentDisposition);
+    const fileName = sanitizeResourceFileName(params.filename ?? contentDispositionName, `${params.fileId}.bin`);
+    const resourcePath = `${getDatasetResourcesDir(params.datasetId)}/${fileName}`;
+    await runDatasetSandboxCommandStep({
+        runtime: params.runtime,
+        sandboxId: params.sandboxId,
+        cmd: "mkdir",
+        args: ["-p", ...getDatasetStandardDirs(params.datasetId)],
+    });
+    await writeDatasetSandboxFilesStep({
+        runtime: params.runtime,
+        sandboxId: params.sandboxId,
+        files: [{ path: resourcePath, contentBase64: file.contentBase64 }],
+    });
+    return { fileName, resourcePath };
+}
 function resolveDatasetSandboxId(state, _targetDatasetId) {
     const sandboxId = String(state.sandboxId ?? "").trim();
     if (sandboxId)
@@ -249,8 +261,7 @@ export async function initializeDatasetStep(params) {
         sandboxId: params.sandboxId,
         title: params.title ?? params.datasetId,
         instructions: params.instructions,
-        sources: params.sources,
-        sourceKinds: params.sourceKinds,
+        contextId: params.contextId,
         schema: params.schema,
         status: "building",
     });
@@ -259,12 +270,12 @@ export async function initializeDatasetStep(params) {
         sandboxId: params.sandboxId,
     };
 }
-export async function prepareDatasetSourcesStep(params) {
+export async function prepareDatasetResourcesStep(params) {
     "use step";
     if (params.kind === "file") {
-        const fileId = params.source.kind === "file"
-            ? params.source.fileId
-            : await uploadInlineTextSource(params.runtime, params.datasetId, params.source);
+        const fileId = params.resource.kind === "file"
+            ? params.resource.fileId
+            : await uploadInlineTextResource(params.runtime, params.datasetId, params.resource);
         return {
             kind: "file",
             datasetId: params.datasetId,
@@ -273,31 +284,31 @@ export async function prepareDatasetSourcesStep(params) {
             sandboxState: { initialized: false, filePath: "" },
             filePreview: undefined,
             schema: params.schema ?? null,
-            filename: params.source.kind === "file" ? params.source.filename : params.source.name,
-            mediaType: params.source.kind === "file" ? params.source.mediaType : params.source.mimeType,
+            filename: params.resource.kind === "file" ? params.resource.filename : params.resource.name,
+            mediaType: params.resource.kind === "file" ? params.resource.mediaType : params.resource.mimeType,
         };
     }
-    const initialized = await ensureTransformSourcesInSandboxStep({
+    const initialized = await ensureTransformInputsInSandboxStep({
         runtime: params.runtime,
         sandboxId: params.sandboxId,
         datasetId: params.datasetId,
-        sourceDatasetIds: params.sourceDatasetIds,
-        state: { initialized: false, sourcePaths: [] },
+        inputDatasetIds: params.inputDatasetIds,
+        state: { initialized: false, inputPaths: [] },
     });
-    const sourcePreviews = await generateTransformSourcePreviewsStep({
+    const inputPreviews = await generateTransformInputPreviewsStep({
         runtime: params.runtime,
         sandboxId: params.sandboxId,
         datasetId: params.datasetId,
-        sourcePaths: initialized.sourcePaths,
+        inputPaths: initialized.inputPaths,
     });
     return {
         kind: "transform",
         datasetId: params.datasetId,
         sandboxId: params.sandboxId,
-        sourceDatasetIds: params.sourceDatasetIds,
+        inputDatasetIds: params.inputDatasetIds,
         outputSchema: params.outputSchema,
         sandboxState: initialized.state,
-        sourcePreviews,
+        inputPreviews,
     };
 }
 export async function initializeDatasetContextStep(params) {
@@ -312,9 +323,9 @@ export async function initializeDatasetContextStep(params) {
     return {
         ...params.prepared,
         instructions: params.instructions,
-        prompt: params.prepared.sourceDatasetIds.length === 1
-            ? "Transform the source dataset into a new dataset matching the provided output schema"
-            : `Transform ${params.prepared.sourceDatasetIds.length} source datasets into a new dataset matching the provided output schema`,
+        prompt: params.prepared.inputDatasetIds.length === 1
+            ? "Transform the input dataset into a new dataset matching the provided output schema"
+            : `Transform ${params.prepared.inputDatasetIds.length} input datasets into a new dataset matching the provided output schema`,
     };
 }
 export async function completeDatasetStep(params) {
@@ -363,9 +374,9 @@ export async function completeDatasetStep(params) {
         firstRow: firstResult.row,
     };
 }
-export async function materializeSingleFileLikeSource(state, source, targetDatasetId) {
-    if (source.kind === "file" && !state.outputSchema) {
-        const materializedPdf = await tryMaterializeRawPdfFileSource(state, source, targetDatasetId);
+export async function materializeSingleFileLikeResource(state, resource, targetDatasetId) {
+    if (resource.kind === "file" && !state.outputSchema) {
+        const materializedPdf = await tryMaterializeRawPdfFileResource(state, resource, targetDatasetId);
         if (materializedPdf)
             return materializedPdf;
     }
@@ -379,35 +390,32 @@ export async function materializeSingleFileLikeSource(state, source, targetDatas
         sandboxId,
         title: state.title ?? targetDatasetId,
         instructions: state.instructions,
-        sources: [
-            source.kind === "file"
-                ? {
-                    kind: "file",
-                    fileId: source.fileId,
-                    description: source.description,
-                    filename: source.filename,
-                    mediaType: source.mediaType,
-                }
-                : {
-                    kind: "text",
-                    mimeType: source.mimeType,
-                    name: source.name,
-                    description: source.description,
-                },
-        ],
-        sourceKinds: [source.kind],
+        contextId: state.contextId ?? "",
         schema: state.outputSchema,
     });
-    const prepared = await prepareDatasetSourcesStep({
+    const prepared = await prepareDatasetResourcesStep({
         kind: "file",
         runtime: state.runtime,
         datasetId: targetDatasetId,
         sandboxId,
-        source,
+        resource,
         schema: state.outputSchema,
     });
+    if (prepared.kind !== "file") {
+        throw new Error("dataset_context_kind_mismatch:file");
+    }
+    const preparedFile = await writePreparedFileResourceToSandbox({
+        runtime: state.runtime,
+        sandboxId,
+        datasetId: targetDatasetId,
+        fileId: prepared.fileId,
+        filename: prepared.filename,
+    });
     const context = await initializeDatasetContextStep({
-        prepared,
+        prepared: {
+            ...prepared,
+            filename: prepared.filename ?? preparedFile.fileName,
+        },
         instructions: state.instructions,
         outputSchema: state.outputSchema,
     });
@@ -442,36 +450,40 @@ export async function materializeSingleFileLikeSource(state, source, targetDatas
     });
     return targetDatasetId;
 }
-async function normalizeSourceToDatasetId(state, source, targetDatasetId, sourceIndex) {
-    if (source.kind === "dataset") {
-        return source.datasetId;
+async function normalizeResourceToDatasetId(state, resource, targetDatasetId, resourceIndex) {
+    if (resource.kind === "dataset") {
+        return resource.datasetId;
     }
-    const intermediateDatasetId = makeIntermediateDatasetId(targetDatasetId, source.kind, sourceIndex);
-    if (source.kind === "query") {
-        await materializeQuerySource(state.runtime, source, {
+    const intermediateDatasetId = makeIntermediateDatasetId(targetDatasetId, resource.kind, resourceIndex);
+    if (resource.kind === "query") {
+        await materializeQueryResource(state.runtime, resource, {
             datasetId: intermediateDatasetId,
             sandboxId: state.sandboxId,
-            title: source.title,
+            title: resource.title,
             first: false,
+            contextId: state.contextId ?? "",
         });
         return intermediateDatasetId;
     }
-    if (source.kind === "text") {
-        await materializeRawTextSource({
+    if (resource.kind === "text") {
+        await materializeRawTextResource({
             ...state,
             outputSchema: undefined,
             first: false,
-            instructions: buildRawSourceInstructions(source.kind),
-            title: source.name ?? state.title,
-        }, source, intermediateDatasetId);
+            instructions: buildRawResourceInstructions(resource.kind),
+            title: resource.name ?? state.title,
+        }, resource, intermediateDatasetId);
         return intermediateDatasetId;
     }
-    await materializeSingleFileLikeSource({
+    if (resource.kind === "context") {
+        throw new Error("dataset_context_resource_must_be_resolved_before_materialization");
+    }
+    await materializeSingleFileLikeResource({
         ...state,
         outputSchema: undefined,
         first: false,
-        instructions: buildRawSourceInstructions(source.kind),
-    }, source, intermediateDatasetId);
+        instructions: buildRawResourceInstructions(resource.kind),
+    }, resource, intermediateDatasetId);
     return intermediateDatasetId;
 }
 export async function materializeDerivedDataset(state, targetDatasetId) {
@@ -480,9 +492,9 @@ export async function materializeDerivedDataset(state, targetDatasetId) {
     }
     const sandboxId = resolveDatasetSandboxId(state, targetDatasetId);
     const stateWithSandbox = { ...state, sandboxId };
-    const normalizedSources = [];
-    for (let index = 0; index < stateWithSandbox.sources.length; index++) {
-        normalizedSources.push(await normalizeSourceToDatasetId(stateWithSandbox, stateWithSandbox.sources[index], targetDatasetId, index));
+    const normalizedResources = [];
+    for (let index = 0; index < stateWithSandbox.resources.length; index++) {
+        normalizedResources.push(await normalizeResourceToDatasetId(stateWithSandbox, stateWithSandbox.resources[index], targetDatasetId, index));
     }
     const transformSchema = stateWithSandbox.outputSchema ??
         {
@@ -500,60 +512,51 @@ export async function materializeDerivedDataset(state, targetDatasetId) {
         sandboxId,
         title: stateWithSandbox.title ?? targetDatasetId,
         instructions: stateWithSandbox.instructions,
-        sources: stateWithSandbox.sources.map((source) => source.kind === "query"
-            ? {
-                kind: "query",
-                query: source.query,
-                title: source.title,
-                explanation: source.explanation,
-                ...getDomainDescriptor(source.domain),
-            }
-            : source),
-        sourceKinds: stateWithSandbox.sources.map((source) => source.kind),
+        contextId: stateWithSandbox.contextId ?? "",
         schema: transformSchema,
     });
-    const prepared = await prepareDatasetSourcesStep({
+    const prepared = await prepareDatasetResourcesStep({
         kind: "transform",
         runtime: stateWithSandbox.runtime,
         datasetId: targetDatasetId,
         sandboxId,
-        sourceDatasetIds: normalizedSources,
+        inputDatasetIds: normalizedResources,
         outputSchema: transformSchema,
     });
     const context = await initializeDatasetContextStep({
         prepared,
-        instructions: buildTransformInstructions(normalizedSources.length, stateWithSandbox.instructions, stateWithSandbox.outputSchema),
+        instructions: buildTransformInstructions(normalizedResources.length, stateWithSandbox.instructions, stateWithSandbox.outputSchema),
         outputSchema: transformSchema,
     });
     if (context.kind !== "transform") {
         throw new Error("dataset_context_kind_mismatch:transform");
     }
     const transformContext = createTransformDatasetContext({
-        sourceDatasetIds: context.sourceDatasetIds,
+        inputDatasetIds: context.inputDatasetIds,
         outputSchema: context.outputSchema,
         instructions: context.instructions,
         datasetId: context.datasetId,
         reactor: stateWithSandbox.reactor,
         sandboxId: context.sandboxId,
         sandboxState: context.sandboxState,
-        sourcePreviews: context.sourcePreviews,
+        inputPreviews: context.inputPreviews,
     });
     await transformContext.transform(stateWithSandbox.runtime, {
         durable: await resolveDatasetAgentDurable(stateWithSandbox.durable),
         prompt: context.prompt,
         initialContent: {
             datasetId: context.datasetId,
-            sourceDatasetIds: context.sourceDatasetIds,
+            inputDatasetIds: context.inputDatasetIds,
             outputSchema: context.outputSchema,
             instructions: context.instructions,
             sandboxId: context.sandboxId,
             sandboxState: context.sandboxState,
-            sourcePreviews: context.sourcePreviews,
+            inputPreviews: context.inputPreviews,
         },
     });
     return targetDatasetId;
 }
 registerDatasetAgentMaterializers({
-    materializeSingleFileLikeSource,
+    materializeSingleFileLikeResource,
     materializeDerivedDataset,
 });

package/dist/builder/materializeQuery.d.ts

@@ -1,5 +1,5 @@
-import type { AnyDatasetRuntime, DatasetBuilderState, DatasetSchemaInput, InternalSource } from "./types.js";
-export declare function materializeQuerySource<Runtime extends AnyDatasetRuntime>(runtime: DatasetBuilderState<Runtime>["runtime"], source: Extract<InternalSource, {
+import type { AnyDatasetRuntime, DatasetBuilderState, DatasetSchemaInput, InternalDatasetResource } from "./types.js";
+export declare function materializeQueryResource<Runtime extends AnyDatasetRuntime>(runtime: DatasetBuilderState<Runtime>["runtime"], resource: Extract<InternalDatasetResource, {
     kind: "query";
 }>, params: {
     datasetId: string;
@@ -8,4 +8,5 @@ export declare function materializeQuerySource<Runtime extends AnyDatasetRuntime
     title?: string;
     instructions?: string;
     first?: boolean;
+    contextId: string;
 }): Promise<string>;

package/dist/builder/materializeQuery.js

@@ -1,35 +1,26 @@
 import { materializeRowsToDataset } from "./persistence.js";
-import { getDomainDescriptor, normalizeQueryRows } from "./sourceRows.js";
-async function readQuerySourceRowsStep(params) {
+import { getDomainDescriptor, normalizeQueryRows } from "./rows.js";
+async function readQueryResourceRowsStep(params) {
     "use step";
     const db = await params.runtime.db();
     const result = await db.query(params.query);
     return { rows: normalizeQueryRows(result) };
 }
-export async function materializeQuerySource(runtime, source, params) {
-    const { rows } = await readQuerySourceRowsStep({
+export async function materializeQueryResource(runtime, resource, params) {
+    const { rows } = await readQueryResourceRowsStep({
         runtime,
-        query: source.query,
+        query: resource.query,
     });
-    const domainDescriptor = getDomainDescriptor(source.domain);
+    const domainDescriptor = getDomainDescriptor(resource.domain);
     return await materializeRowsToDataset(runtime, {
         datasetId: params.datasetId,
         sandboxId: params.sandboxId,
-        title: params.title ?? source.title,
+        title: params.title ?? resource.title,
         instructions: params.instructions,
-        sources: [
-            {
-                kind: "query",
-                query: source.query,
-                title: source.title,
-                explanation: source.explanation,
-                ...domainDescriptor,
-            },
-        ],
-        sourceKinds: ["query"],
+        contextId: params.contextId,
         analysis: {
-            query: source.query,
-            explanation: source.explanation,
+            query: resource.query,
+            explanation: resource.explanation,
             ...domainDescriptor,
         },
         rows,

package/dist/builder/persistence.d.ts

@@ -1,19 +1,18 @@
-import type { AnyDatasetRuntime, DatasetBuildResult, DatasetTextSourceInput, MaterializeRowsParams } from "./types.js";
-export declare function defaultTextSourceName(source: DatasetTextSourceInput): string;
+import type { AnyDatasetRuntime, DatasetBuildResult, DatasetTextResourceInput, MaterializeRowsParams } from "./types.js";
+export declare function defaultTextResourceName(resource: DatasetTextResourceInput): string;
 export declare function getDatasetDb<Runtime extends AnyDatasetRuntime>(runtime: Runtime): Promise<any>;
 export declare function createOrUpdateDatasetMetadata<Runtime extends AnyDatasetRuntime>(runtime: Runtime, params: {
     datasetId: string;
     sandboxId?: string;
     title?: string;
     instructions?: string;
-    sources: any[];
-    sourceKinds: string[];
+    contextId: string;
     analysis?: any;
     schema?: any;
     status?: string;
 }): Promise<void>;
 export declare function materializeRowsToDataset<Runtime extends AnyDatasetRuntime>(runtime: Runtime, params: MaterializeRowsParams): Promise<string>;
-export declare function uploadInlineTextSource<Runtime extends AnyDatasetRuntime>(runtime: Runtime, datasetId: string, source: DatasetTextSourceInput): Promise<string>;
+export declare function uploadInlineTextResource<Runtime extends AnyDatasetRuntime>(runtime: Runtime, datasetId: string, resource: DatasetTextResourceInput): Promise<string>;
 export declare function finalizeBuildResult<Runtime extends AnyDatasetRuntime>(runtime: Runtime, datasetId: string, withFirst: boolean): Promise<DatasetBuildResult>;
 export declare function createDatasetBuildResult<Runtime extends AnyDatasetRuntime>(runtime: Runtime, params: {
     datasetId: string;

package/dist/builder/persistence.js

@@ -2,18 +2,18 @@ import { DatasetService } from "../service.js";
 import { datasetDomain } from "../schema.js";
 import { datasetGetByIdStep, datasetPreviewRowsStep, datasetReadOneStep, datasetReadRowsStep, } from "../dataset/steps.js";
 import { inferDatasetSchema, validateRows } from "./schemaInference.js";
-import { rowsToJsonl } from "./sourceRows.js";
-export function defaultTextSourceName(source) {
-    if (source.name?.trim())
-        return source.name.trim();
-    const mimeType = String(source.mimeType ?? "").toLowerCase();
+import { rowsToJsonl } from "./rows.js";
+export function defaultTextResourceName(resource) {
+    if (resource.name?.trim())
+        return resource.name.trim();
+    const mimeType = String(resource.mimeType ?? "").toLowerCase();
     if (mimeType.includes("csv"))
-        return "source.csv";
+        return "resource.csv";
     if (mimeType.includes("json"))
-        return "source.json";
+        return "resource.json";
     if (mimeType.includes("yaml") || mimeType.includes("yml"))
-        return "source.yaml";
-    return "source.txt";
+        return "resource.yaml";
+    return "resource.txt";
 }
 export async function getDatasetDb(runtime) {
     const scoped = await runtime.use(datasetDomain);
@@ -21,6 +21,9 @@ export async function getDatasetDb(runtime) {
 }
 export async function createOrUpdateDatasetMetadata(runtime, params) {
     "use step";
+    if (!params.contextId.trim()) {
+        throw new Error("dataset_context_required");
+    }
     const db = await getDatasetDb(runtime);
     const service = new DatasetService(db);
     const result = await service.createDataset({
@@ -28,8 +31,7 @@ export async function createOrUpdateDatasetMetadata(runtime, params) {
         sandboxId: params.sandboxId,
         title: params.title ?? params.datasetId,
         instructions: params.instructions ?? "",
-        sources: params.sources,
-        sourceKinds: params.sourceKinds,
+        contextId: params.contextId,
         analysis: params.analysis,
         schema: params.schema,
         status: params.status ?? "building",
@@ -52,8 +54,7 @@ export async function materializeRowsToDataset(runtime, params) {
         sandboxId: params.sandboxId,
         title: params.title,
         instructions: params.instructions,
-        sources: params.sources,
-        sourceKinds: params.sourceKinds,
+        contextId: params.contextId,
         analysis: params.analysis,
         schema: resolvedSchema,
         status: "building",
@@ -78,18 +79,18 @@ export async function materializeRowsToDataset(runtime, params) {
     }
     return params.datasetId;
 }
-export async function uploadInlineTextSource(runtime, datasetId, source) {
+export async function uploadInlineTextResource(runtime, datasetId, resource) {
     "use step";
     const db = await getDatasetDb(runtime);
-    const fileName = defaultTextSourceName(source);
-    const storagePath = `/dataset/source/${datasetId}/${Date.now()}-${fileName}`;
-    const uploadResult = await db.storage.uploadFile(storagePath, Buffer.from(source.text, "utf-8"), {
-        contentType: source.mimeType ?? "text/plain",
+    const fileName = defaultTextResourceName(resource);
+    const storagePath = `/dataset/resource/${datasetId}/${Date.now()}-${fileName}`;
+    const uploadResult = await db.storage.uploadFile(storagePath, Buffer.from(resource.text, "utf-8"), {
+        contentType: resource.mimeType ?? "text/plain",
         contentDisposition: fileName,
     });
     const fileId = uploadResult?.data?.id;
     if (!fileId) {
-        throw new Error("dataset_text_source_upload_failed");
+        throw new Error("dataset_text_resource_upload_failed");
     }
     return fileId;
 }