deepline 0.1.149 → 0.1.150

package/dist/bundling-sources/sdk/src/client.ts

@@ -84,6 +84,7 @@ import type { EnrichCompiledConfig } from './cli/enrich-play-compiler.js';
 const TERMINAL_PLAY_STATUSES = new Set(['completed', 'failed', 'cancelled']);
 const INCLUDE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
 const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
+const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
 const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-response';
 const COMPILE_MANIFEST_RETRY_DELAYS_MS = [250, 1_000];
 const REGISTER_PLAY_ARTIFACTS_COMPILE_CONCURRENCY = 3;
@@ -163,6 +164,7 @@ function chunkRegisterPlayArtifacts<T>(artifacts: T[]): T[][] {
 
 type ExecuteToolRawOptions = {
   includeToolMetadata?: boolean;
+  responseIntent?: 'raw' | 'row_artifact';
 };
 
 /**
@@ -1156,6 +1158,9 @@ export class DeeplineClient {
       ...(options?.includeToolMetadata
         ? { [INCLUDE_TOOL_METADATA_HEADER]: 'true' }
         : {}),
+      ...(options?.responseIntent
+        ? { [EXECUTE_RESPONSE_INTENT_HEADER]: options.responseIntent }
+        : {}),
     };
     return this.http.post<ToolExecution<TData, TMeta>>(
       `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`,

package/dist/bundling-sources/sdk/src/release.ts

@@ -102,10 +102,10 @@ export const SDK_RELEASE = {
   // the SDK enrich generator's one-second stale policy.
   // 0.1.110 ships authored V2 prebuilts and required top-level play descriptions.
   // 0.1.111 ships dataset-native tool list getters and result row datasets.
-  version: '0.1.149',
+  version: '0.1.150',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.149',
+    latest: '0.1.150',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
     commandMinimumSupported: [

package/dist/bundling-sources/sdk/src/tool-output.ts

@@ -23,7 +23,13 @@
  *
  * @module
  */
-import { mkdirSync, writeFileSync } from 'node:fs';
+import {
+  closeSync,
+  mkdirSync,
+  openSync,
+  writeFileSync,
+  writeSync,
+} from 'node:fs';
 import { homedir } from 'node:os';
 import { dirname, join } from 'node:path';
 
@@ -54,6 +60,22 @@ export type ListConversionResult = {
   sourcePath: string | null;
 };
 
+export type RowOutputProjection = {
+  rows: Array<Record<string, unknown>>;
+  rowCount: number;
+  columns: string[];
+  previewRows: Array<Record<string, unknown>>;
+  strategy: ListConversionResult['strategy'];
+  sourcePath: string | null;
+};
+
+export type CsvOutputArtifact = {
+  path: string;
+  rowCount: number;
+  columns: string[];
+  preview: string;
+};
+
 type Scalar = string | number | boolean | null;
 
 function isPlainObject(value: unknown): value is Record<string, unknown> {
@@ -109,6 +131,20 @@ function normalizeRows(value: unknown): Array<Record<string, unknown>> | null {
   });
 }
 
+function columnsForRows(rows: readonly Record<string, unknown>[]): string[] {
+  const seen = new Set<string>();
+  const columns: string[] = [];
+  for (const row of rows) {
+    for (const key of Object.keys(row)) {
+      if (!seen.has(key)) {
+        seen.add(key);
+        columns.push(key);
+      }
+    }
+  }
+  return columns;
+}
+
 /**
  * Generate candidate root objects to search for lists.
  * Tries: raw payload → V2 toolResponse.raw → legacy payload.output.body → legacy payload.result → legacy payload.result.data.
@@ -271,6 +307,19 @@ export function tryConvertToList(
   return null;
 }
 
+export function projectRowOutput(
+  conversion: ListConversionResult,
+): RowOutputProjection {
+  return {
+    rows: conversion.rows,
+    rowCount: conversion.rows.length,
+    columns: columnsForRows(conversion.rows),
+    previewRows: conversion.rows.slice(0, 5),
+    strategy: conversion.strategy,
+    sourcePath: conversion.sourcePath,
+  };
+}
+
 /** Ensure the shared output directory exists. Returns its path. */
 function ensureOutputDir(): string {
   const outputDir = join(homedir(), '.local', 'share', 'deepline', 'data');
@@ -330,21 +379,12 @@ export function writeCsvOutputFile(
   rows: Array<Record<string, unknown>>,
   stem: string,
   options?: { outPath?: string },
-): { path: string; rowCount: number; columns: string[]; preview: string } {
+): CsvOutputArtifact {
   const outputPath = options?.outPath
     ? options.outPath
     : join(ensureOutputDir(), `${stem}_${Date.now()}.csv`);
   mkdirSync(dirname(outputPath), { recursive: true });
-  const seen = new Set<string>();
-  const columns: string[] = [];
-  for (const row of rows) {
-    for (const key of Object.keys(row)) {
-      if (!seen.has(key)) {
-        seen.add(key);
-        columns.push(key);
-      }
-    }
-  }
+  const columns = columnsForRows(rows);
 
   const escapeCell = (value: unknown): string => {
     const normalized =
@@ -361,12 +401,18 @@ export function writeCsvOutputFile(
     return normalized;
   };
 
-  const lines: string[] = [];
-  lines.push(columns.map(escapeCell).join(','));
-  for (const row of rows) {
-    lines.push(columns.map((column) => escapeCell(row[column])).join(','));
+  const fd = openSync(outputPath, 'w');
+  try {
+    writeSync(fd, `${columns.map(escapeCell).join(',')}\n`);
+    for (const row of rows) {
+      writeSync(
+        fd,
+        `${columns.map((column) => escapeCell(row[column])).join(',')}\n`,
+      );
+    }
+  } finally {
+    closeSync(fd);
   }
-  writeFileSync(outputPath, `${lines.join('\n')}\n`, 'utf-8');
 
   const previewRows = rows.slice(0, 5);
   const previewColumns = columns.slice(0, 5);

package/dist/bundling-sources/shared_libs/play-runtime/tool-result-types.ts

@@ -1,6 +1,6 @@
 import type { EmailStatusExtractorConfig } from './email-status';
 import type { DeeplineGetterValueMap } from './extractor-targets';
-import type { PlayDataset } from '../plays/dataset';
+import type { PlayDataset, SerializedPlayDataset } from '../plays/dataset';
 
 export type ToolResultExecutionMetadata = {
   idempotent: true;
@@ -73,10 +73,22 @@ export type ToolResultEnvelope<
 export type SerializedToolExecuteResult = {
   __kind: 'deepline.tool_execute_result.v1';
   status: string;
+  job_id?: string;
+  /** Deepline-owned execution/result metadata. */
+  meta?: Record<string, unknown>;
   toolResponse: {
+    /**
+     * Raw provider/tool data with declared row-list payloads clipped to bounded
+     * previews. Use list getters for row-shaped data.
+     */
     raw: unknown;
     meta?: Record<string, unknown>;
   };
+  listDatasets?: Record<string, SerializedPlayDataset<Record<string, unknown>>>;
+  /** Full declared list rows used to rehydrate list Dataset Handles. */
+  listRows?: Record<string, Array<Record<string, unknown>>>;
+  /** Computed scalar/object getters preserved before raw list previews are clipped. */
+  targetValues?: Record<string, ToolResultTargetMetadata>;
   metadata: ToolResultMetadataInput;
   execution: ToolResultExecutionMetadata;
 };
@@ -142,9 +154,10 @@ export type ToolExecuteResultAccessors<
  * getters live under `extractedValues.<name>.get()`, and list getters live
  * under `extractedLists.<name>.get()`.
  *
- * Use extractors first when a tool contract exposes them. Drop to
- * `toolResponse.raw` when you need provider-specific fields or when debugging
- * from persisted run rows.
+ * Use extractors first when a tool contract exposes them. Use list getters for
+ * row-shaped data. Drop to `toolResponse.raw` only for provider-specific scalar
+ * fields or bounded debugging context; persisted rows may clip declared lists to
+ * previews.
  *
  * @sdkReference runtime 200
  */

package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts

@@ -38,12 +38,21 @@ import type {
   ToolResultTargetAccessor,
   ToolResultTargetMetadata,
 } from './tool-result-types';
-import { createPlayDataset, type PlayDataset } from '../plays/dataset';
+import {
+  createDeferredPlayDataset,
+  createPlayDataset,
+  isSerializedPlayDataset,
+  trimSerializedPlayDatasetPreview,
+  type PlayDataset,
+  type SerializedPlayDataset,
+} from '../plays/dataset';
 import { normalizeTableNamespace, sha256Hex } from '../plays/row-identity';
 
 type PathSegment = string | number | '*';
 
 const SERIALIZED_TOOL_EXECUTE_RESULT_KIND = 'deepline.tool_execute_result.v1';
+const SERIALIZED_TOOL_RESULT_LIST_PREVIEW_LIMIT = 5;
+const SERIALIZED_TOOL_LIST_ROWS = Symbol('deepline.serialized_tool_list_rows');
 
 const TARGET_FALLBACK_KEYS: Record<string, readonly RegExp[]> = {
   email: [/^email$/i, /^address$/i, /email/i],
@@ -949,6 +958,23 @@ function buildExtractedAccessors(
   );
 }
 
+function serializedTargetValuesFromResult(
+  value: ToolExecuteResult,
+): Record<string, ToolResultTargetMetadata> | undefined {
+  const targets = value._metadata.targets;
+  return Object.keys(targets).length > 0 ? targets : undefined;
+}
+
+function applySerializedTargetValues(
+  result: ToolExecuteResult,
+  targetValues: Record<string, ToolResultTargetMetadata> | undefined,
+): ToolExecuteResult {
+  if (!targetValues || Object.keys(targetValues).length === 0) return result;
+  result._metadata.targets = targetValues;
+  result.extractedValues = buildExtractedAccessors(targetValues);
+  return result;
+}
+
 function buildListAccessors(
   resolved: Record<string, { path: string; rows: Record<string, unknown>[] }>,
   lists: Record<string, ToolResultListMetadata>,
@@ -1142,13 +1168,286 @@ export function readList(
 ): PlayDataset<Record<string, unknown>> | Record<string, unknown>[] {
   if (selector) {
     const paths = Array.isArray(selector) ? selector : [selector];
+    const declaredList = findDeclaredListAccessor(result, paths);
     const found = findFirstTargetByPath(resultRootOf(result), paths)?.value;
     const rows = normalizeRows(found);
-    if (rows) return rows;
+    if (rows) {
+      if (
+        declaredList &&
+        typeof declaredList.count === 'number' &&
+        declaredList.count > rows.length
+      ) {
+        return declaredList.get();
+      }
+      return rows;
+    }
+    if (declaredList) return declaredList.get();
   }
   return Object.values(result.extractedLists)[0]?.get() ?? [];
 }
 
+function normalizeListPathForMatch(path: string): string {
+  return path.replace(/\[(?:\*|\d+)\]$/g, '');
+}
+
+function findDeclaredListAccessor(
+  result: ToolExecuteResult,
+  paths: readonly string[],
+): ToolResultListAccessor | undefined {
+  const candidates = new Set(
+    paths.flatMap((path) =>
+      candidateResultPaths(path).flatMap((candidate) => [
+        candidate,
+        normalizeListPathForMatch(candidate),
+      ]),
+    ),
+  );
+  return Object.values(result.extractedLists).find((list) =>
+    candidates.has(normalizeListPathForMatch(list.path)),
+  );
+}
+
+function replaceAtPath(
+  root: unknown,
+  path: string,
+  replacement: unknown,
+): unknown {
+  const segments = parsePath(path);
+  if (segments.length === 0 || segments.includes('*')) return root;
+  const replace = (current: unknown, index: number): unknown => {
+    if (index >= segments.length) return replacement;
+    const segment = segments[index]!;
+    if (typeof segment === 'number') {
+      if (!Array.isArray(current)) return current;
+      const copy = [...current];
+      copy[segment] = replace(copy[segment], index + 1);
+      return copy;
+    }
+    if (!isRecord(current)) return current;
+    return {
+      ...current,
+      [segment]: replace(current[segment], index + 1),
+    };
+  };
+  return replace(root, 0);
+}
+
+function serializedListDatasetsFromResult(
+  value: ToolExecuteResult,
+): Record<string, SerializedPlayDataset<Record<string, unknown>>> | undefined {
+  const entries = Object.entries(value.extractedLists).flatMap(
+    ([name, accessor]) => {
+      const serialized = accessor.get().toJSON();
+      if (!isSerializedPlayDataset<Record<string, unknown>>(serialized)) {
+        return [];
+      }
+      return [
+        [
+          name,
+          trimSerializedPlayDatasetPreview(
+            serialized,
+            SERIALIZED_TOOL_RESULT_LIST_PREVIEW_LIMIT,
+          ),
+        ],
+      ] as const;
+    },
+  );
+  return entries.length > 0 ? Object.fromEntries(entries) : undefined;
+}
+
+function serializedListRowsFromRaw(input: {
+  raw: unknown;
+  metadata: ToolResultMetadataInput;
+}): Record<string, Array<Record<string, unknown>>> | undefined {
+  const resolved = resolveListRows(
+    { toolResponse: { raw: input.raw } },
+    input.metadata.listExtractorPaths,
+  );
+  const entries = Object.entries(resolved).map(([name, list]) => [
+    name,
+    list.rows,
+  ]);
+  return entries.length > 0 ? Object.fromEntries(entries) : undefined;
+}
+
+function preservedSerializedListRows(
+  value: ToolExecuteResult,
+): Record<string, Array<Record<string, unknown>>> | undefined {
+  const rows = (value as ToolExecuteResult & {
+    [SERIALIZED_TOOL_LIST_ROWS]?:
+      | Record<string, Array<Record<string, unknown>>>
+      | undefined;
+  })[SERIALIZED_TOOL_LIST_ROWS];
+  return rows && Object.keys(rows).length > 0 ? rows : undefined;
+}
+
+function serializedListRowsFromResult(input: {
+  value: ToolExecuteResult;
+  metadata: ToolResultMetadataInput;
+  listDatasets:
+    | Record<string, SerializedPlayDataset<Record<string, unknown>>>
+    | undefined;
+}): Record<string, Array<Record<string, unknown>>> | undefined {
+  const preserved = preservedSerializedListRows(input.value);
+  const rawRows = serializedListRowsFromRaw({
+    raw: input.value.toolResponse.raw,
+    metadata: input.metadata,
+  });
+  const names = new Set([
+    ...Object.keys(input.listDatasets ?? {}),
+    ...Object.keys(preserved ?? {}),
+    ...Object.keys(rawRows ?? {}),
+  ]);
+  const entries = [...names].flatMap((name) => {
+    const dataset = input.listDatasets?.[name];
+    const preservedRows = preserved?.[name];
+    if (
+      preservedRows &&
+      (!dataset || preservedRows.length >= dataset.count)
+    ) {
+      return [[name, preservedRows]] as const;
+    }
+    const rows = rawRows?.[name];
+    if (rows && (!dataset || rows.length >= dataset.count)) {
+      return [[name, rows]] as const;
+    }
+    return [];
+  });
+  return entries.length > 0 ? Object.fromEntries(entries) : undefined;
+}
+
+function serializedRawWithListPreviews(input: {
+  raw: unknown;
+  metadata: ToolResultMetadataInput;
+  listDatasets:
+    | Record<string, SerializedPlayDataset<Record<string, unknown>>>
+    | undefined;
+}): unknown {
+  if (!input.listDatasets) return input.raw;
+  let root: unknown = { toolResponse: { raw: input.raw } };
+  const datasetsBySource = new Map(
+    Object.values(input.listDatasets).flatMap((dataset) =>
+      dataset.sourceLabel ? [[dataset.sourceLabel, dataset]] : [],
+    ),
+  );
+  for (const rawPath of input.metadata.listExtractorPaths ?? []) {
+    const candidates = [...candidateResultPaths(rawPath)].filter(
+      (candidate, index, all) => all.indexOf(candidate) === index,
+    );
+    const dataset =
+      candidates
+        .map((candidate) => datasetsBySource.get(candidate))
+        .find(Boolean) ?? Object.values(input.listDatasets)[0];
+    if (!dataset) continue;
+    for (const candidate of candidates) {
+      if (!Array.isArray(getAtPath(root, candidate))) continue;
+      root = replaceAtPath(root, candidate, dataset.preview);
+      break;
+    }
+  }
+  return isRecord(root) &&
+    isRecord(root.toolResponse) &&
+    Object.prototype.hasOwnProperty.call(root.toolResponse, 'raw')
+    ? root.toolResponse.raw
+    : input.raw;
+}
+
+function createDatasetFromSerializedToolList(
+  serialized: SerializedPlayDataset<Record<string, unknown>>,
+  rows?: Array<Record<string, unknown>>,
+): PlayDataset<Record<string, unknown>> {
+  const preview = serialized.preview;
+  const sourceRows = rows ?? preview;
+  const isPartialPreview =
+    rows === undefined && serialized.count > preview.length;
+  const partialPreviewError = (): Error =>
+    new Error(
+      `Serialized tool list ${serialized.datasetId} only carries ${preview.length} preview row(s) for ${serialized.count} total row(s). ` +
+        'Return the Dataset Handle from play code or export rows through runs export for full data.',
+    );
+  const collectPreview = (limit?: number): Record<string, unknown>[] => {
+    if (limit === undefined && isPartialPreview) {
+      throw partialPreviewError();
+    }
+    const requested =
+      limit === undefined ? sourceRows.length : Math.max(0, Math.floor(limit));
+    if (requested > preview.length && isPartialPreview) {
+      throw partialPreviewError();
+    }
+    return sourceRows.slice(0, requested);
+  };
+  return createDeferredPlayDataset({
+    datasetKind: serialized.datasetKind,
+    datasetId: serialized.datasetId,
+    count: serialized.count,
+    backing: serialized.backing,
+    previewRows: preview,
+    sourceLabel: serialized.sourceLabel ?? null,
+    tableNamespace: serialized.tableNamespace ?? null,
+    workProgress: serialized._metadata?.workProgress,
+    resolvers: {
+      count: async () => serialized.count,
+      peek: async (limit) => collectPreview(limit),
+      materialize: async (limit) => collectPreview(limit),
+      iterate: () =>
+        ({
+          async *[Symbol.asyncIterator]() {
+            if (isPartialPreview) {
+              throw partialPreviewError();
+            }
+            yield* sourceRows;
+          },
+        }) as AsyncIterable<Record<string, unknown>>,
+    },
+  });
+}
+
+function applySerializedListDatasets(
+  result: ToolExecuteResult,
+  listDatasets:
+    | Record<string, SerializedPlayDataset<Record<string, unknown>>>
+    | undefined,
+  listRows: Record<string, Array<Record<string, unknown>>> | undefined,
+): ToolExecuteResult {
+  if (!listDatasets) return result;
+  if (listRows && Object.keys(listRows).length > 0) {
+    Object.defineProperty(result, SERIALIZED_TOOL_LIST_ROWS, {
+      value: listRows,
+      enumerable: false,
+      configurable: false,
+      writable: false,
+    });
+  }
+  for (const [name, serialized] of Object.entries(listDatasets)) {
+    if (!isSerializedPlayDataset<Record<string, unknown>>(serialized)) {
+      continue;
+    }
+    const dataset = createDatasetFromSerializedToolList(
+      serialized,
+      listRows?.[name],
+    );
+    const existingMetadata = result._metadata.lists[name];
+    result._metadata.lists[name] = {
+      path: serialized.sourceLabel ?? existingMetadata?.path ?? name,
+      count: serialized.count,
+      keys: existingMetadata?.keys ?? {},
+    };
+    const accessor = {
+      path: result._metadata.lists[name].path,
+      count: serialized.count,
+      keys: result._metadata.lists[name].keys,
+    } as ToolResultListAccessor;
+    Object.defineProperty(accessor, 'get', {
+      value() {
+        return dataset;
+      },
+      enumerable: false,
+    });
+    result.extractedLists[name] = accessor;
+  }
+  return result;
+}
+
 function metadataInputFromToolExecuteResult(
   value: ToolExecuteResult,
 ): ToolResultMetadataInput {
@@ -1177,14 +1476,31 @@ function metadataInputFromToolExecuteResult(
 export function serializeToolExecuteResult(
   value: ToolExecuteResult,
 ): SerializedToolExecuteResult {
+  const metadata = metadataInputFromToolExecuteResult(value);
+  const listDatasets = serializedListDatasetsFromResult(value);
+  const listRows = serializedListRowsFromResult({
+    value,
+    listDatasets,
+    metadata,
+  });
+  const targetValues = serializedTargetValuesFromResult(value);
   return {
     __kind: SERIALIZED_TOOL_EXECUTE_RESULT_KIND,
     status: value.status,
+    ...(typeof value.job_id === 'string' ? { job_id: value.job_id } : {}),
+    ...(isRecord(value.meta) ? { meta: value.meta } : {}),
     toolResponse: {
-      raw: value.toolResponse.raw,
+      raw: serializedRawWithListPreviews({
+        raw: value.toolResponse.raw,
+        metadata,
+        listDatasets,
+      }),
       ...(value.toolResponse.meta ? { meta: value.toolResponse.meta } : {}),
     },
-    metadata: metadataInputFromToolExecuteResult(value),
+    ...(listDatasets ? { listDatasets } : {}),
+    ...(listRows ? { listRows } : {}),
+    ...(targetValues ? { targetValues } : {}),
+    metadata,
     execution: value._metadata.execution,
   };
 }
@@ -1205,33 +1521,34 @@ export function isSerializedToolExecuteResult(
 export function deserializeToolExecuteResult(
   value: SerializedToolExecuteResult,
 ): ToolExecuteResult {
-  return createToolExecuteResult({
-    status: value.status,
-    result: {
-      data: value.toolResponse.raw,
-      ...(value.toolResponse.meta ? { meta: value.toolResponse.meta } : {}),
-    },
-    metadata: value.metadata,
-    execution: value.execution,
-  });
+  return applySerializedListDatasets(
+    applySerializedTargetValues(
+      createToolExecuteResult({
+        status: value.status,
+        jobId: value.job_id,
+        result: {
+          data: value.toolResponse.raw,
+          ...(value.toolResponse.meta
+            ? { meta: value.toolResponse.meta }
+            : {}),
+        },
+        metadata: value.metadata,
+        execution: value.execution,
+        meta: value.meta,
+      }),
+      value.targetValues,
+    ),
+    value.listDatasets,
+    value.listRows,
+  );
 }
 
 export function cloneToolExecuteResultWithExecution<TResult>(
   value: ToolExecuteResult<TResult>,
   execution: ToolResultExecutionMetadata,
 ): ToolExecuteResult<TResult> {
-  return createToolExecuteResult<TResult>({
-    status: value.status,
-    jobId:
-      typeof (value as unknown as { job_id?: unknown }).job_id === 'string'
-        ? (value as unknown as { job_id: string }).job_id
-        : undefined,
-    result: {
-      data: value.toolResponse.raw,
-      ...(value.toolResponse.meta ? { meta: value.toolResponse.meta } : {}),
-    } as TResult,
-    metadata: metadataInputFromToolExecuteResult(value),
+  return deserializeToolExecuteResult({
+    ...serializeToolExecuteResult(value),
     execution,
-    meta: isRecord(value.meta) ? value.meta : undefined,
-  });
+  }) as ToolExecuteResult<TResult>;
 }