deepline 0.1.148 → 0.1.150

package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts

@@ -38,12 +38,21 @@ import type {
   ToolResultTargetAccessor,
   ToolResultTargetMetadata,
 } from './tool-result-types';
-import { createPlayDataset, type PlayDataset } from '../plays/dataset';
+import {
+  createDeferredPlayDataset,
+  createPlayDataset,
+  isSerializedPlayDataset,
+  trimSerializedPlayDatasetPreview,
+  type PlayDataset,
+  type SerializedPlayDataset,
+} from '../plays/dataset';
 import { normalizeTableNamespace, sha256Hex } from '../plays/row-identity';
 
 type PathSegment = string | number | '*';
 
 const SERIALIZED_TOOL_EXECUTE_RESULT_KIND = 'deepline.tool_execute_result.v1';
+const SERIALIZED_TOOL_RESULT_LIST_PREVIEW_LIMIT = 5;
+const SERIALIZED_TOOL_LIST_ROWS = Symbol('deepline.serialized_tool_list_rows');
 
 const TARGET_FALLBACK_KEYS: Record<string, readonly RegExp[]> = {
   email: [/^email$/i, /^address$/i, /email/i],
@@ -949,6 +958,23 @@ function buildExtractedAccessors(
   );
 }
 
+function serializedTargetValuesFromResult(
+  value: ToolExecuteResult,
+): Record<string, ToolResultTargetMetadata> | undefined {
+  const targets = value._metadata.targets;
+  return Object.keys(targets).length > 0 ? targets : undefined;
+}
+
+function applySerializedTargetValues(
+  result: ToolExecuteResult,
+  targetValues: Record<string, ToolResultTargetMetadata> | undefined,
+): ToolExecuteResult {
+  if (!targetValues || Object.keys(targetValues).length === 0) return result;
+  result._metadata.targets = targetValues;
+  result.extractedValues = buildExtractedAccessors(targetValues);
+  return result;
+}
+
 function buildListAccessors(
   resolved: Record<string, { path: string; rows: Record<string, unknown>[] }>,
   lists: Record<string, ToolResultListMetadata>,
@@ -1142,13 +1168,286 @@ export function readList(
 ): PlayDataset<Record<string, unknown>> | Record<string, unknown>[] {
   if (selector) {
     const paths = Array.isArray(selector) ? selector : [selector];
+    const declaredList = findDeclaredListAccessor(result, paths);
     const found = findFirstTargetByPath(resultRootOf(result), paths)?.value;
     const rows = normalizeRows(found);
-    if (rows) return rows;
+    if (rows) {
+      if (
+        declaredList &&
+        typeof declaredList.count === 'number' &&
+        declaredList.count > rows.length
+      ) {
+        return declaredList.get();
+      }
+      return rows;
+    }
+    if (declaredList) return declaredList.get();
   }
   return Object.values(result.extractedLists)[0]?.get() ?? [];
 }
 
+function normalizeListPathForMatch(path: string): string {
+  return path.replace(/\[(?:\*|\d+)\]$/g, '');
+}
+
+function findDeclaredListAccessor(
+  result: ToolExecuteResult,
+  paths: readonly string[],
+): ToolResultListAccessor | undefined {
+  const candidates = new Set(
+    paths.flatMap((path) =>
+      candidateResultPaths(path).flatMap((candidate) => [
+        candidate,
+        normalizeListPathForMatch(candidate),
+      ]),
+    ),
+  );
+  return Object.values(result.extractedLists).find((list) =>
+    candidates.has(normalizeListPathForMatch(list.path)),
+  );
+}
+
+function replaceAtPath(
+  root: unknown,
+  path: string,
+  replacement: unknown,
+): unknown {
+  const segments = parsePath(path);
+  if (segments.length === 0 || segments.includes('*')) return root;
+  const replace = (current: unknown, index: number): unknown => {
+    if (index >= segments.length) return replacement;
+    const segment = segments[index]!;
+    if (typeof segment === 'number') {
+      if (!Array.isArray(current)) return current;
+      const copy = [...current];
+      copy[segment] = replace(copy[segment], index + 1);
+      return copy;
+    }
+    if (!isRecord(current)) return current;
+    return {
+      ...current,
+      [segment]: replace(current[segment], index + 1),
+    };
+  };
+  return replace(root, 0);
+}
+
+function serializedListDatasetsFromResult(
+  value: ToolExecuteResult,
+): Record<string, SerializedPlayDataset<Record<string, unknown>>> | undefined {
+  const entries = Object.entries(value.extractedLists).flatMap(
+    ([name, accessor]) => {
+      const serialized = accessor.get().toJSON();
+      if (!isSerializedPlayDataset<Record<string, unknown>>(serialized)) {
+        return [];
+      }
+      return [
+        [
+          name,
+          trimSerializedPlayDatasetPreview(
+            serialized,
+            SERIALIZED_TOOL_RESULT_LIST_PREVIEW_LIMIT,
+          ),
+        ],
+      ] as const;
+    },
+  );
+  return entries.length > 0 ? Object.fromEntries(entries) : undefined;
+}
+
+function serializedListRowsFromRaw(input: {
+  raw: unknown;
+  metadata: ToolResultMetadataInput;
+}): Record<string, Array<Record<string, unknown>>> | undefined {
+  const resolved = resolveListRows(
+    { toolResponse: { raw: input.raw } },
+    input.metadata.listExtractorPaths,
+  );
+  const entries = Object.entries(resolved).map(([name, list]) => [
+    name,
+    list.rows,
+  ]);
+  return entries.length > 0 ? Object.fromEntries(entries) : undefined;
+}
+
+function preservedSerializedListRows(
+  value: ToolExecuteResult,
+): Record<string, Array<Record<string, unknown>>> | undefined {
+  const rows = (value as ToolExecuteResult & {
+    [SERIALIZED_TOOL_LIST_ROWS]?:
+      | Record<string, Array<Record<string, unknown>>>
+      | undefined;
+  })[SERIALIZED_TOOL_LIST_ROWS];
+  return rows && Object.keys(rows).length > 0 ? rows : undefined;
+}
+
+function serializedListRowsFromResult(input: {
+  value: ToolExecuteResult;
+  metadata: ToolResultMetadataInput;
+  listDatasets:
+    | Record<string, SerializedPlayDataset<Record<string, unknown>>>
+    | undefined;
+}): Record<string, Array<Record<string, unknown>>> | undefined {
+  const preserved = preservedSerializedListRows(input.value);
+  const rawRows = serializedListRowsFromRaw({
+    raw: input.value.toolResponse.raw,
+    metadata: input.metadata,
+  });
+  const names = new Set([
+    ...Object.keys(input.listDatasets ?? {}),
+    ...Object.keys(preserved ?? {}),
+    ...Object.keys(rawRows ?? {}),
+  ]);
+  const entries = [...names].flatMap((name) => {
+    const dataset = input.listDatasets?.[name];
+    const preservedRows = preserved?.[name];
+    if (
+      preservedRows &&
+      (!dataset || preservedRows.length >= dataset.count)
+    ) {
+      return [[name, preservedRows]] as const;
+    }
+    const rows = rawRows?.[name];
+    if (rows && (!dataset || rows.length >= dataset.count)) {
+      return [[name, rows]] as const;
+    }
+    return [];
+  });
+  return entries.length > 0 ? Object.fromEntries(entries) : undefined;
+}
+
+function serializedRawWithListPreviews(input: {
+  raw: unknown;
+  metadata: ToolResultMetadataInput;
+  listDatasets:
+    | Record<string, SerializedPlayDataset<Record<string, unknown>>>
+    | undefined;
+}): unknown {
+  if (!input.listDatasets) return input.raw;
+  let root: unknown = { toolResponse: { raw: input.raw } };
+  const datasetsBySource = new Map(
+    Object.values(input.listDatasets).flatMap((dataset) =>
+      dataset.sourceLabel ? [[dataset.sourceLabel, dataset]] : [],
+    ),
+  );
+  for (const rawPath of input.metadata.listExtractorPaths ?? []) {
+    const candidates = [...candidateResultPaths(rawPath)].filter(
+      (candidate, index, all) => all.indexOf(candidate) === index,
+    );
+    const dataset =
+      candidates
+        .map((candidate) => datasetsBySource.get(candidate))
+        .find(Boolean) ?? Object.values(input.listDatasets)[0];
+    if (!dataset) continue;
+    for (const candidate of candidates) {
+      if (!Array.isArray(getAtPath(root, candidate))) continue;
+      root = replaceAtPath(root, candidate, dataset.preview);
+      break;
+    }
+  }
+  return isRecord(root) &&
+    isRecord(root.toolResponse) &&
+    Object.prototype.hasOwnProperty.call(root.toolResponse, 'raw')
+    ? root.toolResponse.raw
+    : input.raw;
+}
+
+function createDatasetFromSerializedToolList(
+  serialized: SerializedPlayDataset<Record<string, unknown>>,
+  rows?: Array<Record<string, unknown>>,
+): PlayDataset<Record<string, unknown>> {
+  const preview = serialized.preview;
+  const sourceRows = rows ?? preview;
+  const isPartialPreview =
+    rows === undefined && serialized.count > preview.length;
+  const partialPreviewError = (): Error =>
+    new Error(
+      `Serialized tool list ${serialized.datasetId} only carries ${preview.length} preview row(s) for ${serialized.count} total row(s). ` +
+        'Return the Dataset Handle from play code or export rows through runs export for full data.',
+    );
+  const collectPreview = (limit?: number): Record<string, unknown>[] => {
+    if (limit === undefined && isPartialPreview) {
+      throw partialPreviewError();
+    }
+    const requested =
+      limit === undefined ? sourceRows.length : Math.max(0, Math.floor(limit));
+    if (requested > preview.length && isPartialPreview) {
+      throw partialPreviewError();
+    }
+    return sourceRows.slice(0, requested);
+  };
+  return createDeferredPlayDataset({
+    datasetKind: serialized.datasetKind,
+    datasetId: serialized.datasetId,
+    count: serialized.count,
+    backing: serialized.backing,
+    previewRows: preview,
+    sourceLabel: serialized.sourceLabel ?? null,
+    tableNamespace: serialized.tableNamespace ?? null,
+    workProgress: serialized._metadata?.workProgress,
+    resolvers: {
+      count: async () => serialized.count,
+      peek: async (limit) => collectPreview(limit),
+      materialize: async (limit) => collectPreview(limit),
+      iterate: () =>
+        ({
+          async *[Symbol.asyncIterator]() {
+            if (isPartialPreview) {
+              throw partialPreviewError();
+            }
+            yield* sourceRows;
+          },
+        }) as AsyncIterable<Record<string, unknown>>,
+    },
+  });
+}
+
+function applySerializedListDatasets(
+  result: ToolExecuteResult,
+  listDatasets:
+    | Record<string, SerializedPlayDataset<Record<string, unknown>>>
+    | undefined,
+  listRows: Record<string, Array<Record<string, unknown>>> | undefined,
+): ToolExecuteResult {
+  if (!listDatasets) return result;
+  if (listRows && Object.keys(listRows).length > 0) {
+    Object.defineProperty(result, SERIALIZED_TOOL_LIST_ROWS, {
+      value: listRows,
+      enumerable: false,
+      configurable: false,
+      writable: false,
+    });
+  }
+  for (const [name, serialized] of Object.entries(listDatasets)) {
+    if (!isSerializedPlayDataset<Record<string, unknown>>(serialized)) {
+      continue;
+    }
+    const dataset = createDatasetFromSerializedToolList(
+      serialized,
+      listRows?.[name],
+    );
+    const existingMetadata = result._metadata.lists[name];
+    result._metadata.lists[name] = {
+      path: serialized.sourceLabel ?? existingMetadata?.path ?? name,
+      count: serialized.count,
+      keys: existingMetadata?.keys ?? {},
+    };
+    const accessor = {
+      path: result._metadata.lists[name].path,
+      count: serialized.count,
+      keys: result._metadata.lists[name].keys,
+    } as ToolResultListAccessor;
+    Object.defineProperty(accessor, 'get', {
+      value() {
+        return dataset;
+      },
+      enumerable: false,
+    });
+    result.extractedLists[name] = accessor;
+  }
+  return result;
+}
+
 function metadataInputFromToolExecuteResult(
   value: ToolExecuteResult,
 ): ToolResultMetadataInput {
@@ -1177,14 +1476,31 @@ function metadataInputFromToolExecuteResult(
 export function serializeToolExecuteResult(
   value: ToolExecuteResult,
 ): SerializedToolExecuteResult {
+  const metadata = metadataInputFromToolExecuteResult(value);
+  const listDatasets = serializedListDatasetsFromResult(value);
+  const listRows = serializedListRowsFromResult({
+    value,
+    listDatasets,
+    metadata,
+  });
+  const targetValues = serializedTargetValuesFromResult(value);
   return {
     __kind: SERIALIZED_TOOL_EXECUTE_RESULT_KIND,
     status: value.status,
+    ...(typeof value.job_id === 'string' ? { job_id: value.job_id } : {}),
+    ...(isRecord(value.meta) ? { meta: value.meta } : {}),
     toolResponse: {
-      raw: value.toolResponse.raw,
+      raw: serializedRawWithListPreviews({
+        raw: value.toolResponse.raw,
+        metadata,
+        listDatasets,
+      }),
       ...(value.toolResponse.meta ? { meta: value.toolResponse.meta } : {}),
     },
-    metadata: metadataInputFromToolExecuteResult(value),
+    ...(listDatasets ? { listDatasets } : {}),
+    ...(listRows ? { listRows } : {}),
+    ...(targetValues ? { targetValues } : {}),
+    metadata,
     execution: value._metadata.execution,
   };
 }
@@ -1205,33 +1521,34 @@ export function isSerializedToolExecuteResult(
 export function deserializeToolExecuteResult(
   value: SerializedToolExecuteResult,
 ): ToolExecuteResult {
-  return createToolExecuteResult({
-    status: value.status,
-    result: {
-      data: value.toolResponse.raw,
-      ...(value.toolResponse.meta ? { meta: value.toolResponse.meta } : {}),
-    },
-    metadata: value.metadata,
-    execution: value.execution,
-  });
+  return applySerializedListDatasets(
+    applySerializedTargetValues(
+      createToolExecuteResult({
+        status: value.status,
+        jobId: value.job_id,
+        result: {
+          data: value.toolResponse.raw,
+          ...(value.toolResponse.meta
+            ? { meta: value.toolResponse.meta }
+            : {}),
+        },
+        metadata: value.metadata,
+        execution: value.execution,
+        meta: value.meta,
+      }),
+      value.targetValues,
+    ),
+    value.listDatasets,
+    value.listRows,
+  );
 }
 
 export function cloneToolExecuteResultWithExecution<TResult>(
   value: ToolExecuteResult<TResult>,
   execution: ToolResultExecutionMetadata,
 ): ToolExecuteResult<TResult> {
-  return createToolExecuteResult<TResult>({
-    status: value.status,
-    jobId:
-      typeof (value as unknown as { job_id?: unknown }).job_id === 'string'
-        ? (value as unknown as { job_id: string }).job_id
-        : undefined,
-    result: {
-      data: value.toolResponse.raw,
-      ...(value.toolResponse.meta ? { meta: value.toolResponse.meta } : {}),
-    } as TResult,
-    metadata: metadataInputFromToolExecuteResult(value),
+  return deserializeToolExecuteResult({
+    ...serializeToolExecuteResult(value),
     execution,
-    meta: isRecord(value.meta) ? value.meta : undefined,
-  });
+  }) as ToolExecuteResult<TResult>;
 }