deepline 0.1.163 → 0.1.165

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -3559,6 +3559,18 @@ function resolveSheetContractFromReq(
   }
 }
 
+function staticPipelineFromReq(req: RunRequest): PlayStaticPipeline | null {
+  const snapshot = req.contractSnapshot as
+    | { staticPipeline?: unknown }
+    | undefined
+    | null;
+  const pipeline = snapshot?.staticPipeline;
+  if (!pipeline || typeof pipeline !== 'object' || Array.isArray(pipeline)) {
+    return null;
+  }
+  return pipeline as PlayStaticPipeline;
+}
+
 /**
  * Direct-Neon writes from the shared harness Worker. Resolves the org runtime
  * Postgres URL via `create_db_session` inside that long-lived Worker, then
@@ -4290,6 +4302,7 @@ function createMinimalWorkerCtx(
       mapName: name,
       rowCountHint,
       executionPlan: plan,
+      staticPipeline: staticPipelineFromReq(req),
     });
     const outputFields = fieldEntries.map(([field]) => field);
     const updateMapProgress = (

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts

@@ -1,83 +1,319 @@
 import {
   chooseMapChunkSize,
-  EXECUTION_PLAN_DEFAULTS,
   type ExecutionPlan,
 } from '../../../../shared_libs/play-runtime/execution-plan';
+import { getPlayRuntimeBatchStrategy } from '../../../../shared_libs/play-runtime/play-runtime-batching-registry';
 import { TOOL_CALLING_MAP_CHUNK_SIZE } from '../../../../shared_libs/play-runtime/map-chunk-limits';
+import {
+  flattenStaticSubsteps,
+  getTopLevelPipelineSubsteps,
+  type PlayStaticColumnProducer,
+  type PlayStaticPipeline,
+  type PlayStaticSubstep,
+} from '../../../../shared_libs/plays/static-pipeline';
 
 export const CACHE_ENABLED_SIMPLE_MAP_CHUNK_SIZE = 10_000;
 export { TOOL_CALLING_MAP_CHUNK_SIZE };
-export const WORKER_SUBREQUEST_SAFE_TOOL_CALLS_PER_CHUNK =
-  TOOL_CALLING_MAP_CHUNK_SIZE;
+// Cloudflare preview Workers enforce the 50-subrequest invocation limit. Leave
+// headroom for coordinator/storage calls around row-level unbatched tool RPCs.
+export const UNBATCHED_TOOL_SUBREQUESTS_PER_CHUNK_BUDGET = 40;
+// Fresh unbatched tool calls use one RUNTIME_API integration execute RPC and
+// one HARNESS durable-receipt completion RPC. Batch-cap rows by both.
+export const SUBREQUESTS_PER_UNBATCHED_TOOL_CALL = 2;
 
 export type WorkerMapChunkPlanInput = {
   mapName: string;
   rowCountHint: number | null;
   executionPlan?: ExecutionPlan | null;
+  staticPipeline?: PlayStaticPipeline | null;
+};
+
+type PlanMap = ExecutionPlan['maps'][number];
+type ChunkSizingMap = Pick<
+  PlanMap,
+  | 'mapName'
+  | 'tableNamespace'
+  | 'outputFields'
+  | 'defaultChunkSize'
+  | 'stepsPerChunk'
+> &
+  Partial<Pick<PlanMap, 'externalStepFields'>>;
+type ChunkSizingPlan = {
+  maps: ChunkSizingMap[];
+  toolDeclarations?: ExecutionPlan['toolDeclarations'];
+  softWorkflowStepBudget?: number | null;
 };
 
+type MapToolStats = [totalToolCount: number, unbatchedToolCount: number];
+
+function declarationBelongsToMapOutput(
+  declaration: { field?: string | null },
+  outputFields: readonly string[],
+  externalStepFields: readonly string[] = [],
+): boolean {
+  const fieldName = declaration.field;
+  if (fieldName == null) return true;
+  if (
+    outputFields.some(
+      (field) => fieldName === field || fieldName.startsWith(`${field}.`),
+    )
+  ) {
+    return true;
+  }
+  return externalStepFields.some(
+    (field) => fieldName === field || fieldName.endsWith(`.${field}`),
+  );
+}
+
+function isUnbatchedTool(toolId: string): boolean {
+  return getPlayRuntimeBatchStrategy(toolId) === null;
+}
+
+function countToolSubsteps(
+  substeps: readonly PlayStaticSubstep[],
+): MapToolStats {
+  let totalToolCount = 0;
+  let unbatchedToolCount = 0;
+
+  for (const substep of substeps) {
+    if (substep.type === 'tool') {
+      totalToolCount += 1;
+      if (isUnbatchedTool(substep.toolId)) {
+        unbatchedToolCount += 1;
+      }
+      continue;
+    }
+
+    if (substep.type === 'waterfall') {
+      for (const step of substep.steps ?? []) {
+        if (!step.toolId) continue;
+        totalToolCount += 1;
+        if (isUnbatchedTool(step.toolId)) {
+          unbatchedToolCount += 1;
+        }
+      }
+      continue;
+    }
+
+    if (
+      substep.type === 'step_suite' ||
+      substep.type === 'control_flow' ||
+      substep.type === 'dataset'
+    ) {
+      const [nestedTotal, nestedUnbatched] = countToolSubsteps(
+        substep.steps ?? [],
+      );
+      totalToolCount += nestedTotal;
+      unbatchedToolCount += nestedUnbatched;
+    }
+  }
+
+  return [totalToolCount, unbatchedToolCount];
+}
+
+function countProducerTools(
+  producers: readonly PlayStaticColumnProducer[],
+): MapToolStats {
+  let totalToolCount = 0;
+  let unbatchedToolCount = 0;
+
+  for (const producer of producers) {
+    if (producer.kind === 'tool' && producer.toolId) {
+      totalToolCount += 1;
+      if (isUnbatchedTool(producer.toolId)) {
+        unbatchedToolCount += 1;
+      }
+    }
+    if (producer.substep.type === 'waterfall') {
+      for (const step of producer.substep.steps ?? []) {
+        if (!step.toolId) continue;
+        totalToolCount += 1;
+        if (isUnbatchedTool(step.toolId)) {
+          unbatchedToolCount += 1;
+        }
+      }
+    }
+    if (producer.steps?.length) {
+      const [nestedTotal, nestedUnbatched] = countProducerTools(producer.steps);
+      totalToolCount += nestedTotal;
+      unbatchedToolCount += nestedUnbatched;
+    }
+  }
+
+  return [totalToolCount, unbatchedToolCount];
+}
+
+function datasetBelongsToPlanMap(
+  dataset: Extract<PlayStaticSubstep, { type: 'dataset' }>,
+  planMap: ChunkSizingMap,
+  mapName: string,
+): boolean {
+  return [dataset.name, dataset.tableNamespace, dataset.field].some(
+    (datasetName) =>
+      typeof datasetName === 'string' &&
+      datasetName.length > 0 &&
+      (datasetName === planMap.mapName ||
+        datasetName === planMap.tableNamespace ||
+        datasetName === mapName),
+  );
+}
+
+function countMapLocalTools(input: {
+  staticPipeline: PlayStaticPipeline | null | undefined;
+  planMap: ChunkSizingMap | null | undefined;
+  mapName: string;
+}): MapToolStats | null {
+  if (!input.staticPipeline || !input.planMap) return null;
+  const planMap = input.planMap;
+  const dataset = flattenStaticSubsteps(
+    getTopLevelPipelineSubsteps(input.staticPipeline),
+  ).find(
+    (substep): substep is Extract<PlayStaticSubstep, { type: 'dataset' }> =>
+      substep.type === 'dataset' &&
+      datasetBelongsToPlanMap(substep, planMap, input.mapName),
+  );
+  if (!dataset) return null;
+
+  if (dataset.steps?.length) {
+    return countToolSubsteps(dataset.steps);
+  }
+
+  if (dataset.columns?.length) {
+    return countProducerTools(
+      dataset.columns.flatMap((column) => column.producers),
+    );
+  }
+
+  return [0, 0];
+}
+
+function countFallbackDeclarationTools(input: {
+  declarations: readonly { toolId: string; field?: string | null }[];
+  outputFields: readonly string[];
+  externalStepFields?: readonly string[];
+}): MapToolStats {
+  let totalToolCount = 0;
+  let unbatchedToolCount = 0;
+
+  for (const declaration of input.declarations) {
+    if (
+      !declarationBelongsToMapOutput(
+        declaration,
+        input.outputFields,
+        input.externalStepFields ?? [],
+      )
+    ) {
+      continue;
+    }
+    totalToolCount += 1;
+    if (isUnbatchedTool(declaration.toolId)) {
+      unbatchedToolCount += 1;
+    }
+  }
+
+  return [totalToolCount, unbatchedToolCount];
+}
+
+function countDeclarationTools(
+  declarations: readonly { toolId: string; field?: string | null }[],
+): MapToolStats {
+  return [
+    declarations.length,
+    declarations.filter((declaration) => isUnbatchedTool(declaration.toolId))
+      .length,
+  ];
+}
+
+function countNestedExecutionSteps(
+  substeps: readonly PlayStaticSubstep[] | undefined,
+): number {
+  let stepsPerChunk = 1;
+  for (const substep of substeps ?? []) {
+    if (substep.type === 'waterfall') {
+      stepsPerChunk = Math.max(stepsPerChunk, substep.steps?.length ?? 1);
+      continue;
+    }
+    if (substep.type === 'step_suite') {
+      stepsPerChunk = Math.max(stepsPerChunk, substep.steps.length);
+      continue;
+    }
+    if (substep.type === 'control_flow' || substep.type === 'dataset') {
+      stepsPerChunk = Math.max(
+        stepsPerChunk,
+        countNestedExecutionSteps(substep.steps),
+      );
+    }
+  }
+  return stepsPerChunk;
+}
+
+function buildStaticChunkSizingPlan(
+  staticPipeline: PlayStaticPipeline | null,
+): ChunkSizingPlan | null {
+  if (!staticPipeline) return null;
+  const datasets = flattenStaticSubsteps(
+    getTopLevelPipelineSubsteps(staticPipeline),
+  ).filter(
+    (substep): substep is Extract<PlayStaticSubstep, { type: 'dataset' }> =>
+      substep.type === 'dataset',
+  );
+  if (datasets.length === 0) return null;
+  return {
+    maps: datasets.map((dataset) => {
+      const stepsPerChunk = countNestedExecutionSteps(dataset.steps);
+      return {
+        mapName: dataset.name ?? dataset.field,
+        tableNamespace: dataset.tableNamespace ?? dataset.field,
+        outputFields: dataset.outputFields ?? [],
+        defaultChunkSize: stepsPerChunk > 1 ? 1_000 : 5_000,
+        stepsPerChunk,
+      };
+    }),
+  };
+}
+
 export function chooseWorkerMapRowsPerChunk(
   input: WorkerMapChunkPlanInput,
 ): number {
-  const plan = input.executionPlan ?? null;
+  const plan: ChunkSizingPlan | null = input.executionPlan
+    ? {
+        maps: input.executionPlan.maps,
+        toolDeclarations: input.executionPlan.toolDeclarations,
+        softWorkflowStepBudget:
+          input.executionPlan.chunkPlan.softWorkflowStepBudget,
+      }
+    : buildStaticChunkSizingPlan(input.staticPipeline ?? null);
   const planMap = plan?.maps.find(
     (candidate) =>
       candidate.mapName === input.mapName ||
       candidate.tableNamespace === input.mapName,
   );
+  const staticMapToolStats = countMapLocalTools({
+    staticPipeline: input.staticPipeline,
+    planMap,
+    mapName: input.mapName,
+  });
+  const mapToolStats =
+    staticMapToolStats ??
+    (plan && planMap
+      ? countFallbackDeclarationTools({
+          declarations: plan.toolDeclarations ?? [],
+          outputFields: planMap.outputFields,
+          externalStepFields: planMap.externalStepFields ?? [],
+        })
+      : plan
+        ? countDeclarationTools(plan.toolDeclarations ?? [])
+        : [0, 0]);
   const rowsPerChunk = chooseMapChunkSize({
     totalRows: input.rowCountHint,
     mapCount: Math.max(1, plan?.maps.length ?? 1),
     stepsPerChunk: planMap?.stepsPerChunk ?? 1,
     preferredChunkSize: planMap?.defaultChunkSize,
-    softWorkflowStepBudget: plan?.chunkPlan.softWorkflowStepBudget,
+    softWorkflowStepBudget: plan?.softWorkflowStepBudget,
   });
-  const estimatedSubrequestProducingSteps =
-    estimateSubrequestProducingStepsForMap(plan, planMap);
-  if (estimatedSubrequestProducingSteps > 0) {
-    const subrequestSafeRows = Math.max(
-      1,
-      Math.floor(
-        WORKER_SUBREQUEST_SAFE_TOOL_CALLS_PER_CHUNK /
-          estimatedSubrequestProducingSteps,
-      ),
-    );
-    if (subrequestSafeRows < rowsPerChunk) {
-      if (input.rowCountHint === null || !Number.isFinite(input.rowCountHint)) {
-        return subrequestSafeRows;
-      }
-      const totalRows = Math.max(0, Math.floor(input.rowCountHint));
-      const mapCount = Math.max(1, plan?.maps.length ?? 1);
-      const softBudget =
-        plan?.chunkPlan.softWorkflowStepBudget ??
-        EXECUTION_PLAN_DEFAULTS.workflowSoftStepBudget;
-      const maxChunksPerMap = Math.max(
-        1,
-        Math.floor(
-          Math.max(
-            mapCount,
-            Math.floor(
-              (softBudget -
-                EXECUTION_PLAN_DEFAULTS.ingestStepCount -
-                EXECUTION_PLAN_DEFAULTS.finalizationStepCount) /
-                estimatedSubrequestProducingSteps,
-            ),
-          ) / mapCount,
-        ),
-      );
-      const requiredChunks = Math.ceil(totalRows / subrequestSafeRows);
-      if (requiredChunks <= maxChunksPerMap) return subrequestSafeRows;
-      throw new Error(
-        `Worker budget exceeded for map "${input.mapName}": ${requiredChunks}/${maxChunksPerMap} chunks.`,
-      );
-    }
-    return rowsPerChunk;
-  }
 
   const toolFreeSimpleMap =
-    !!planMap &&
-    planMap.stepsPerChunk === 1 &&
-    estimateSubrequestProducingStepsForMap(plan, planMap) === 0;
+    !!planMap && planMap.stepsPerChunk === 1 && mapToolStats[0] === 0;
   if (
     toolFreeSimpleMap &&
     (input.rowCountHint === null ||
@@ -86,75 +322,22 @@ export function chooseWorkerMapRowsPerChunk(
     return Math.max(rowsPerChunk, CACHE_ENABLED_SIMPLE_MAP_CHUNK_SIZE);
   }
 
-  return rowsPerChunk;
-}
-
-function estimateSubrequestProducingStepsForMap(
-  plan: ExecutionPlan | null,
-  planMap: ExecutionPlan['maps'][number] | undefined,
-): number {
-  const toolDeclarations = plan?.toolDeclarations ?? [];
-  if (!planMap) {
-    return toolDeclarations.length;
-  }
-  const explicitExternalSteps = Array.isArray(planMap.externalStepFields)
-    ? planMap.externalStepFields.length
-    : 0;
-  const mapStepsPerChunk =
-    explicitExternalSteps > 0
-      ? 0
-      : (planMap.stepsPerChunk ?? 1) > 1
-        ? (planMap.stepsPerChunk ?? 1)
-        : 0;
-  if (toolDeclarations.length === 0) {
-    return Math.max(mapStepsPerChunk, explicitExternalSteps);
-  }
-  const outputFields = new Set(
-    planMap.outputFields.map((field) => field.trim()).filter(Boolean),
-  );
-  const explicitExternalStepFields = new Set(
-    (planMap.externalStepFields ?? [])
-      .map((field) => field.trim())
-      .filter(Boolean),
-  );
-  const hasModernStepMetadata =
-    Array.isArray(planMap.stepFields) ||
-    Array.isArray(planMap.externalStepFields);
-  const mapFields = new Set([
-    planMap.mapName,
-    planMap.tableNamespace,
-    ...planMap.outputFields,
-    ...(planMap.externalStepFields ?? []),
-    ...(hasModernStepMetadata ? [] : (planMap.stepFields ?? [])),
-    ...planMap.waterfallStages.flatMap((stage) => [
-      stage.waterfallId,
-      ...stage.stageIds,
-    ]),
-  ]);
-  let scopedToolDeclarations = 0;
-  for (const declaration of toolDeclarations) {
-    const field = declaration.field?.trim();
-    if (!field) continue;
-    if (mapFields.has(field)) {
-      scopedToolDeclarations += 1;
-      continue;
-    }
-    if (!field.includes('.')) {
-      continue;
-    }
-    const firstSegment = field.slice(0, field.indexOf('.'));
-    const lastSegment = field.slice(field.lastIndexOf('.') + 1);
-    if (
-      outputFields.has(firstSegment) ||
-      explicitExternalStepFields.has(lastSegment) ||
-      (!hasModernStepMetadata && outputFields.has(lastSegment))
-    ) {
-      scopedToolDeclarations += 1;
-    }
+  if (mapToolStats[1] > 0) {
+    const unbatchedToolCallsPerRow =
+      staticMapToolStats !== null
+        ? mapToolStats[1]
+        : Math.max(planMap?.stepsPerChunk ?? 1, mapToolStats[1]);
+    const unbatchedSubrequestsPerRow =
+      unbatchedToolCallsPerRow * SUBREQUESTS_PER_UNBATCHED_TOOL_CALL;
+    const unbatchedToolRowsPerChunk = Math.max(
+      1,
+      Math.floor(
+        UNBATCHED_TOOL_SUBREQUESTS_PER_CHUNK_BUDGET /
+          unbatchedSubrequestsPerRow,
+      ),
+    );
+    return Math.min(rowsPerChunk, unbatchedToolRowsPerChunk);
   }
-  return Math.max(
-    mapStepsPerChunk,
-    explicitExternalSteps,
-    scopedToolDeclarations,
-  );
+
+  return rowsPerChunk;
 }

package/dist/bundling-sources/sdk/src/release.ts

@@ -104,10 +104,10 @@ export const SDK_RELEASE = {
   // 0.1.111 ships dataset-native tool list getters and result row datasets.
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
-  version: '0.1.163',
+  version: '0.1.165',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.163',
+    latest: '0.1.165',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
     commandMinimumSupported: [

package/dist/bundling-sources/sdk/src/types.ts

@@ -319,6 +319,15 @@ export interface ToolSearchResult {
   search_mode?: 'v1' | 'v2';
   /** Whether search fell back to category matching. */
   search_fallback_to_category?: boolean;
+  /** Explanation and next commands when filters/search succeed but match zero tools. */
+  emptyResult?: {
+    reason: string;
+    message: string;
+    suggestions: Array<{
+      label: string;
+      command: string;
+    }>;
+  };
   /** Hint explaining omitted play results when searching tools only. */
   omitted_plays_hint?: string;
   /** Copyable CLI command templates for follow-up discovery/execution. */