deepline 0.2.53 → 0.2.55

package/dist/bundling-sources/shared_libs/plays/bundling/index.ts

@@ -14,7 +14,6 @@ import {
 } from 'node:path';
 import { builtinModules } from 'node:module';
 import { Parser } from 'acorn';
-import { tsPlugin } from 'acorn-typescript';
 import { build, transformSync, type Message, type Plugin } from 'esbuild';
 import {
   PLAY_ARTIFACT_KINDS,
@@ -29,15 +28,39 @@ import type {
   PlayRuntimeFeature,
 } from '../artifact-types';
 import { buildPlayContractCompatibility } from '../contracts';
+import { parsePlayDocflowFile, type PlayDocflowBinding } from '../docflow';
+import {
+  TypeScriptParser,
+  astArray,
+  astNodeBounds,
+  buildDocflowParentIndex,
+  docflowExpressionWrapsStepBuilder,
+  docflowOutputRoot,
+  findDocflowBoundStatement,
+  isAstNode,
+  resolveDocflowBinding,
+  sourceLineStarts,
+  type AstNode,
+} from '../docflow-binding';
+export {
+  collectDocflowBindingDrift,
+  resolveDocflowBindingLines,
+  resolveDocflowBindingSymbol,
+  type DocflowBindingDriftDetail,
+  type DocflowBindingResolution,
+  type DocflowSymbolResolution,
+} from '../docflow-binding';
 import type { ToolExecutionErrorSchemaVersion } from '../../tool-execution-error';
 import type { PlaySandboxRuntimeDeclaration } from '../../play-runtime/sandbox-runtime-limits';
 import { validatePlaySourceFilesHaveNoInlineSecrets } from '../secret-guardrails';
 import { MAX_PLAY_BUNDLE_BYTES } from './limits';
 import { PLAY_AUTHORING_CONTRACT_EDITION } from '../authoring-contract';
 
-// The authored tool-error schema is part of the artifact bytes. Do not reuse a
-// local bundle cached before compatibility selection entered graph analysis.
-const PLAY_BUNDLE_CACHE_VERSION = 28;
+// The authored tool-error schema and the authored docflow block are both part
+// of the artifact bytes. Do not reuse a local bundle cached before either
+// compatibility selection or docflow instrumentation entered graph analysis.
+// Keep this aligned with the app and SDK adapters' cache namespace.
+const PLAY_BUNDLE_CACHE_VERSION = 33;
 const PLAY_ARTIFACT_CACHE_DIR = join(
   tmpdir(),
   `deepline-play-artifacts-v${PLAY_BUNDLE_CACHE_VERSION}`,
@@ -515,11 +538,6 @@ export function extractDefinedPlayDescriptionForExport(
   );
 }
 
-type AstNode = {
-  type: string;
-  [key: string]: unknown;
-};
-
 type PlayMetadataExtractionContext = {
   declarations: Map<string, AstNode | null>;
   namedExports: Map<string, string>;
@@ -535,16 +553,6 @@ type ExtractedPlayMetadata = {
   toolErrorSchemaVersionUnknown: boolean;
 };
 
-const TypeScriptParser = Parser.extend(
-  tsPlugin({
-    allowSatisfies: true,
-    jsx: {
-      allowNamespaces: true,
-      allowNamespacedObjects: true,
-    },
-  }) as unknown as (BaseParser: typeof Parser) => typeof Parser,
-);
-
 function parsePlaySourceAst(sourceCode: string): AstNode | null {
   try {
     return TypeScriptParser.parse(sourceCode, {
@@ -580,14 +588,6 @@ function getIdentifierName(node: unknown): string | null {
     : null;
 }
 
-function isAstNode(value: unknown): value is AstNode {
-  return value !== null && typeof value === 'object' && 'type' in value;
-}
-
-function astArray(value: unknown): AstNode[] {
-  return Array.isArray(value) ? value.filter(isAstNode) : [];
-}
-
 function memberExpressionPath(
   node: AstNode | null | undefined,
 ): string[] | null {
@@ -1345,6 +1345,348 @@ function localSdkAliasPlugin(adapter: PlayBundlingAdapter): Plugin | null {
   };
 }
 
+function sourceLoaderForPath(path: string): 'ts' | 'tsx' | 'js' | 'jsx' {
+  const extension = extname(path).toLowerCase();
+  if (extension === '.tsx') return 'tsx';
+  if (extension === '.jsx') return 'jsx';
+  if (extension === '.js' || extension === '.mjs' || extension === '.cjs') {
+    return 'js';
+  }
+  return 'ts';
+}
+
+function docflowContextIdentifierFromCall(call: AstNode): string | null {
+  if (!isDefinePlayCallExpression(call)) return null;
+  const args = astArray(call.arguments);
+  const callback = args.find(
+    (argument) =>
+      argument.type === 'ArrowFunctionExpression' ||
+      argument.type === 'FunctionExpression',
+  );
+  if (!callback) return null;
+  return getIdentifierName(astArray(callback.params)[0]);
+}
+
+/**
+ * Finds the context parameter for the definePlay call that owns one binding.
+ * A file may export several plays with different parameter names, so a single
+ * file-global context identifier is not safe for instrumentation.
+ */
+function findDocflowContextIdentifier(input: {
+  ast: AstNode;
+  statement: AstNode | null;
+  bindingLine: number;
+  lineStarts: readonly number[];
+  parents: ReadonlyMap<AstNode, AstNode>;
+}): string | null {
+  let owner: AstNode | null = input.statement;
+  while (owner) {
+    const contextName = docflowContextIdentifierFromCall(owner);
+    if (contextName) return contextName;
+    owner = input.parents.get(owner) ?? null;
+  }
+
+  // Positional instrumentation can survive an AST resolver abstention. In that
+  // case, choose the smallest definePlay call containing the bound line.
+  const lineStart = input.lineStarts[input.bindingLine - 1];
+  const lineEnd =
+    input.lineStarts[input.bindingLine] ?? Number.POSITIVE_INFINITY;
+  if (lineStart === undefined) return null;
+  let best: AstNode | null = null;
+  const pending: unknown[] = [input.ast];
+  while (pending.length > 0) {
+    const value = pending.pop();
+    if (!isAstNode(value)) continue;
+    if (isDefinePlayCallExpression(value)) {
+      const bounds = astNodeBounds(value);
+      if (
+        bounds &&
+        bounds.start < lineEnd &&
+        bounds.end > lineStart &&
+        (!best ||
+          bounds.end - bounds.start <
+            (astNodeBounds(best)?.end ?? 0) - (astNodeBounds(best)?.start ?? 0))
+      ) {
+        best = value;
+      }
+    }
+    for (const child of Object.values(value)) {
+      if (Array.isArray(child)) pending.push(...child);
+      else if (isAstNode(child)) pending.push(child);
+    }
+  }
+  return best ? docflowContextIdentifierFromCall(best) : null;
+}
+
+type DocflowInstrumentationReplacement = {
+  start: number;
+  end: number;
+  value: string;
+};
+
+function docflowInputCaptureSource(paths: readonly string[]): string {
+  return `[${paths
+    .map((path) => {
+      const [root, ...properties] = path.split('.');
+      return `{path:${JSON.stringify(path)},readRoot:()=>${root},properties:${JSON.stringify(properties)}}`;
+    })
+    .join(',')}]`;
+}
+
+/** The expression span a bound statement exposes for runtime observation. */
+function docflowObservationTarget(
+  statement: ReturnType<typeof findDocflowBoundStatement>,
+): {
+  start: number;
+  end: number;
+  awaitsResult: boolean;
+  expression: AstNode;
+} | null {
+  if (!statement) return null;
+  let expression: AstNode | null = null;
+  if (statement.type === 'VariableDeclaration') {
+    const declarations = astArray(statement.declarations);
+    if (declarations.length === 1 && isAstNode(declarations[0]!.init)) {
+      expression = declarations[0]!.init;
+    }
+  } else if (
+    statement.type === 'ReturnStatement' &&
+    isAstNode(statement.argument)
+  ) {
+    expression = statement.argument;
+  } else if (
+    statement.type === 'ExpressionStatement' &&
+    isAstNode(statement.expression)
+  ) {
+    expression = statement.expression;
+  } else if (statement.type === 'IfStatement' && isAstNode(statement.test)) {
+    expression = statement.test;
+  }
+  if (!expression) return null;
+  const bounds = astNodeBounds(expression);
+  if (!bounds) return null;
+  return {
+    ...bounds,
+    awaitsResult: expressionContainsTopLevelAwait(expression),
+    expression,
+  };
+}
+
+function docflowObservedExpression(input: {
+  contextName: string;
+  nodeId: string;
+  inputs: readonly string[];
+  outputs: readonly string[];
+  expression: string;
+  awaitsResult: boolean;
+}): string {
+  const callback = input.awaitsResult
+    ? `async()=>(${input.expression})`
+    : `()=>(${input.expression})`;
+  const observation = `${input.contextName}.__deeplineObserveDocflowNode(${JSON.stringify(input.nodeId)},${docflowInputCaptureSource(input.inputs)},${JSON.stringify(input.outputs)},${callback})`;
+  return input.awaitsResult ? `await ${observation}` : observation;
+}
+
+function expressionContainsTopLevelAwait(expression: AstNode): boolean {
+  const pending: AstNode[] = [expression];
+  while (pending.length > 0) {
+    const node = pending.pop()!;
+    if (node.type === 'AwaitExpression') return true;
+    if (
+      node !== expression &&
+      (node.type === 'ArrowFunctionExpression' ||
+        node.type === 'FunctionExpression' ||
+        node.type === 'FunctionDeclaration')
+    ) {
+      continue;
+    }
+    for (const child of Object.values(node)) {
+      if (Array.isArray(child)) pending.push(...child.filter(isAstNode));
+      else if (isAstNode(child)) pending.push(child);
+    }
+  }
+  return false;
+}
+
+/**
+ * Instruments authored nodes at AST expression boundaries. Declarations,
+ * expressions, decisions, and returns are wrapped so the runtime reports the
+ * actual result or failure. Rewrites remain single-line so source mappings and
+ * the executable program's block scope do not change.
+ */
+export function instrumentPlayDocflowRuntimeHits(sourceCode: string): string {
+  // EVERY block's bindings, not just the export being bundled. Two exports'
+  // statements are disjoint, so instrumenting both is correct for whichever one
+  // runs, and the esbuild plugin never has to learn which export it is building.
+  const parsed = parsePlayDocflowFile(sourceCode);
+  if (parsed.blocks.length === 0 || parsed.errors.length > 0) return sourceCode;
+  const ast = parsePlaySourceAst(sourceCode);
+  if (!ast) return sourceCode;
+  const lineStarts = sourceLineStarts(sourceCode);
+  const replacements: DocflowInstrumentationReplacement[] = [];
+
+  // Several annotations may bind one statement — stacked above the dataset
+  // call, or on a chained `.withColumn(...)` line INSIDE it, where the bound
+  // target is an inner expression whose span overlaps (not equals) the
+  // statement's. Overlapping text replacements corrupt the rewritten source,
+  // so every set of overlapping spans collapses into ONE group at the
+  // outermost span, emitted as a single nested observation.
+  type ObservationTarget = {
+    start: number;
+    end: number;
+    awaitsResult: boolean;
+    contextName: string;
+    binding: PlayDocflowBinding;
+  };
+  const targets: ObservationTarget[] = [];
+
+  // Build the parent index once so every binding's symbol resolution reuses it.
+  const parents = buildDocflowParentIndex(ast);
+  for (const binding of parsed.bindings) {
+    // ADR 0016 rule 1: prefer the statement the `out:` symbol names; fall back
+    // to positional resolution. Instrumentation and `plays check` share this
+    // resolver, so the observed statement and the lint agree.
+    const resolution = resolveDocflowBinding(ast, binding, lineStarts, parents);
+    const contextName = findDocflowContextIdentifier({
+      ast,
+      statement: resolution.statement,
+      bindingLine: binding.line,
+      lineStarts,
+      parents,
+    });
+    if (!contextName) continue;
+    const target = docflowObservationTarget(resolution.statement);
+    // A binding naming a `.step('<leg>', …)` leg of a waterfall is NOT
+    // instrumented, and the omission is deliberate rather than a gap.
+    //
+    // A leg's enclosing statement is the whole `steps().step(…).step(…)` builder:
+    // one expression, shared by every leg, which runs once and synchronously to
+    // produce a program object. Wrapping it would report the BUILDER's
+    // construction — instantly settled, with the program as its "output" — under
+    // each leg's node id, and every leg in the cascade would observe the same
+    // span. That is not partial truth, it is a false one, and the canvas would
+    // render a row of completed steps before a single provider had been called.
+    //
+    // Observing the leg itself would mean a new rewrite shape wrapping each
+    // step's RESOLVER, which runs per row inside a dataset — one observation row
+    // per leg per row, against ADR 0016's bound that maps observe at the node
+    // level because per-row truth already lives in sheets. So a waterfall region
+    // is observation-free by construction: the canvas suppresses run state on leg
+    // members and marks only the answering leg, read from the cascade's own
+    // durable result.
+    //
+    // The test is the SHAPE of the observed expression, not how the binding
+    // resolved: every shipped prebuilt builds its cascade in a module-level
+    // helper, which the enclosing-`definePlay` symbol scope deliberately cannot
+    // see, so those legs resolve positionally and would otherwise slip through.
+    const observedRoot = docflowOutputRoot(binding);
+    if (
+      target &&
+      observedRoot &&
+      docflowExpressionWrapsStepBuilder(target.expression, observedRoot)
+    ) {
+      continue;
+    }
+    if (target) {
+      targets.push({ ...target, contextName, binding });
+      continue;
+    }
+
+    const lineStart = lineStarts[binding.line - 1];
+    if (lineStart !== undefined) {
+      const indentationLength =
+        /^\s*/.exec(sourceCode.slice(lineStart))?.[0].length ?? 0;
+      replacements.push({
+        start: lineStart + indentationLength,
+        end: lineStart + indentationLength,
+        value: `await ${contextName}.__deeplineDocflowHit(${JSON.stringify(binding.nodeId)}); `,
+      });
+    }
+  }
+
+  // AST spans nest, so after sorting (container first: earliest start, then
+  // longest) any overlapping target is contained by the open group. The sort
+  // is stable, so same-span bindings keep their annotation order.
+  targets.sort(
+    (left, right) => left.start - right.start || right.end - left.end,
+  );
+  type ObservationGroup = {
+    start: number;
+    end: number;
+    awaitsResult: boolean;
+    contextName: string;
+    bindings: PlayDocflowBinding[];
+  };
+  const groups: ObservationGroup[] = [];
+  for (const target of targets) {
+    const open = groups[groups.length - 1];
+    if (open && target.start < open.end) {
+      open.bindings.push(target.binding);
+    } else {
+      groups.push({
+        start: target.start,
+        end: target.end,
+        awaitsResult: target.awaitsResult,
+        contextName: target.contextName,
+        bindings: [target.binding],
+      });
+    }
+  }
+
+  for (const group of groups) {
+    // Wrap in reverse order so the first-listed annotation observes outermost.
+    let expression = sourceCode.slice(group.start, group.end);
+    for (const binding of [...group.bindings].reverse()) {
+      expression = docflowObservedExpression({
+        contextName: group.contextName,
+        nodeId: binding.nodeId,
+        inputs: binding.inputs ?? [],
+        outputs: binding.outputs ?? [],
+        expression,
+        awaitsResult: group.awaitsResult,
+      });
+    }
+    replacements.push({
+      start: group.start,
+      end: group.end,
+      value: expression,
+    });
+  }
+
+  return replacements
+    .sort((left, right) => right.start - left.start)
+    .reduce(
+      (source, replacement) =>
+        `${source.slice(0, replacement.start)}${replacement.value}${source.slice(replacement.end)}`,
+      sourceCode,
+    );
+}
+
+function docflowRuntimeInstrumentationPlugin(
+  customerSourceFilePaths: readonly string[],
+): Plugin {
+  const customerSourceFiles = new Set(
+    customerSourceFilePaths
+      .filter((path) => extname(path).toLowerCase() !== '.json')
+      .map((path) => resolve(path)),
+  );
+  return {
+    name: 'deepline-docflow-runtime-instrumentation',
+    setup(buildContext) {
+      buildContext.onLoad({ filter: /./ }, (args) => {
+        if (!customerSourceFiles.has(resolve(args.path))) return undefined;
+        return {
+          contents: instrumentPlayDocflowRuntimeHits(
+            readFileSync(args.path, 'utf8'),
+          ),
+          loader: sourceLoaderForPath(args.path),
+          resolveDir: dirname(args.path),
+        };
+      });
+    },
+  };
+}
+
 function buildImportedPlayProxyModule(playName: string): string {
   const serializedName = JSON.stringify(playName);
   return `
@@ -1793,13 +2135,17 @@ type EsbuildBundleOutput = {
 
 async function runEsbuildForCjsNode(
   entryFile: string,
-  _customerSourceFilePaths: string[],
+  customerSourceFilePaths: string[],
   importedPlayDependencies: ImportedPlayDependency[],
   adapter: PlayBundlingAdapter,
   exportName: string,
 ): Promise<EsbuildBundleOutput | string[]> {
   const sdkAliasPlugin = localSdkAliasPlugin(adapter);
   const playProxyPlugin = importedPlayProxyPlugin(importedPlayDependencies);
+  const docflowPlugin = docflowRuntimeInstrumentationPlugin([
+    entryFile,
+    ...customerSourceFilePaths,
+  ]);
   const namedExportShim =
     exportName === 'default'
       ? null
@@ -1827,7 +2173,7 @@ async function runEsbuildForCjsNode(
     sourcesContent: false,
     logLevel: 'silent',
     legalComments: 'none',
-    plugins: [sdkAliasPlugin, playProxyPlugin].filter(
+    plugins: [docflowPlugin, sdkAliasPlugin, playProxyPlugin].filter(
       (plugin): plugin is Plugin => plugin != null,
     ),
   });