deepline 0.1.119 → 0.1.121

package/dist/bundling-sources/shared_libs/plays/runtime-validation.ts

@@ -0,0 +1,417 @@
+import * as acorn from 'acorn';
+import { fullAncestor as walkFullAncestor } from 'acorn-walk';
+import type { PlayBundleArtifact } from './artifact-types';
+import type { PlayStructuredDefinition } from './definition';
+import { validatePlayStructuredDefinition } from './definition';
+import {
+  MAP_KEY_NAMESPACE_MAX_LENGTH,
+  normalizeTableNamespace,
+} from './row-identity';
+import { DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE } from '../play-runtime/runtime-constraints';
+
+export async function validatePlay(
+  code: string | null | undefined,
+  definition?: PlayStructuredDefinition | null,
+  codeFormat: 'function' | 'cjs_module' | 'esm_module' = 'function',
+  validationSource?: string | null,
+  artifact?: PlayBundleArtifact | null,
+): Promise<{ valid: boolean; errors: string[] }> {
+  const errors: string[] = [];
+
+  if (definition) {
+    return validatePlayStructuredDefinition(definition);
+  }
+
+  const sourceForValidation = validationSource ?? code ?? '';
+
+  if (!sourceForValidation.trim()) {
+    return { valid: false, errors: ['Play code is required.'] };
+  }
+
+  try {
+    if (codeFormat === 'cjs_module' && code?.trim()) {
+      new Function('module', 'exports', 'require', code);
+    } else if (codeFormat === 'esm_module' && code?.trim()) {
+      // esm_module bundles target Cloudflare Workers; their top-level uses
+      // import/export which `new Function` rejects. Skip the in-host parse
+      // check — the bundler already typechecked + esbuild parsed the source.
+      // The play Worker runtime catches any actual runtime errors.
+    } else if (codeFormat !== 'cjs_module') {
+      new Function('ctx', 'input', `return (${code})(ctx, input)`);
+    }
+  } catch (e) {
+    errors.push(`Parse error: ${e instanceof Error ? e.message : String(e)}`);
+  }
+
+  errors.push(...validatePlayMapStructure(sourceForValidation));
+  errors.push(...validateRuntimeSyntax(sourceForValidation));
+
+  if (
+    artifact &&
+    artifact.codeFormat !== 'cjs_module' &&
+    artifact.codeFormat !== 'esm_module'
+  ) {
+    errors.push(
+      'Play artifact codeFormat must be "cjs_module" or "esm_module".',
+    );
+  }
+
+  return { valid: errors.length === 0, errors };
+}
+
+function validatePlayMapStructure(code: string): string[] {
+  const errors: string[] = [];
+  const calledPlayNames = new Set<string>();
+  let ast: acorn.Node;
+
+  try {
+    ast = parsePlayAst(code);
+  } catch {
+    return errors;
+  }
+
+  walkFullAncestor(ast, (node, _state, ancestors) => {
+    if (
+      node.type === 'CallExpression' &&
+      usesDisallowedRunJavascriptTool(node as acorn.CallExpression) &&
+      hasMapResolverAncestor(ancestors)
+    ) {
+      errors.push(DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE);
+    }
+
+    if (isDeprecatedCtxMapCall(node)) {
+      errors.push(
+        'ctx.map(...) has been replaced by ctx.dataset(...). Use ctx.dataset("leads", rows).withColumn("company", row => row.domain).run({ key: "lead_id" }).',
+      );
+      return;
+    }
+
+    if (!isCtxDatasetCall(node)) {
+      const callExpression =
+        (node as acorn.Node).type === 'CallExpression'
+          ? (node as unknown as acorn.CallExpression)
+          : null;
+      if (
+        callExpression &&
+        callExpression.callee.type === 'MemberExpression' &&
+        callExpression.callee.property.type === 'Identifier' &&
+        callExpression.callee.property.name === 'runPlay'
+      ) {
+        const firstArgument = callExpression.arguments[0];
+        if (
+          firstArgument?.type === 'Literal' &&
+          typeof firstArgument.value === 'string'
+        ) {
+          calledPlayNames.add(firstArgument.value);
+        }
+      }
+      if (
+        callExpression &&
+        callExpression.callee.type === 'MemberExpression' &&
+        callExpression.callee.property.type === 'Identifier' &&
+        callExpression.callee.property.name === 'waterfall'
+      ) {
+        const firstArgument = callExpression.arguments[0];
+        if (firstArgument?.type === 'ObjectExpression') {
+          const minResultsProperty = firstArgument.properties.find(
+            (prop) =>
+              prop.type === 'Property' &&
+              prop.key.type === 'Identifier' &&
+              prop.key.name === 'minResults',
+          );
+          if (!minResultsProperty) {
+            errors.push(
+              'Inline ctx.waterfall({...}) calls must declare minResults.',
+            );
+          }
+        }
+      }
+      return;
+    }
+
+    if (hasMapResolverAncestor(ancestors)) {
+      errors.push(
+        'Nested ctx.dataset() is not supported. Flatten work into a single dataset definition.',
+      );
+      return;
+    }
+
+    extractValidatedMapTableNamespace(node, errors);
+  });
+
+  const playNameMatch = code.match(
+    /define(?:Play|Workflow)\s*\(\s*['"`]([^'"`]+)['"`]/,
+  );
+  const definedPlayName = playNameMatch?.[1]?.trim();
+  if (definedPlayName && calledPlayNames.has(definedPlayName)) {
+    errors.push(
+      `Recursive play graph detected: ${definedPlayName} -> ${definedPlayName}. Use a different child play or refactor the shared logic.`,
+    );
+  }
+
+  return [...new Set(errors)];
+}
+
+function validateRuntimeSyntax(code: string): string[] {
+  const errors: string[] = [];
+  let ast: acorn.Node;
+
+  try {
+    ast = parsePlayAst(code);
+  } catch {
+    return errors;
+  }
+
+  walkFullAncestor(ast, (node) => {
+    if (node.type === 'ImportExpression') {
+      errors.push(
+        'Dynamic import() is not allowed in plays. Use static imports instead.',
+      );
+      return;
+    }
+
+    if (node.type !== 'CallExpression') {
+      return;
+    }
+
+    const callNode = node as acorn.CallExpression;
+    if (
+      callNode.callee.type !== 'Identifier' ||
+      callNode.callee.name !== 'require'
+    ) {
+      return;
+    }
+
+    const firstArgument = callNode.arguments[0];
+    const isLiteralString =
+      firstArgument?.type === 'Literal' &&
+      typeof firstArgument.value === 'string';
+
+    if (!isLiteralString) {
+      errors.push(
+        'Dynamic require() is not allowed in plays. Use require("literal") only.',
+      );
+    }
+  });
+
+  return [...new Set(errors)];
+}
+
+function parsePlayAst(code: string): acorn.Node {
+  try {
+    return acorn.parse(code, {
+      ecmaVersion: 'latest',
+      sourceType: 'module',
+      allowAwaitOutsideFunction: true,
+    }) as acorn.Node;
+  } catch {
+    return acorn.parse(`const __play = ${code};`, {
+      ecmaVersion: 'latest',
+      sourceType: 'module',
+    }) as acorn.Node;
+  }
+}
+
+function usesDisallowedRunJavascriptTool(node: acorn.CallExpression): boolean {
+  const callee = node.callee;
+  if (callee.type !== 'MemberExpression') {
+    return false;
+  }
+
+  if (
+    callee.property.type !== 'Identifier' ||
+    callee.property.name !== 'tool'
+  ) {
+    return false;
+  }
+
+  const firstArgument = node.arguments[0];
+  return (
+    firstArgument?.type === 'Literal' &&
+    firstArgument.value === 'run_javascript'
+  );
+}
+
+function extractValidatedMapTableNamespace(
+  node: acorn.CallExpression,
+  errors: string[],
+): string | null {
+  const keyArgument = node.arguments[0];
+  const rowsArgument = node.arguments[1];
+  if (!keyArgument) {
+    errors.push(
+      'ctx.dataset() requires a string literal dataset key as the first argument, e.g. ctx.dataset("leads", rows).withColumn("company", row => row.domain).run({ key: "lead_id" }).',
+    );
+    return null;
+  }
+
+  if (!rowsArgument) {
+    errors.push(
+      'ctx.dataset() requires rows as the second argument, e.g. ctx.dataset("leads", rows).withColumn("company", row => row.domain).run({ key: "lead_id" }).',
+    );
+    return null;
+  }
+
+  if (keyArgument.type !== 'Literal' || typeof keyArgument.value !== 'string') {
+    errors.push(
+      'ctx.dataset() requires a string literal key as the first argument so Deepline can precompute idempotency.',
+    );
+    return null;
+  }
+
+  if (!keyArgument.value.trim()) {
+    errors.push(
+      'ctx.dataset() requires a non-empty string key as the first argument.',
+    );
+    return null;
+  }
+
+  try {
+    normalizeTableNamespace(keyArgument.value);
+  } catch (error) {
+    errors.push(
+      error instanceof Error
+        ? `${error.message} Example: ctx.dataset("leads", rows).withColumn("company", row => row.domain).run({ key: "lead_id", description: "..." }).`
+        : `ctx.dataset() key must normalize to <= ${MAP_KEY_NAMESPACE_MAX_LENGTH} characters.`,
+    );
+    return null;
+  }
+
+  if (rowsArgument.type === 'ObjectExpression') {
+    errors.push(
+      'ctx.dataset() key must not be an object. Use ctx.dataset("leads", rows).withColumn(...).run({ key: "lead_id" }).',
+    );
+    return null;
+  }
+  if (
+    rowsArgument.type === 'FunctionExpression' ||
+    isFunctionNode(rowsArgument)
+  ) {
+    errors.push('ctx.dataset() requires rows as the second argument.');
+    return null;
+  }
+
+  const optionsArgument = node.arguments[2];
+  if (optionsArgument) {
+    errors.push(
+      'ctx.dataset() accepts only a dataset key and rows. Add columns with .withColumn(...) and pass row identity options to .run({ key: "lead_id" }).',
+    );
+    return null;
+  }
+
+  return keyArgument.value.trim();
+}
+
+function isCtxDatasetCall(node: acorn.Node): node is acorn.CallExpression {
+  if (node.type !== 'CallExpression') {
+    return false;
+  }
+  const callee = (node as acorn.CallExpression).callee;
+  if (callee.type !== 'MemberExpression') {
+    return false;
+  }
+  if (
+    callee.property.type !== 'Identifier' ||
+    callee.property.name !== 'dataset'
+  ) {
+    return false;
+  }
+  if (callee.object.type !== 'Identifier') {
+    return false;
+  }
+  return callee.object.name === 'ctx' || callee.object.name.endsWith('Ctx');
+}
+
+function isDeprecatedCtxMapCall(node: acorn.Node): node is acorn.CallExpression {
+  if (node.type !== 'CallExpression') return false;
+  const callee = (node as acorn.CallExpression).callee;
+  return (
+    callee.type === 'MemberExpression' &&
+    callee.property.type === 'Identifier' &&
+    callee.property.name === 'map' &&
+    callee.object.type === 'Identifier' &&
+    (callee.object.name === 'ctx' || callee.object.name.endsWith('Ctx'))
+  );
+}
+
+function isFunctionNode(
+  node: acorn.Node | null | undefined,
+): node is
+  | acorn.FunctionDeclaration
+  | acorn.FunctionExpression
+  | acorn.ArrowFunctionExpression {
+  return Boolean(
+    node &&
+    (node.type === 'FunctionDeclaration' ||
+      node.type === 'FunctionExpression' ||
+      node.type === 'ArrowFunctionExpression'),
+  );
+}
+
+function hasMapResolverAncestor(ancestors: acorn.Node[]): boolean {
+  for (let index = 0; index < ancestors.length; index += 1) {
+    const node = ancestors[index];
+    if (!isFunctionNode(node)) {
+      continue;
+    }
+
+    const parent = index >= 1 ? ancestors[index - 1] : null;
+    const grandparent = index >= 2 ? ancestors[index - 2] : null;
+    const greatGrandparent = index >= 3 ? ancestors[index - 3] : null;
+
+    if (
+      parent?.type === 'CallExpression' &&
+      isCtxDatasetCall(parent) &&
+      parent.arguments[2] === node
+    ) {
+      return true;
+    }
+
+    if (
+      parent?.type === 'CallExpression' &&
+      isMapBuilderStepCall(parent) &&
+      parent.arguments[1] === node
+    ) {
+      return true;
+    }
+
+    if (
+      parent?.type === 'Property' &&
+      (parent as acorn.Property).value === node &&
+      grandparent?.type === 'ObjectExpression' &&
+      greatGrandparent?.type === 'CallExpression' &&
+      isCtxDatasetCall(greatGrandparent) &&
+      greatGrandparent.arguments[2] === grandparent
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+function isMapBuilderStepCall(node: acorn.Node): node is acorn.CallExpression {
+  if (node.type !== 'CallExpression') return false;
+  const callee = (node as acorn.CallExpression).callee;
+  if (callee.type !== 'MemberExpression') return false;
+  if (
+    callee.property.type !== 'Identifier' ||
+    callee.property.name !== 'withColumn'
+  ) {
+    return false;
+  }
+  let current: acorn.Node = callee.object as acorn.Node;
+  while (current.type === 'CallExpression') {
+    if (isCtxDatasetCall(current)) return true;
+    const nestedCallee = (current as acorn.CallExpression).callee;
+    if (
+      nestedCallee.type !== 'MemberExpression' ||
+      nestedCallee.property.type !== 'Identifier' ||
+      nestedCallee.property.name !== 'withColumn'
+    ) {
+      return false;
+    }
+    current = nestedCallee.object as acorn.Node;
+  }
+  return false;
+}

package/dist/bundling-sources/shared_libs/plays/tool-codegen.ts

@@ -0,0 +1,142 @@
+const KNOWN_GETTER_NAMES = new Set([
+  'company_domain',
+  'company_linkedin_url',
+  'company_name',
+  'domain',
+  'email',
+  'email_status',
+  'first_name',
+  'full_name',
+  'job_change',
+  'job_change_status',
+  'job_changed',
+  'last_name',
+  'linkedin',
+  'personal_email',
+  'phone',
+  'phone_status',
+  'status',
+  'title',
+]);
+
+export function renderToolCodegenString(value: string): string {
+  return JSON.stringify(value);
+}
+
+export function renderToolRowPathExpression(
+  path: string,
+  rowName = 'row',
+): string {
+  const parts = String(path || '')
+    .replace(/\[(\d+)\]/g, '.$1')
+    .split('.')
+    .map((part) => part.trim())
+    .filter(Boolean);
+  if (parts.length === 0) return rowName;
+  return parts.reduce((expr, part, index) => {
+    if (index === 0) return `${rowName}[${renderToolCodegenString(part)}]`;
+    return `(${expr} as Record<string, unknown> | null | undefined)?.[${renderToolCodegenString(part)}]`;
+  }, rowName);
+}
+
+export function renderToolPayloadExpression(
+  value: unknown,
+  rowName = 'row',
+): string {
+  if (Array.isArray(value)) {
+    return `[${value.map((entry) => renderToolPayloadExpression(entry, rowName)).join(', ')}]`;
+  }
+  if (value && typeof value === 'object') {
+    const entries = Object.entries(value as Record<string, unknown>).sort(
+      ([left], [right]) => left.localeCompare(right),
+    );
+    return `{ ${entries
+      .map(
+        ([key, entry]) =>
+          `${renderToolCodegenString(key)}: ${renderToolPayloadExpression(entry, rowName)}`,
+      )
+      .join(', ')} }`;
+  }
+  if (typeof value !== 'string') return JSON.stringify(value);
+
+  const exact = value.match(/^\{\{\s*([^{}]+?)\s*\}\}$/);
+  if (exact) return renderToolRowPathExpression(exact[1] ?? '', rowName);
+
+  const pieces: string[] = [];
+  let cursor = 0;
+  for (const match of value.matchAll(/\{\{\s*([^{}]+?)\s*\}\}/g)) {
+    const index = match.index ?? 0;
+    if (index > cursor) {
+      pieces.push(renderToolCodegenString(value.slice(cursor, index)));
+    }
+    pieces.push(
+      `String(${renderToolRowPathExpression(match[1] ?? '', rowName)} ?? '')`,
+    );
+    cursor = index + match[0].length;
+  }
+  if (pieces.length === 0) return renderToolCodegenString(value);
+  if (cursor < value.length) {
+    pieces.push(renderToolCodegenString(value.slice(cursor)));
+  }
+  return pieces.join(' + ');
+}
+
+export function renderExtractedValueGetterExpression(
+  toolResultVar: string,
+  targetKey: string,
+): string {
+  return /^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(targetKey)
+    ? `${toolResultVar}.extractedValues.${targetKey}?.get()`
+    : `${toolResultVar}.extractedValues[${renderToolCodegenString(targetKey)}]?.get()`;
+}
+
+export function getterFromLegacyPath(path: string): string | null {
+  const last = String(path || '')
+    .replace(/\[(\d+)\]/g, '.$1')
+    .split('.')
+    .map((part) => part.trim())
+    .filter(Boolean)
+    .at(-1);
+  if (!last) return null;
+  const normalized = last.replace(/[^A-Za-z0-9_]/g, '_');
+  return KNOWN_GETTER_NAMES.has(normalized) ? normalized : null;
+}
+
+export function getterFromLegacyExtractJs(
+  extractJs: string | undefined,
+  fallbackAlias?: string,
+): string | null {
+  const source = extractJs?.trim();
+  if (!source) {
+    const aliasGetter = fallbackAlias?.replace(/[^A-Za-z0-9_]/g, '_');
+    return aliasGetter && KNOWN_GETTER_NAMES.has(aliasGetter)
+      ? aliasGetter
+      : null;
+  }
+
+  const directExtract = source.match(
+    /\bextract\(\s*["'][^"']+["']\s*,\s*output_data\s*,\s*["']([^"']+)["']\s*\)/,
+  );
+  if (directExtract?.[1]) {
+    return getterFromLegacyPath(
+      directExtract[1] === 'linkedin_url' ? 'linkedin' : directExtract[1],
+    );
+  }
+
+  const pick = source.match(
+    /\b(?:pick|extract|target)\(\s*(\[[\s\S]*?\]|["'][^"']+["'])\s*\)/,
+  );
+  if (!pick?.[1]) return null;
+  try {
+    const parsed = JSON.parse(pick[1].replace(/'/g, '"')) as unknown;
+    const paths = Array.isArray(parsed) ? parsed : [parsed];
+    for (const path of paths) {
+      if (typeof path !== 'string') continue;
+      const getter = getterFromLegacyPath(path);
+      if (getter) return getter;
+    }
+  } catch {
+    return null;
+  }
+  return null;
+}