deepline 0.0.1 → 0.1.1

package/dist/repo/shared_libs/plays/definition.ts

@@ -0,0 +1,264 @@
+export type PlayPrimitive = string | number | boolean | null;
+
+export type PlayValueTemplate =
+  | PlayPrimitive
+  | { $ref: string }
+  | { [key: string]: PlayValueTemplate }
+  | PlayValueTemplate[];
+
+export type SerializedPlayJavascript = {
+  kind: 'serialized_javascript';
+  source: string;
+  displayName?: string;
+};
+
+export type PlayJavascriptExecutorContext = {
+  row: Record<string, unknown>;
+  steps: Record<string, unknown>;
+  input: Record<string, unknown>;
+  index: number;
+};
+
+export type PlayJavascriptExecutor<TResult = unknown> = (
+  context: PlayJavascriptExecutorContext,
+) => TResult | Promise<TResult>;
+
+export type PlayStructuredStep =
+  | {
+      type: 'tool';
+      alias: string;
+      toolId: string;
+      input: Record<string, PlayValueTemplate>;
+      description?: string;
+    }
+  | {
+      type: 'waterfall';
+      alias: string;
+      tool: string;
+      input: Record<string, PlayValueTemplate>;
+      providers?: string[];
+      description?: string;
+    }
+  | {
+      type: 'run_javascript';
+      alias: string;
+      execute: SerializedPlayJavascript;
+      input?: Record<string, PlayValueTemplate>;
+      description?: string;
+      displayAs?: 'run_javascript';
+    };
+
+export type PlayStructuredDefinition = {
+  version: 1;
+  map: {
+    key: string;
+    steps: PlayStructuredStep[];
+  };
+  result?: {
+    mode?: 'rows';
+  };
+};
+
+export function serializeExecutableJs<TResult = unknown>(
+  execute: PlayJavascriptExecutor<TResult>,
+  displayName?: string,
+): SerializedPlayJavascript {
+  const source = execute.toString().trim();
+  if (!source) {
+    throw new Error('Executable JavaScript function source must be non-empty.');
+  }
+  return {
+    kind: 'serialized_javascript',
+    source,
+    displayName,
+  };
+}
+
+export function runJavascriptStep<TResult = unknown>(input: {
+  alias: string;
+  execute: PlayJavascriptExecutor<TResult>;
+  description?: string;
+  input?: Record<string, PlayValueTemplate>;
+}): Extract<PlayStructuredStep, { type: 'run_javascript' }> {
+  return {
+    type: 'run_javascript',
+    alias: input.alias,
+    execute: serializeExecutableJs(input.execute, input.alias),
+    ...(input.description ? { description: input.description } : {}),
+    ...(input.input ? { input: input.input } : {}),
+    displayAs: 'run_javascript',
+  };
+}
+
+export function definePlayDefinition(
+  definition: PlayStructuredDefinition,
+): PlayStructuredDefinition {
+  return definition;
+}
+
+function formatTemplate(template: PlayValueTemplate): string {
+  if (
+    template === null ||
+    typeof template === 'string' ||
+    typeof template === 'number' ||
+    typeof template === 'boolean'
+  ) {
+    return JSON.stringify(template);
+  }
+  if (Array.isArray(template)) {
+    return `[${template.map((value) => formatTemplate(value)).join(', ')}]`;
+  }
+  if ('$ref' in template && typeof template.$ref === 'string') {
+    return `ref(${JSON.stringify(template.$ref)})`;
+  }
+  return `{ ${Object.entries(template)
+    .map(([key, value]) => `${JSON.stringify(key)}: ${formatTemplate(value)}`)
+    .join(', ')} }`;
+}
+
+export function renderPlayDefinitionSource(
+  definition: PlayStructuredDefinition,
+): string {
+  const lines: string[] = [];
+  lines.push('definePlayDefinition({');
+  lines.push('  version: 1,');
+  lines.push('  map: {');
+  lines.push(`    key: ${JSON.stringify(definition.map.key)},`);
+  lines.push('    steps: [');
+  for (const step of definition.map.steps) {
+    if (step.type === 'tool') {
+      lines.push('      {');
+      lines.push('        type: "tool",');
+      lines.push(`        alias: ${JSON.stringify(step.alias)},`);
+      lines.push(`        toolId: ${JSON.stringify(step.toolId)},`);
+      lines.push(`        input: ${formatTemplate(step.input)},`);
+      if (step.description) {
+        lines.push(`        description: ${JSON.stringify(step.description)},`);
+      }
+      lines.push('      },');
+      continue;
+    }
+    if (step.type === 'waterfall') {
+      lines.push('      {');
+      lines.push('        type: "waterfall",');
+      lines.push(`        alias: ${JSON.stringify(step.alias)},`);
+      lines.push(`        tool: ${JSON.stringify(step.tool)},`);
+      lines.push(`        input: ${formatTemplate(step.input)},`);
+      if (step.providers?.length) {
+        lines.push(`        providers: ${JSON.stringify(step.providers)},`);
+      }
+      if (step.description) {
+        lines.push(`        description: ${JSON.stringify(step.description)},`);
+      }
+      lines.push('      },');
+      continue;
+    }
+    lines.push(`      runJavascriptStep({
+        alias: ${JSON.stringify(step.alias)},
+        execute: ${step.execute.source},
+        ${step.input ? `input: ${formatTemplate(step.input)},` : ''}
+      }),`);
+  }
+  lines.push('    ],');
+  lines.push('  },');
+  lines.push('});');
+  return lines.join('\n');
+}
+
+export function validatePlayStructuredDefinition(
+  definition: unknown,
+): { valid: boolean; errors: string[] } {
+  const errors: string[] = [];
+  if (!definition || typeof definition !== 'object') {
+    return { valid: false, errors: ['Definition must be an object.'] };
+  }
+  const candidate = definition as Partial<PlayStructuredDefinition>;
+  if (candidate.version !== 1) {
+    errors.push('Definition version must be 1.');
+  }
+  if (typeof candidate.map?.key !== 'string' || !candidate.map.key.trim()) {
+    errors.push('Definition map.key must be a non-empty string.');
+  }
+  if (!Array.isArray(candidate.map?.steps) || candidate.map.steps.length === 0) {
+    errors.push('Definition must contain at least one step.');
+  }
+  for (const [index, step] of (candidate.map?.steps ?? []).entries()) {
+    if (!step || typeof step !== 'object') {
+      errors.push(`Step ${index} must be an object.`);
+      continue;
+    }
+    const typedStep = step as PlayStructuredStep;
+    if (!typedStep.alias?.trim()) {
+      errors.push(`Step ${index} requires a non-empty alias.`);
+    }
+    if (typedStep.type === 'tool' && !typedStep.toolId?.trim()) {
+      errors.push(`Tool step ${typedStep.alias || index} requires toolId.`);
+    }
+    if (typedStep.type === 'waterfall' && !typedStep.tool?.trim()) {
+      errors.push(`Waterfall step ${typedStep.alias || index} requires tool.`);
+    }
+    if (typedStep.type === 'run_javascript') {
+      if (!typedStep.execute?.source?.trim()) {
+        errors.push(`run_javascript step ${typedStep.alias || index} requires executable source.`);
+        continue;
+      }
+      try {
+        new Function(`return (${typedStep.execute.source});`);
+      } catch (error) {
+        errors.push(
+          `run_javascript step ${typedStep.alias || index} parse error: ${
+            error instanceof Error ? error.message : String(error)
+          }`,
+        );
+      }
+    }
+  }
+  return { valid: errors.length === 0, errors };
+}
+
+function getPathValue(
+  value: unknown,
+  path: string,
+): unknown {
+  if (!path) return value;
+  const normalized = path.replace(/^\$?/, '');
+  const parts = normalized.split('.').filter(Boolean);
+  let current = value;
+  for (const part of parts) {
+    if (current == null || typeof current !== 'object') return undefined;
+    current = (current as Record<string, unknown>)[part];
+  }
+  return current;
+}
+
+export function resolvePlayValueTemplate(
+  template: PlayValueTemplate,
+  context: PlayJavascriptExecutorContext,
+): unknown {
+  if (
+    template === null ||
+    typeof template === 'string' ||
+    typeof template === 'number' ||
+    typeof template === 'boolean'
+  ) {
+    return template;
+  }
+  if (Array.isArray(template)) {
+    return template.map((item) => resolvePlayValueTemplate(item, context));
+  }
+  if ('$ref' in template && typeof template.$ref === 'string') {
+    const [root, ...rest] = template.$ref.split('.');
+    const path = rest.join('.');
+    if (root === 'row') return getPathValue(context.row, path);
+    if (root === 'steps') return getPathValue(context.steps, path);
+    if (root === 'input') return getPathValue(context.input, path);
+    if (root === 'index') return context.index;
+    return undefined;
+  }
+  return Object.fromEntries(
+    Object.entries(template).map(([key, value]) => [
+      key,
+      resolvePlayValueTemplate(value, context),
+    ]),
+  );
+}

package/dist/repo/shared_libs/plays/file-refs.ts

@@ -0,0 +1,11 @@
+export interface PlayR2FileRef {
+  storageKind: 'r2';
+  storageKey: string;
+  logicalPath: string;
+  fileName: string;
+  contentHash: string;
+  contentType: string;
+  bytes: number;
+}
+
+export type PlayExecutionFileRef = PlayR2FileRef;

package/dist/repo/shared_libs/plays/rate-limit-scheduler.ts

@@ -0,0 +1,206 @@
+const DEFAULT_SUGGESTED_MAX_PARALLELISM = 50;
+const MIN_CONCURRENCY_WAIT_MS = 10;
+
+type BucketState = {
+  windowStartedAt: number;
+  startedInWindow: number;
+  activeCount: number;
+};
+
+export interface PlayQueueHint {
+  bucketId: string;
+  provider: string;
+  operation: string;
+  ruleId: string;
+  requestsPerWindow: number;
+  windowMs: number;
+  maxConcurrency: number | null;
+}
+
+type SchedulerOptions = {
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+  maxSuggestedParallelism?: number;
+  getQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
+};
+
+export async function getSuggestedPlayToolParallelism(
+  toolId: string,
+  fallback: number,
+  maxSuggestedParallelism = DEFAULT_SUGGESTED_MAX_PARALLELISM,
+  getQueueHints: (toolId: string) => Promise<readonly PlayQueueHint[]> = async () => [],
+): Promise<number> {
+  const hints = await getQueueHints(toolId);
+  if (hints.length === 0) {
+    return fallback;
+  }
+
+  const limits = hints.flatMap((hint) =>
+    hint.maxConcurrency != null
+      ? [hint.requestsPerWindow, hint.maxConcurrency]
+      : [hint.requestsPerWindow],
+  );
+
+  return Math.max(1, Math.min(maxSuggestedParallelism, ...limits));
+}
+
+export class PlayRateLimitScheduler {
+  private readonly bucketStates = new Map<string, BucketState>();
+  private lock: Promise<void> = Promise.resolve();
+  private readonly now;
+  private readonly sleep;
+  private readonly maxSuggestedParallelism;
+  private readonly getQueueHints;
+
+  constructor(options: SchedulerOptions = {}) {
+    this.now = options.now ?? (() => Date.now());
+    this.sleep = options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms)));
+    this.maxSuggestedParallelism =
+      options.maxSuggestedParallelism ?? DEFAULT_SUGGESTED_MAX_PARALLELISM;
+    this.getQueueHints = options.getQueueHints ?? (async () => []);
+  }
+
+  async getSuggestedParallelism(toolId: string, fallback: number): Promise<number> {
+    return await getSuggestedPlayToolParallelism(
+      toolId,
+      fallback,
+      this.maxSuggestedParallelism,
+      this.getQueueHints,
+    );
+  }
+
+  async run<T>(toolId: string, task: () => Promise<T>): Promise<T> {
+    const hints = await this.getQueueHints(toolId);
+    if (hints.length === 0) {
+      return await task();
+    }
+
+    await this.acquire(hints);
+    try {
+      return await task();
+    } finally {
+      await this.release(hints);
+    }
+  }
+
+  private async acquire(hints: readonly PlayQueueHint[]): Promise<void> {
+    const orderedHints = [...hints].sort((left, right) =>
+      left.bucketId.localeCompare(right.bucketId),
+    );
+
+    while (true) {
+      const decision = await this.withLock(() => {
+        const now = this.now();
+        let waitMs = 0;
+
+        for (const hint of orderedHints) {
+          const state = this.getBucketState(hint.bucketId, now);
+          this.resetExpiredWindow(state, hint.windowMs, now);
+
+          if (state.startedInWindow >= hint.requestsPerWindow) {
+            waitMs = Math.max(
+              waitMs,
+              state.windowStartedAt + hint.windowMs - now,
+            );
+          }
+
+          if (
+            hint.maxConcurrency != null &&
+            state.activeCount >= hint.maxConcurrency
+          ) {
+            waitMs = Math.max(waitMs, MIN_CONCURRENCY_WAIT_MS);
+          }
+        }
+
+        if (waitMs > 0) {
+          return { acquired: false, waitMs: Math.max(1, waitMs) } as const;
+        }
+
+        for (const hint of orderedHints) {
+          const state = this.getBucketState(hint.bucketId, now);
+          state.startedInWindow += 1;
+          if (hint.maxConcurrency != null) {
+            state.activeCount += 1;
+          }
+        }
+
+        return { acquired: true, waitMs: 0 } as const;
+      });
+
+      if (decision.acquired) {
+        return;
+      }
+
+      await this.sleep(decision.waitMs);
+    }
+  }
+
+  private async release(hints: readonly PlayQueueHint[]): Promise<void> {
+    await this.withLock(() => {
+      for (const hint of hints) {
+        if (hint.maxConcurrency == null) {
+          continue;
+        }
+        const state = this.bucketStates.get(hint.bucketId);
+        if (!state || state.activeCount <= 0) {
+          continue;
+        }
+        state.activeCount -= 1;
+      }
+    });
+  }
+
+  private getBucketState(bucketId: string, now: number): BucketState {
+    const existing = this.bucketStates.get(bucketId);
+    if (existing) {
+      return existing;
+    }
+
+    const created: BucketState = {
+      windowStartedAt: now,
+      startedInWindow: 0,
+      activeCount: 0,
+    };
+    this.bucketStates.set(bucketId, created);
+    return created;
+  }
+
+  private resetExpiredWindow(
+    state: BucketState,
+    windowMs: number,
+    now: number,
+  ): void {
+    if (windowMs <= 0) {
+      state.windowStartedAt = now;
+      state.startedInWindow = 0;
+      return;
+    }
+
+    if (now - state.windowStartedAt < windowMs) {
+      return;
+    }
+
+    const windowsElapsed = Math.floor((now - state.windowStartedAt) / windowMs);
+    state.windowStartedAt += windowsElapsed * windowMs;
+    state.startedInWindow = 0;
+
+    if (now - state.windowStartedAt >= windowMs) {
+      state.windowStartedAt = now;
+    }
+  }
+
+  private async withLock<T>(fn: () => T | Promise<T>): Promise<T> {
+    const previous = this.lock;
+    let releaseLock!: () => void;
+    this.lock = new Promise<void>((resolve) => {
+      releaseLock = resolve;
+    });
+
+    await previous;
+    try {
+      return await fn();
+    } finally {
+      releaseLock();
+    }
+  }
+}

package/dist/repo/shared_libs/plays/resolve-static-pipeline.ts

@@ -0,0 +1,164 @@
+import type {
+  PlaySheetContract,
+  PlayStaticPipeline,
+  PlayStaticSubstep,
+} from './static-pipeline';
+import { compileSheetContract } from './static-pipeline';
+
+export type ResolvablePlayStaticShape = {
+  staticPipeline?: PlayStaticPipeline | null;
+};
+
+export type ResolveStaticPipelineInput = {
+  rootPlayId: string;
+  root: ResolvablePlayStaticShape;
+  resolveReferencedPlay: (playId: string) => Promise<ResolvablePlayStaticShape | null>;
+  maxDepth?: number;
+};
+
+function clonePipeline(pipeline: PlayStaticPipeline): PlayStaticPipeline {
+  return {
+    ...pipeline,
+    inputFields: pipeline.inputFields ? [...pipeline.inputFields] : undefined,
+    fields: [...pipeline.fields],
+    stages: (pipeline.stages ?? []).map((stage) => ({ ...stage })),
+    substeps: pipeline.substeps.map((substep) => ({ ...substep })),
+    sheetContract: cloneSheetContract(pipeline.sheetContract),
+    sheetContractErrors: undefined,
+  };
+}
+
+function cloneSheetContract(contract: PlaySheetContract | null | undefined) {
+  if (!contract) return contract;
+  return {
+    ...contract,
+    columns: contract.columns.map((column) => ({ ...column })),
+  };
+}
+
+function materializePipeline(shape: ResolvablePlayStaticShape | null): PlayStaticPipeline | null {
+  if (!shape) return null;
+  if (shape.staticPipeline) {
+    return clonePipeline(shape.staticPipeline);
+  }
+  return null;
+}
+
+function annotateSubstep(
+  substep: PlayStaticSubstep,
+  input: { callDepth: number; callPath: string[] },
+): PlayStaticSubstep {
+  return {
+    ...substep,
+    callDepth: input.callDepth,
+    callPath: input.callPath,
+  };
+}
+
+async function resolvePipelineSubsteps(input: {
+  steps: PlayStaticSubstep[];
+  resolveReferencedPlay: (playId: string) => Promise<ResolvablePlayStaticShape | null>;
+  stack: string[];
+  maxDepth: number;
+}): Promise<PlayStaticSubstep[]> {
+  const { steps, resolveReferencedPlay, stack, maxDepth } = input;
+  const callDepth = Math.max(0, stack.length - 1);
+
+  return await Promise.all(
+    steps.map(async (substep) => {
+      const annotated = annotateSubstep(substep, {
+        callDepth,
+        callPath: [...stack],
+      });
+
+      if (annotated.type !== 'play_call') {
+        return annotated;
+      }
+
+      if (stack.includes(annotated.playId)) {
+        return {
+          ...annotated,
+          cycleDetected: true,
+          resolutionError: `Static cycle detected: ${[...stack, annotated.playId].join(' -> ')}`,
+          pipeline: null,
+        } satisfies PlayStaticSubstep;
+      }
+
+      if (stack.length >= maxDepth) {
+        return {
+          ...annotated,
+          resolutionError: `Static nesting limit reached at ${annotated.playId}`,
+          pipeline: null,
+        } satisfies PlayStaticSubstep;
+      }
+
+      const childShape = await resolveReferencedPlay(annotated.playId);
+      const childPipeline = materializePipeline(childShape);
+      if (!childPipeline) {
+        return {
+          ...annotated,
+          resolutionError: `Unable to statically resolve play "${annotated.playId}"`,
+          pipeline: null,
+        } satisfies PlayStaticSubstep;
+      }
+
+      const resolvedChildPipeline = {
+        ...childPipeline,
+        stages: await resolvePipelineSubsteps({
+          steps: childPipeline.stages ?? [],
+          resolveReferencedPlay,
+          stack: [...stack, annotated.playId],
+          maxDepth,
+        }),
+        substeps: await resolvePipelineSubsteps({
+          steps: childPipeline.substeps,
+          resolveReferencedPlay,
+          stack: [...stack, annotated.playId],
+          maxDepth,
+        }),
+      };
+      const childContract = compileSheetContract(resolvedChildPipeline);
+
+      return {
+        ...annotated,
+        pipeline: {
+          ...resolvedChildPipeline,
+          sheetContract: childContract.contract,
+          sheetContractErrors: childContract.errors,
+        },
+      } satisfies PlayStaticSubstep;
+    }),
+  );
+}
+
+export async function resolveStaticPipelineTree(
+  input: ResolveStaticPipelineInput,
+): Promise<PlayStaticPipeline | null> {
+  const maxDepth = input.maxDepth ?? 6;
+  const rootPipeline = materializePipeline(input.root);
+  if (!rootPipeline) {
+    return null;
+  }
+
+  const resolvedPipeline = {
+    ...rootPipeline,
+    stages: await resolvePipelineSubsteps({
+      steps: rootPipeline.stages ?? [],
+      resolveReferencedPlay: input.resolveReferencedPlay,
+      stack: [input.rootPlayId],
+      maxDepth,
+    }),
+    substeps: await resolvePipelineSubsteps({
+      steps: rootPipeline.substeps,
+      resolveReferencedPlay: input.resolveReferencedPlay,
+      stack: [input.rootPlayId],
+      maxDepth,
+    }),
+  };
+  const contract = compileSheetContract(resolvedPipeline);
+  return {
+    ...resolvedPipeline,
+    sheetContract: contract.contract,
+    sheetContractErrors: contract.errors,
+  };
+}