deepline 0.1.0 → 0.1.1

package/dist/repo/shared_libs/play-runtime/suspension.ts

@@ -0,0 +1,68 @@
+export type PlayExecutionSuspension =
+  | {
+      kind: 'sleep';
+      boundaryId: string;
+      delayMs: number;
+    }
+  | {
+      kind: 'integration_event';
+      boundaryId: string;
+      eventKey: string;
+      timeoutMs: number;
+    }
+  | {
+      kind: 'integration_event_batch';
+      boundaries: Array<{
+        boundaryId: string;
+        eventKey: string;
+        timeoutMs: number;
+      }>;
+    };
+
+export class PlayExecutionSuspendedError extends Error {
+  readonly suspension: PlayExecutionSuspension;
+
+  constructor(suspension: PlayExecutionSuspension) {
+    super(
+      suspension.kind === 'sleep'
+        ? `Play execution suspended for durable sleep (${suspension.delayMs}ms).`
+        : suspension.kind === 'integration_event_batch'
+          ? `Play execution suspended waiting for ${suspension.boundaries.length} integration events.`
+        : `Play execution suspended waiting for integration event ${JSON.stringify(suspension.eventKey)}.`,
+    );
+    this.name = 'PlayExecutionSuspendedError';
+    this.suspension = suspension;
+  }
+}
+
+export class PlayRowExecutionSuspendedError extends Error {
+  readonly boundary: {
+    boundaryId: string;
+    eventKey: string;
+    timeoutMs: number;
+  };
+
+  constructor(boundary: {
+    boundaryId: string;
+    eventKey: string;
+    timeoutMs: number;
+  }) {
+    super(
+      `Play row execution suspended waiting for integration event ${JSON.stringify(boundary.eventKey)}.`,
+    );
+    this.name = 'PlayRowExecutionSuspendedError';
+    this.boundary = boundary;
+  }
+}
+
+export function isPlayRowExecutionSuspendedError(
+  value: unknown,
+): value is PlayRowExecutionSuspendedError {
+  return value instanceof PlayRowExecutionSuspendedError;
+}
+
+export function isPlayExecutionSuspendedError(
+  value: unknown,
+): value is PlayExecutionSuspendedError {
+  return value instanceof PlayExecutionSuspendedError;
+}

package/dist/repo/shared_libs/play-runtime/tool-batch-executor.ts

@@ -0,0 +1,146 @@
+export type ToolBatchItem = {
+  itemKey: string;
+  payload: Record<string, unknown>;
+  inputHash?: string | null;
+};
+
+export type ToolBatchRequest = {
+  runId: string;
+  orgId: string;
+  toolId: string;
+  operation: string;
+  provider: string;
+  items: ToolBatchItem[];
+  waterfallId?: string | null;
+  stageId?: string | null;
+  fieldName?: string | null;
+  mapName?: string | null;
+  chunkIndex?: number | null;
+  userProvidedRateLimitKey?: string | null;
+  providerBatchSize: number;
+};
+
+export type ToolBatchItemResult = {
+  itemKey: string;
+  result: unknown;
+  cached?: boolean;
+};
+
+export type ToolBatchResult = {
+  runId: string;
+  toolId: string;
+  operation: string;
+  provider: string;
+  batchCount: number;
+  itemCount: number;
+  results: ToolBatchItemResult[];
+};
+
+export type ToolBatchExecutorTransport = {
+  executeProviderBatch(input: {
+    request: ToolBatchRequest;
+    batchIndex: number;
+    idempotencyKeys: string[];
+    rateLimitKey: string;
+    items: ToolBatchItem[];
+  }): Promise<ToolBatchItemResult[]>;
+};
+
+export type ToolBatchExecutor = {
+  executeToolBatch(request: ToolBatchRequest): Promise<ToolBatchResult>;
+};
+
+export function createToolBatchExecutor(
+  transport: ToolBatchExecutorTransport,
+): ToolBatchExecutor {
+  return {
+    async executeToolBatch(request) {
+      const providerBatchSize = Math.max(1, Math.floor(request.providerBatchSize));
+      const batches = chunkToolBatchItems(request.items, providerBatchSize);
+      const results: ToolBatchItemResult[] = [];
+      for (let batchIndex = 0; batchIndex < batches.length; batchIndex += 1) {
+        const items = batches[batchIndex]!;
+        results.push(
+          ...(await transport.executeProviderBatch({
+            request,
+            batchIndex,
+            items,
+            rateLimitKey: buildToolBatchRateLimitKey(request),
+            idempotencyKeys: items.map((item) =>
+              buildToolBatchIdempotencyKey(request, item),
+            ),
+          })),
+        );
+      }
+      return {
+        runId: request.runId,
+        toolId: request.toolId,
+        operation: request.operation,
+        provider: request.provider,
+        batchCount: batches.length,
+        itemCount: request.items.length,
+        results,
+      };
+    },
+  };
+}
+
+export function buildToolBatchIdempotencyKey(
+  request: ToolBatchRequest,
+  item: ToolBatchItem,
+): string {
+  return [
+    request.runId,
+    request.mapName ?? '',
+    request.chunkIndex ?? '',
+    item.itemKey,
+    request.fieldName ?? '',
+    request.waterfallId ?? '',
+    request.stageId ?? '',
+    item.inputHash ?? stableToolBatchHash(item.payload),
+  ].join(':');
+}
+
+export function buildToolBatchRateLimitKey(request: ToolBatchRequest): string {
+  return [
+    request.orgId,
+    request.provider,
+    request.operation,
+    request.userProvidedRateLimitKey ?? '',
+  ].join(':');
+}
+
+function chunkToolBatchItems(
+  items: readonly ToolBatchItem[],
+  size: number,
+): ToolBatchItem[][] {
+  const chunks: ToolBatchItem[][] = [];
+  for (let index = 0; index < items.length; index += size) {
+    chunks.push(items.slice(index, index + size));
+  }
+  return chunks;
+}
+
+function stableToolBatchHash(value: unknown): string {
+  const text = stableStringify(value);
+  let hash = 2166136261;
+  for (let index = 0; index < text.length; index += 1) {
+    hash ^= text.charCodeAt(index);
+    hash = Math.imul(hash, 16777619);
+  }
+  return (hash >>> 0).toString(36);
+}
+
+function stableStringify(value: unknown): string {
+  if (value === null || typeof value !== 'object') {
+    return JSON.stringify(value);
+  }
+  if (Array.isArray(value)) {
+    return `[${value.map(stableStringify).join(',')}]`;
+  }
+  const record = value as Record<string, unknown>;
+  return `{${Object.keys(record)
+    .sort()
+    .map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`)
+    .join(',')}}`;
+}

package/dist/repo/shared_libs/play-runtime/tool-result.ts

@@ -0,0 +1,387 @@
+export type ToolResultExecutionMetadata = {
+  idempotent: true;
+  cached: boolean;
+  source: 'live' | 'checkpoint' | 'cache';
+  cacheKey?: string;
+};
+
+export type ToolResultTargetMetadata = {
+  value: unknown;
+  path: string;
+};
+
+export type ToolResultListMetadata = {
+  path: string;
+  count: number | null;
+  keys: Record<string, string>;
+};
+
+export type ToolResultMetadata = {
+  toolId: string;
+  execution: ToolResultExecutionMetadata;
+  targets: Record<string, ToolResultTargetMetadata>;
+  lists: Record<string, ToolResultListMetadata>;
+};
+
+export type ToolResultMetadataInput = {
+  toolId: string;
+  resultIdentityGetters?: Record<string, readonly string[]>;
+  listExtractorPaths?: readonly string[];
+  listIdentityGetters?: Record<string, readonly string[]>;
+};
+
+export type ToolExecuteResult<TResult = unknown> = {
+  status: string;
+  result: TResult;
+  _metadata: ToolResultMetadata;
+  get<T = unknown>(target: string): T | null;
+  getEmail(): string | null;
+  getPhone(): string | null;
+  getLinkedin(): string | null;
+  list<T = Record<string, unknown>>(name?: string): T[] | null;
+  listPick<const TKeys extends readonly string[]>(
+    keys: TKeys,
+    name?: string,
+  ): Array<Record<TKeys[number], unknown>> | null;
+  listKeys(name?: string): Record<string, string>;
+};
+
+type PathSegment = string | number;
+
+const TARGET_FALLBACK_KEYS: Record<string, readonly RegExp[]> = {
+  email: [/^email$/i, /email/i],
+  phone: [/^phone$/i, /mobile/i, /phone/i, /telephone/i],
+  linkedin: [/^linkedin_url$/i, /^linkedin$/i, /linkedin/i],
+  domain: [/^domain$/i, /company_domain/i, /domain/i],
+  status: [/^email_status$/i, /^status$/i],
+};
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}
+
+function isMeaningfulValue(value: unknown): boolean {
+  if (value == null) return false;
+  if (typeof value === 'string') return value.trim().length > 0;
+  if (Array.isArray(value)) return value.length > 0;
+  if (typeof value === 'object') return Object.keys(value).length > 0;
+  return true;
+}
+
+function parsePath(path: string): PathSegment[] {
+  const segments: PathSegment[] = [];
+  for (const rawPart of path.split('.').filter(Boolean)) {
+    const bracketPattern = /([^\[\]]+)|\[(\d+)\]/g;
+    let matched = false;
+    for (const match of rawPart.matchAll(bracketPattern)) {
+      matched = true;
+      if (match[1]) {
+        segments.push(match[1]);
+      } else if (match[2]) {
+        segments.push(Number(match[2]));
+      }
+    }
+    if (!matched) {
+      segments.push(rawPart);
+    }
+  }
+  return segments;
+}
+
+function pathToString(segments: readonly PathSegment[]): string {
+  return segments
+    .map((segment, index) =>
+      typeof segment === 'number'
+        ? `[${segment}]`
+        : index === 0
+          ? segment
+          : `.${segment}`,
+    )
+    .join('');
+}
+
+function getAtPath(root: unknown, path: string): unknown {
+  let current = root;
+  for (const segment of parsePath(path)) {
+    if (typeof segment === 'number') {
+      if (!Array.isArray(current)) return undefined;
+      current = current[segment];
+      continue;
+    }
+    if (!isRecord(current)) return undefined;
+    current = current[segment];
+  }
+  return current;
+}
+
+function normalizeRows(value: unknown): Record<string, unknown>[] | null {
+  if (!Array.isArray(value)) return null;
+  return value.map((entry) => (isRecord(entry) ? entry : { value: entry }));
+}
+
+function findFirstTargetByPath(
+  result: unknown,
+  paths: readonly string[] | undefined,
+): ToolResultTargetMetadata | null {
+  for (const path of paths ?? []) {
+    const normalizedPath = String(path || '').trim().replace(/^result\./, '');
+    if (!normalizedPath) continue;
+    const value = getAtPath(result, normalizedPath);
+    if (isMeaningfulValue(value)) {
+      return { value, path: normalizedPath };
+    }
+  }
+  return null;
+}
+
+function findFirstTargetByKey(
+  result: unknown,
+  target: string,
+  depth = 0,
+  path: PathSegment[] = [],
+): ToolResultTargetMetadata | null {
+  if (depth > 6) return null;
+  if (Array.isArray(result)) {
+    for (let index = 0; index < result.length; index += 1) {
+      const found = findFirstTargetByKey(result[index], target, depth + 1, [
+        ...path,
+        index,
+      ]);
+      if (found) return found;
+    }
+    return null;
+  }
+  if (!isRecord(result)) return null;
+
+  const patterns = TARGET_FALLBACK_KEYS[target] ?? [
+    new RegExp(`^${target}$`, 'i'),
+  ];
+  for (const [key, value] of Object.entries(result)) {
+    if (patterns.some((pattern) => pattern.test(key)) && isMeaningfulValue(value)) {
+      return { value, path: pathToString([...path, key]) };
+    }
+  }
+  for (const [key, value] of Object.entries(result)) {
+    const found = findFirstTargetByKey(value, target, depth + 1, [...path, key]);
+    if (found) return found;
+  }
+  return null;
+}
+
+function resolveListRows(
+  result: unknown,
+  listExtractorPaths: readonly string[] | undefined,
+): Record<string, { path: string; rows: Record<string, unknown>[] }> {
+  const lists: Record<
+    string,
+    { path: string; rows: Record<string, unknown>[] }
+  > = {};
+  for (const rawPath of listExtractorPaths ?? []) {
+    const path = String(rawPath || '').trim().replace(/^result\./, '');
+    if (!path) continue;
+    const rows = normalizeRows(getAtPath(result, path));
+    if (!rows) continue;
+    const name = path.split('.').filter(Boolean).at(-1)?.replace(/\[\d+\]$/, '');
+    lists[name || path] = { path, rows };
+  }
+  return lists;
+}
+
+function deriveListKeys(input: {
+  listPath: string;
+  rows: readonly Record<string, unknown>[];
+  resultIdentityGetters?: Record<string, readonly string[]>;
+  listIdentityGetters?: Record<string, readonly string[]>;
+}): Record<string, string> {
+  const keys: Record<string, string> = {};
+  for (const [target, paths] of Object.entries(input.listIdentityGetters ?? {})) {
+    const firstPath = paths
+      .map((rawPath) => String(rawPath || '').trim().replace(/^result\./, ''))
+      .find(Boolean);
+    if (firstPath) {
+      keys[target] = firstPath;
+    }
+  }
+  if (Object.keys(keys).length > 0) {
+    return keys;
+  }
+
+  const listPrefix = input.listPath.replace(/\[\d+\]$/, '');
+  for (const [target, paths] of Object.entries(input.resultIdentityGetters ?? {})) {
+    for (const rawPath of paths) {
+      const path = String(rawPath || '').trim().replace(/^result\./, '');
+      if (!path) continue;
+      const directPrefix = `${listPrefix}.`;
+      if (path.startsWith(directPrefix)) {
+        keys[target] = path.slice(directPrefix.length).replace(/^\[\d+\]\.?/, '');
+        break;
+      }
+      const indexedPrefix = `${listPrefix}[0].`;
+      if (path.startsWith(indexedPrefix)) {
+        keys[target] = path.slice(indexedPrefix.length);
+        break;
+      }
+      const dottedIndexPrefix = `${listPrefix}.0.`;
+      if (path.startsWith(dottedIndexPrefix)) {
+        keys[target] = path.slice(dottedIndexPrefix.length);
+        break;
+      }
+    }
+  }
+  if (Object.keys(keys).length === 0 && input.rows[0]) {
+    for (const key of Object.keys(input.rows[0])) {
+      keys[key] = key;
+    }
+  }
+  return keys;
+}
+
+function buildTargets(
+  result: unknown,
+  resultIdentityGetters?: Record<string, readonly string[]>,
+): Record<string, ToolResultTargetMetadata> {
+  const targets: Record<string, ToolResultTargetMetadata> = {};
+  const metadataTargets = new Set(Object.keys(resultIdentityGetters ?? {}));
+  for (const target of metadataTargets) {
+    const fromMetadata = findFirstTargetByPath(
+      result,
+      resultIdentityGetters?.[target],
+    );
+    if (fromMetadata) {
+      targets[target] = fromMetadata;
+    }
+  }
+  if (metadataTargets.size > 0) return targets;
+
+  for (const target of ['email', 'phone', 'linkedin', 'domain', 'status']) {
+    const found = findFirstTargetByKey(result, target);
+    if (found) targets[target] = found;
+  }
+  return targets;
+}
+
+function buildLists(
+  result: unknown,
+  metadata: ToolResultMetadataInput,
+): Record<string, ToolResultListMetadata> {
+  const lists: Record<string, ToolResultListMetadata> = {};
+  const resolved = resolveListRows(result, metadata.listExtractorPaths);
+  for (const [name, list] of Object.entries(resolved)) {
+    lists[name] = {
+      path: list.path,
+      count: list.rows.length,
+      keys: deriveListKeys({
+        listPath: list.path,
+        rows: list.rows,
+        resultIdentityGetters: metadata.resultIdentityGetters,
+        listIdentityGetters: metadata.listIdentityGetters,
+      }),
+    };
+  }
+  return lists;
+}
+
+export function createToolExecuteResult<TResult = unknown>(input: {
+  status: string;
+  result: TResult;
+  metadata: ToolResultMetadataInput;
+  execution: ToolResultExecutionMetadata;
+}): ToolExecuteResult<TResult> {
+  const targets = buildTargets(input.result, input.metadata.resultIdentityGetters);
+  const lists = buildLists(input.result, input.metadata);
+  const wrapper = {
+    status: input.status,
+    result: input.result,
+    _metadata: {
+      toolId: input.metadata.toolId,
+      execution: input.execution,
+      targets,
+      lists,
+    },
+    get<T = unknown>(target: string): T | null {
+      return (this._metadata.targets[target]?.value as T | undefined) ?? null;
+    },
+    getEmail(): string | null {
+      const value = this.get('email');
+      return typeof value === 'string' && value.trim() ? value : null;
+    },
+    getPhone(): string | null {
+      const value = this.get('phone');
+      return typeof value === 'string' && value.trim() ? value : null;
+    },
+    getLinkedin(): string | null {
+      const value = this.get('linkedin');
+      return typeof value === 'string' && value.trim() ? value : null;
+    },
+    list<T = Record<string, unknown>>(name?: string): T[] | null {
+      const entryName = name ?? Object.keys(this._metadata.lists)[0];
+      if (!entryName) return null;
+      const list = this._metadata.lists[entryName];
+      if (!list) return null;
+      return (normalizeRows(getAtPath(this.result, list.path)) as T[] | null) ?? null;
+    },
+    listPick<const TKeys extends readonly string[]>(
+      keys: TKeys,
+      name?: string,
+    ): Array<Record<TKeys[number], unknown>> | null {
+      const entryName = name ?? Object.keys(this._metadata.lists)[0];
+      if (!entryName) return null;
+      const listMetadata = this._metadata.lists[entryName];
+      if (!listMetadata) return null;
+      const rows = normalizeRows(getAtPath(this.result, listMetadata.path));
+      if (!rows) return null;
+      return rows.map((row) => {
+        const picked: Record<string, unknown> = {};
+        for (const key of keys) {
+          const path = listMetadata.keys[key];
+          picked[key] = path ? getAtPath(row, path) ?? null : null;
+        }
+        return picked as Record<TKeys[number], unknown>;
+      });
+    },
+    listKeys(name?: string): Record<string, string> {
+      const entryName = name ?? Object.keys(this._metadata.lists)[0];
+      if (!entryName) return {};
+      return this._metadata.lists[entryName]?.keys ?? {};
+    },
+  } satisfies ToolExecuteResult<TResult>;
+  return wrapper;
+}
+
+export function isToolExecuteResult(value: unknown): value is ToolExecuteResult {
+  return (
+    isRecord(value) &&
+    typeof value.status === 'string' &&
+    '_metadata' in value &&
+    isRecord(value._metadata) &&
+    'result' in value
+  );
+}
+
+export function cloneToolExecuteResultWithExecution<TResult>(
+  value: ToolExecuteResult<TResult>,
+  execution: ToolResultExecutionMetadata,
+): ToolExecuteResult<TResult> {
+  return createToolExecuteResult({
+    status: value.status,
+    result: value.result,
+    metadata: {
+      toolId: value._metadata.toolId,
+      resultIdentityGetters: Object.fromEntries(
+        Object.entries(value._metadata.targets).map(([target, info]) => [
+          target,
+          [info.path],
+        ]),
+      ),
+      listExtractorPaths: Object.values(value._metadata.lists).map(
+        (list) => list.path,
+      ),
+      listIdentityGetters: Object.fromEntries(
+        Object.values(value._metadata.lists)
+          .flatMap((list) => Object.entries(list.keys))
+          .map(([target, path]) => [target, [path]]),
+      ),
+    },
+    execution,
+  });
+}

package/dist/repo/shared_libs/play-runtime/tracing.ts

@@ -0,0 +1,31 @@
+type SpanLike = {
+  setAttribute: (key: string, value: unknown) => void;
+};
+
+type SpanOptions = {
+  tracer?: string;
+  attributes?: Record<string, unknown>;
+};
+
+export function setSpanAttributes(
+  span: SpanLike | null | undefined,
+  attributes: Record<string, unknown>,
+): void {
+  for (const [key, value] of Object.entries(attributes)) {
+    span?.setAttribute?.(key, value);
+  }
+}
+
+export async function withActiveSpan<T>(
+  _name: string,
+  options: SpanOptions,
+  fn: (span: SpanLike) => Promise<T>,
+): Promise<T> {
+  const attributes = new Map<string, unknown>(Object.entries(options.attributes ?? {}));
+  const span: SpanLike = {
+    setAttribute(key, value) {
+      attributes.set(key, value);
+    },
+  };
+  return await fn(span);
+}

package/dist/repo/shared_libs/play-runtime/waterfall-replay.ts

@@ -0,0 +1,75 @@
+import type { BatchResult, PlayCheckpoint, WaterfallRequest } from './ctx-types';
+
+export class WaterfallReplayStore {
+  constructor(private readonly checkpoint: PlayCheckpoint) {}
+
+  checkpointKey(input: { rowId: number; rowKey?: string | null }): string {
+    return input.rowKey?.trim() || String(input.rowId);
+  }
+
+  getResolved(
+    queueKey: string,
+    input: {
+      rowId: number;
+      rowKey?: string | null;
+    },
+  ): { found: boolean; value: unknown } {
+    const resolved = this.checkpoint.resolvedWaterfalls[queueKey];
+    if (!resolved) {
+      return { found: false, value: undefined };
+    }
+    const durableKey = this.checkpointKey(input);
+    if (Object.prototype.hasOwnProperty.call(resolved, durableKey)) {
+      return { found: true, value: resolved[durableKey] };
+    }
+    return { found: false, value: undefined };
+  }
+
+  setResolved(
+    queueKey: string,
+    input: {
+      rowId: number;
+      rowKey?: string | null;
+    },
+    value: unknown,
+  ): void {
+    this.checkpoint.resolvedWaterfalls[queueKey] ??= {};
+    this.checkpoint.resolvedWaterfalls[queueKey]![this.checkpointKey(input)] =
+      value;
+  }
+
+  readProviderBatch(input: {
+    batchKey: string;
+    requests: readonly WaterfallRequest[];
+  }): Array<{ request: WaterfallRequest; result: unknown | null }> | null {
+    const cached = this.checkpoint.completedBatches[input.batchKey];
+    if (!cached) {
+      return null;
+    }
+    const recovered = cached.flatMap((entry) => {
+      const request = input.requests.find((candidate) =>
+        entry.rowKey
+          ? candidate.rowKey === entry.rowKey
+          : candidate.rowId === entry.rowId,
+      );
+      return request ? [{ request, result: entry.result }] : [];
+    });
+    return recovered.length > 0 ? recovered : null;
+  }
+
+  writeProviderBatch(
+    batchKey: string,
+    results: ReadonlyArray<{
+      request: WaterfallRequest;
+      result: unknown | null;
+    }>,
+  ): void {
+    this.checkpoint.completedBatches[batchKey] = results.map(
+      (entry): BatchResult => ({
+        rowId: entry.request.rowId,
+        rowKey: entry.request.rowKey ?? null,
+        result: entry.result,
+      }),
+    );
+  }
+}