deepline 0.1.0 → 0.1.2

package/dist/repo/shared_libs/plays/compiler-manifest.ts

@@ -0,0 +1,186 @@
+import type { PlayArtifactKind } from '../play-runtime/backend';
+
+export interface PlayStaticPipelineSnapshot {
+  tableNamespace?: string;
+  inputFields?: string[];
+  csvArg?: string;
+  hasInlineData?: boolean;
+  csvDescription?: string;
+  mapDescription?: string;
+  fields: string[];
+  stages?: PlayStaticSubstepSnapshot[];
+  substeps: PlayStaticSubstepSnapshot[];
+  sheetContract?: PlaySheetContractSnapshot | null;
+  sheetContractErrors?: string[];
+}
+
+export type PlaySheetColumnSourceSnapshot =
+  | 'input'
+  | 'mapField'
+  | 'waterfallStep'
+  | 'childPlayColumn';
+
+export interface PlaySheetColumnContractSnapshot {
+  id: string;
+  sqlName: string;
+  source: PlaySheetColumnSourceSnapshot;
+  field?: string;
+  parentField?: string;
+  playId?: string;
+  waterfallId?: string;
+  outputField?: string;
+  outputSqlName?: string;
+  stepId?: string;
+  toolId?: string;
+}
+
+export interface PlaySheetContractSnapshot {
+  tableNamespace: string;
+  columns: PlaySheetColumnContractSnapshot[];
+}
+
+export interface PlayStaticSourceRangeSnapshot {
+  sourcePath?: string;
+  startLine: number;
+  endLine: number;
+  startColumn: number;
+  endColumn: number;
+}
+
+type PlayStaticSubstepMetadataSnapshot = {
+  conditional?: boolean;
+};
+
+export type PlayStaticSubstepSnapshot = PlayStaticSubstepMetadataSnapshot &
+  (
+    | {
+        type: 'csv';
+        field: string;
+        path?: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRangeSnapshot;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'map';
+        field: string;
+        name?: string;
+        tableNamespace?: string;
+        inputFields?: string[];
+        outputFields?: string[];
+        waterfallIds?: string[];
+        sheetContract?: PlaySheetContractSnapshot | null;
+        description?: string;
+        sourceRange?: PlayStaticSourceRangeSnapshot;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'tool';
+        toolId: string;
+        field: string;
+        description?: string;
+        inLoop?: boolean;
+        isEventWait?: boolean;
+        sourceRange?: PlayStaticSourceRangeSnapshot;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'waterfall';
+        tool?: string;
+        field: string;
+        inLoop?: boolean;
+        id?: string;
+        output?: string;
+        minResults?: number;
+        sourceText?: string;
+        steps?: Array<{
+          id: string;
+          kind?: 'tool' | 'code';
+          toolId?: string;
+          paramsSource?: string;
+        }>;
+        description?: string;
+        sourceRange?: PlayStaticSourceRangeSnapshot;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'step_suite';
+        field: string;
+        steps: PlayStaticSubstepSnapshot[];
+        returnSource?: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRangeSnapshot;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'play_call';
+        playId: string;
+        field: string;
+        inLoop?: boolean;
+        pipeline?: PlayStaticPipelineSnapshot | null;
+        cycleDetected?: boolean;
+        resolutionError?: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRangeSnapshot;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'run_javascript';
+        alias: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRangeSnapshot;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'code';
+        field: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRangeSnapshot;
+        callDepth?: number;
+        callPath?: string[];
+      }
+  );
+
+export const PLAY_COMPILER_MANIFEST_VERSION = 1;
+
+export type PlayCompilerDependencyManifest = {
+  playName: string;
+  sourceHash: string;
+  graphHash: string;
+  artifactHash: string;
+  staticPipeline: PlayStaticPipelineSnapshot;
+};
+
+export type PlayCompilerManifest = {
+  compilerVersion: number;
+  playName: string;
+  sourceHash: string;
+  graphHash: string;
+  artifactHash: string;
+  artifactKind?: PlayArtifactKind;
+  staticPipeline: PlayStaticPipelineSnapshot;
+  importedPlayDependencies: PlayCompilerDependencyManifest[];
+};
+
+export type PlayRuntimeManifest = {
+  playName?: string;
+  graphHash: string;
+  artifactHash: string;
+  artifactStorageKey: string;
+  staticPipelineHash: string;
+  staticPipeline: PlayStaticPipelineSnapshot;
+  compiledAt: number;
+  compilerVersion: string;
+  sourceCode?: string | null;
+  dynamicWorkerCode?: string | null;
+  maxCreditsPerRun?: number | null;
+};
+
+export type PlayRuntimeManifestMap = Record<string, PlayRuntimeManifest>;

package/dist/repo/shared_libs/plays/contracts.ts

@@ -0,0 +1,51 @@
+export type PlayContractSource = 'ad_hoc' | 'draft' | 'published';
+
+export type PlayRuntimeFeature =
+  | 'artifact_storage'
+  | 'checkpoint_resume'
+  | 'durable_sleep'
+  | 'packaged_files';
+
+export const PLAY_PUBLIC_API_VERSION = 1;
+export const PLAY_ARTIFACT_VERSION = 1;
+export const PLAY_MIN_RUNNER_VERSION = 1;
+export const PLAY_RUNTIME_FEATURES: PlayRuntimeFeature[] = [
+  'artifact_storage',
+  'checkpoint_resume',
+  'durable_sleep',
+  'packaged_files',
+];
+
+export type PlayContractCompatibilitySnapshot = {
+  apiVersion: number;
+  artifactVersion: number;
+  minRunnerVersion: number;
+  runtimeFeatures: PlayRuntimeFeature[];
+  runtimeBackend?: string | null;
+};
+
+export function buildPlayContractCompatibility(input?: {
+  runtimeBackend?: string | null;
+}): PlayContractCompatibilitySnapshot {
+  return {
+    apiVersion: PLAY_PUBLIC_API_VERSION,
+    artifactVersion: PLAY_ARTIFACT_VERSION,
+    minRunnerVersion: PLAY_MIN_RUNNER_VERSION,
+    runtimeFeatures: [...PLAY_RUNTIME_FEATURES],
+    runtimeBackend: input?.runtimeBackend ?? null,
+  };
+}
+
+export type PlayRunContractSnapshot = {
+  source: PlayContractSource;
+  revisionVersion?: number | null;
+  staticPipeline?: unknown;
+  billingLimit?: {
+    maxCreditsPerRun?: number | null;
+  } | null;
+  structuredDefinition?: unknown;
+  artifactMetadata?: Record<string, unknown> | null;
+  codeFormat?: 'function' | 'cjs_module' | 'esm_module' | null;
+  sourceCode?: string | null;
+  compatibility?: PlayContractCompatibilitySnapshot | null;
+};

package/dist/repo/shared_libs/plays/dataset.ts

@@ -0,0 +1,308 @@
+import type { PlayExecutionFileRef } from './file-refs';
+
+const PLAY_DATASET_BRAND = Symbol.for('deepline.play.dataset');
+const NODE_INSPECT_CUSTOM = Symbol.for('nodejs.util.inspect.custom');
+const DEFAULT_MATERIALIZE_LIMIT = 10_000;
+
+export type PlayDatasetKind = 'csv' | 'map';
+
+export type PlayDatasetBacking =
+  | {
+      storage: 'neon_sheet';
+      sheet: {
+        playName: string;
+        tableNamespace: string;
+      };
+    }
+  | {
+      storage: 'r2_file';
+      file: PlayExecutionFileRef;
+    };
+
+export interface SerializedPlayDataset<T> {
+  kind: 'dataset';
+  datasetKind: PlayDatasetKind;
+  datasetId: string;
+  count: number;
+  backing?: PlayDatasetBacking;
+  sourceLabel?: string | null;
+  tableNamespace?: string | null;
+  columns?: string[];
+  preview: T[];
+}
+
+export type PlayDatasetInput<T> =
+  | ReadonlyArray<T>
+  | Iterable<T>
+  | AsyncIterable<T>
+  | PlayDataset<T>;
+
+export interface PlayDataset<T> extends AsyncIterable<T> {
+  readonly [PLAY_DATASET_BRAND]: true;
+  readonly datasetKind: PlayDatasetKind;
+  readonly datasetId: string;
+  readonly backing?: PlayDatasetBacking;
+  readonly sourceLabel?: string | null;
+  readonly tableNamespace?: string | null;
+  count(): Promise<number>;
+  peek(limit?: number): Promise<T[]>;
+  /**
+   * Explicit escape hatch for bounded result sets.
+   * Large datasets should flow by handle through Neon-backed storage, not
+   * through worker memory as giant arrays.
+   */
+  materialize(limit?: number): Promise<T[]>;
+  toJSON(): {
+    kind: 'dataset';
+    datasetKind: PlayDatasetKind;
+    datasetId: string;
+    count: number;
+    backing?: PlayDatasetBacking;
+    sourceLabel?: string | null;
+    tableNamespace?: string | null;
+    columns?: string[];
+    preview: T[];
+  };
+}
+
+type PlayDatasetResolvers<T> = {
+  count: () => Promise<number>;
+  peek: (limit: number) => Promise<T[]>;
+  materialize: (limit?: number) => Promise<T[]>;
+  iterate: () => AsyncIterable<T>;
+};
+
+function resolveMaterializeLimitCap(): number {
+  const raw = process.env.DEEPLINE_PLAY_DATASET_MATERIALIZE_LIMIT;
+  const parsed = raw ? Number(raw) : NaN;
+  if (Number.isFinite(parsed) && parsed > 0) {
+    return Math.floor(parsed);
+  }
+  return DEFAULT_MATERIALIZE_LIMIT;
+}
+
+function inferPreviewColumns<T>(rows: readonly T[]): string[] | undefined {
+  const columns = new Set<string>();
+  for (const row of rows) {
+    if (!row || typeof row !== 'object' || Array.isArray(row)) {
+      continue;
+    }
+    for (const key of Object.keys(row as Record<string, unknown>)) {
+      columns.add(key);
+    }
+  }
+  return columns.size > 0 ? [...columns] : undefined;
+}
+
+export function isPlayDataset<T>(value: unknown): value is PlayDataset<T> {
+  return Boolean(
+    value &&
+      typeof value === 'object' &&
+      (value as Record<PropertyKey, unknown>)[PLAY_DATASET_BRAND] === true,
+  );
+}
+
+export function isSerializedPlayDataset<T>(
+  value: unknown,
+): value is SerializedPlayDataset<T> {
+  return Boolean(
+    value &&
+      typeof value === 'object' &&
+      !Array.isArray(value) &&
+      (value as Record<string, unknown>).kind === 'dataset' &&
+      typeof (value as Record<string, unknown>).datasetKind === 'string' &&
+      typeof (value as Record<string, unknown>).datasetId === 'string' &&
+      typeof (value as Record<string, unknown>).count === 'number' &&
+      Array.isArray((value as Record<string, unknown>).preview),
+  );
+}
+
+export function trimSerializedPlayDatasetPreview<T>(
+  dataset: SerializedPlayDataset<T>,
+  limit: number,
+): SerializedPlayDataset<T> {
+  return {
+    ...dataset,
+    preview: dataset.preview.slice(0, Math.max(0, limit)),
+  };
+}
+
+class DeferredPlayDataset<T> implements PlayDataset<T> {
+  readonly [PLAY_DATASET_BRAND] = true as const;
+  readonly datasetKind: PlayDatasetKind;
+  readonly datasetId: string;
+  readonly backing?: PlayDatasetBacking;
+  readonly sourceLabel?: string | null;
+  readonly tableNamespace?: string | null;
+  private readonly previewRows: readonly T[];
+  private readonly previewColumns?: string[];
+  private cachedCount: number;
+  private readonly resolvers: PlayDatasetResolvers<T>;
+
+  constructor(input: {
+    datasetKind: PlayDatasetKind;
+    datasetId: string;
+    count: number;
+    backing?: PlayDatasetBacking;
+    previewRows: readonly T[];
+    sourceLabel?: string | null;
+    tableNamespace?: string | null;
+    resolvers: PlayDatasetResolvers<T>;
+  }) {
+    this.datasetKind = input.datasetKind;
+    this.datasetId = input.datasetId;
+    this.cachedCount = input.count;
+    this.backing = input.backing;
+    this.previewRows = input.previewRows;
+    this.previewColumns = inferPreviewColumns(this.previewRows);
+    this.sourceLabel = input.sourceLabel ?? null;
+    this.tableNamespace = input.tableNamespace ?? null;
+    this.resolvers = input.resolvers;
+  }
+
+  async count(): Promise<number> {
+    this.cachedCount = await this.resolvers.count();
+    return this.cachedCount;
+  }
+
+  async peek(limit = 10): Promise<T[]> {
+    if (limit <= this.previewRows.length) {
+      return this.previewRows.slice(0, Math.max(0, limit));
+    }
+    return await this.resolvers.peek(limit);
+  }
+
+  async materialize(limit?: number): Promise<T[]> {
+    const requestedLimit =
+      limit !== undefined ? Math.max(0, Math.floor(limit)) : undefined;
+    const cap = resolveMaterializeLimitCap();
+    if (requestedLimit !== undefined) {
+      if (requestedLimit > cap) {
+        throw new Error(
+          `PlayDataset.materialize(${requestedLimit}) exceeds the hard limit of ${cap} rows. ` +
+            'Return the dataset handle instead, or request a smaller bounded slice.',
+        );
+      }
+      return await this.resolvers.materialize(requestedLimit);
+    }
+
+    const count = await this.count();
+    if (count > cap) {
+      throw new Error(
+        `PlayDataset.materialize() refuses to load ${count} rows into memory. ` +
+          `The hard limit is ${cap}. Return the dataset handle instead or call materialize(limit).`,
+      );
+    }
+    return await this.resolvers.materialize();
+  }
+
+  async *[Symbol.asyncIterator](): AsyncIterator<T> {
+    for await (const row of this.resolvers.iterate()) {
+      yield row;
+    }
+  }
+
+  toJSON() {
+    return {
+      kind: 'dataset' as const,
+      datasetKind: this.datasetKind,
+      datasetId: this.datasetId,
+      count: this.cachedCount,
+      ...(this.backing ? { backing: this.backing } : {}),
+      ...(this.sourceLabel ? { sourceLabel: this.sourceLabel } : {}),
+      ...(this.tableNamespace ? { tableNamespace: this.tableNamespace } : {}),
+      ...(this.previewColumns ? { columns: this.previewColumns } : {}),
+      preview: [...this.previewRows],
+    };
+  }
+
+  [NODE_INSPECT_CUSTOM]() {
+    return this.toJSON();
+  }
+}
+
+export function createDeferredPlayDataset<T>(input: {
+  datasetKind: PlayDatasetKind;
+  datasetId: string;
+  count: number;
+  backing?: PlayDatasetBacking;
+  previewRows?: readonly T[];
+  sourceLabel?: string | null;
+  tableNamespace?: string | null;
+  resolvers: PlayDatasetResolvers<T>;
+}): PlayDataset<T> {
+  return new DeferredPlayDataset({
+    ...input,
+    previewRows: input.previewRows ?? [],
+  });
+}
+
+export function createPlayDataset<T>(
+  rows: readonly T[],
+  metadata?: {
+    kind?: PlayDatasetKind;
+    sourceLabel?: string | null;
+    tableNamespace?: string | null;
+    datasetId?: string;
+  },
+): PlayDataset<T> {
+  // Test/dev helper only. Production play execution should construct
+  // Neon-backed handles via createDeferredPlayDataset + runtime-host helpers.
+  const materializedRows = [...rows];
+  return createDeferredPlayDataset({
+    datasetKind: metadata?.kind ?? 'map',
+    datasetId:
+      metadata?.datasetId ??
+      `${metadata?.kind ?? 'map'}:${metadata?.tableNamespace ?? metadata?.sourceLabel ?? 'inline'}`,
+    count: materializedRows.length,
+    previewRows: materializedRows.slice(0, 10),
+    sourceLabel: metadata?.sourceLabel ?? null,
+    tableNamespace: metadata?.tableNamespace ?? null,
+    resolvers: {
+      count: async () => materializedRows.length,
+      peek: async (limit) =>
+        materializedRows.slice(0, Math.max(0, limit)),
+      materialize: async (limit) =>
+        limit === undefined
+          ? [...materializedRows]
+          : materializedRows.slice(0, Math.max(0, limit)),
+      iterate: () =>
+        ({
+          async *[Symbol.asyncIterator]() {
+            for (const row of materializedRows) {
+              yield row;
+            }
+          },
+        }) as AsyncIterable<T>,
+    },
+  });
+}
+
+export async function materializePlayDatasetInput<T>(
+  input: PlayDatasetInput<T>,
+): Promise<T[]> {
+  if (isPlayDataset(input)) {
+    return await input.materialize();
+  }
+
+  if (Array.isArray(input)) {
+    return [...input];
+  }
+
+  const rows: T[] = [];
+  if (
+    input != null &&
+    typeof input === 'object' &&
+    Symbol.asyncIterator in input
+  ) {
+    for await (const row of input as AsyncIterable<T>) {
+      rows.push(row);
+    }
+    return rows;
+  }
+
+  for (const row of input as Iterable<T>) {
+    rows.push(row);
+  }
+  return rows;
+}