goldenpipe 0.1.0

package/dist/core/index.d.ts

@@ -0,0 +1,439 @@
+import { GoldenMatchConfig } from 'goldenmatch/core';
+
+/**
+ * Core data models — port of goldenpipe/models/{context,config,stage}.py.
+ *
+ * Edge-safe: no `node:` imports. Data flows as `Row[]` (arrays of plain row
+ * objects) instead of Polars DataFrames — the TS siblings all operate on
+ * `Row[]`.
+ */
+/** A single tabular record. Mirrors the siblings' `Row` type. */
+type Row = Record<string, unknown>;
+declare const StageStatus: {
+    readonly SUCCESS: "success";
+    readonly SKIPPED: "skipped";
+    readonly FAILED: "failed";
+};
+type StageStatus = (typeof StageStatus)[keyof typeof StageStatus];
+declare const PipeStatus: {
+    readonly SUCCESS: "success";
+    readonly PARTIAL: "partial";
+    readonly FAILED: "failed";
+};
+type PipeStatus = (typeof PipeStatus)[keyof typeof PipeStatus];
+interface Decision {
+    /** Stage names to drop from the remaining plan. */
+    skip: string[];
+    /** Abort the whole pipeline after this stage. */
+    abort: boolean;
+    /** Stage names to prepend to the remaining plan. */
+    insert: string[];
+    /** Human-readable explanation, surfaced in `reasoning._router`. */
+    reason: string;
+}
+/** Construct a Decision with Python-parity defaults. */
+declare function makeDecision(input?: Partial<Decision>): Decision;
+interface StageResult {
+    status: StageStatus;
+    decision?: Decision | null;
+    error?: string | null;
+}
+interface PipeContext {
+    /** Working data. `null` until the load stage populates it. */
+    df: Row[] | null;
+    artifacts: Record<string, unknown>;
+    metadata: Record<string, unknown>;
+    timing: Record<string, number>;
+    reasoning: Record<string, string>;
+    /** Per-stage config made available to the adapter by the runner. */
+    stageConfig: Record<string, unknown>;
+}
+declare function makePipeContext(input?: Partial<PipeContext>): PipeContext;
+interface PipeResult {
+    status: PipeStatus;
+    source: string;
+    inputRows: number;
+    stages: Record<string, StageResult>;
+    artifacts: Record<string, unknown>;
+    skipped: string[];
+    errors: string[];
+    reasoning: Record<string, string>;
+    timing: Record<string, number>;
+}
+type OnError = "continue" | "abort";
+interface StageSpec {
+    name?: string | undefined;
+    use: string;
+    needs: string[];
+    skipIf?: string | undefined;
+    onError: OnError;
+    config: Record<string, unknown>;
+}
+/** Normalize a raw stage spec (object or bare string) into a full StageSpec. */
+declare function makeStageSpec(input: string | Partial<StageSpec> & {
+    use: string;
+}): StageSpec;
+interface PipelineConfig {
+    pipeline: string;
+    source?: string | undefined;
+    output?: string | undefined;
+    /** Stages may be bare strings or full specs; normalize via `makeStageSpec`. */
+    stages: Array<string | StageSpec>;
+    decisions: string[];
+}
+declare function makePipelineConfig(input: Partial<PipelineConfig> & {
+    pipeline: string;
+    stages: Array<string | StageSpec>;
+}): PipelineConfig;
+interface StageInfo {
+    name: string;
+    produces: string[];
+    consumes: string[];
+}
+/**
+ * Full contract for pipeline stages. `run` is async so it can await the
+ * async GoldenMatch `dedupe` adapter.
+ */
+interface Stage {
+    readonly info: StageInfo;
+    validate(ctx: PipeContext): void | Promise<void>;
+    run(ctx: PipeContext): Promise<StageResult>;
+    rollback?: ((ctx: PipeContext) => void | Promise<void>) | null;
+}
+/**
+ * Wrap a plain async function into a Stage. Port of the Python `@stage`
+ * decorator + `_FunctionStage`.
+ */
+declare function stage(info: StageInfo, fn: (ctx: PipeContext) => Promise<StageResult> | StageResult): Stage;
+
+/**
+ * Column context — shared column metadata flowing between pipeline stages.
+ * Port of goldenpipe/models/column_context.py.
+ *
+ * Built by GoldenCheck (scan), enriched by GoldenFlow (transform), consumed by
+ * GoldenMatch (auto-config) to avoid re-profiling.
+ *
+ * Edge-safe: no `node:` imports.
+ */
+
+declare const ColumnType: {
+    readonly NAME: "name";
+    readonly EMAIL: "email";
+    readonly PHONE: "phone";
+    readonly DATE: "date";
+    readonly GEO: "geo";
+    readonly ADDRESS: "address";
+    readonly ZIP: "zip";
+    readonly IDENTIFIER: "identifier";
+    readonly NUMERIC: "numeric";
+    readonly STRING: "string";
+    readonly DESCRIPTION: "description";
+};
+type ColumnType = (typeof ColumnType)[keyof typeof ColumnType];
+declare const CardinalityBand: {
+    readonly UNSET: "";
+    readonly LOW: "low";
+    readonly MID: "mid";
+    readonly HIGH: "high";
+    readonly SKIP: "skip";
+};
+type CardinalityBand = (typeof CardinalityBand)[keyof typeof CardinalityBand];
+/** Floor — below this, the signal is too weak to act on. */
+declare const MIN_CONFIDENCE = 0.3;
+interface ColumnContext {
+    name: string;
+    inferredType: ColumnType;
+    nullRate: number;
+    cardinality: number;
+    isIdentifier: boolean;
+    transformsApplied: string[];
+    findings: string[];
+    confidence: number;
+    cardinalityBand: CardinalityBand;
+}
+/** Construct a ColumnContext, validating invariants (port of `__post_init__`). */
+declare function makeColumnContext(input: Partial<ColumnContext> & {
+    name: string;
+}): ColumnContext;
+/** Classify a column by name pattern matching. Returns null when no match. */
+declare function classifyByName(colName: string): ColumnType | null;
+/** Map a profiler dtype string to a ColumnType. */
+declare function normalizeDtype(rawType: string): ColumnType;
+/** Minimal shape of a GoldenCheck ColumnProfile that we consume. */
+interface ColumnProfileLike {
+    name: string;
+    inferredType?: string;
+    nullPct?: number;
+    uniqueCount?: number;
+}
+/** Minimal shape of a GoldenCheck finding that we consume. */
+interface FindingLike {
+    column?: string;
+    check?: string;
+    message?: string;
+}
+/**
+ * Build ColumnContexts from GoldenCheck scan results. Combines three signals:
+ * 1. Column-name heuristics (regex patterns).
+ * 2. Profile data (null rate, cardinality, dtype).
+ * 3. Cardinality IQR bands.
+ */
+declare function buildContextsFromCheck(findings: readonly FindingLike[], columnProfiles: readonly ColumnProfileLike[] | null | undefined): ColumnContext[];
+/** Minimal shape of a GoldenFlow manifest record we consume. */
+interface ManifestRecordLike {
+    column?: string;
+    transform?: string;
+    affectedRows?: number;
+}
+/** Enrich ColumnContexts with GoldenFlow transform information. */
+declare function enrichContextsFromFlow(contexts: ColumnContext[], records: readonly ManifestRecordLike[] | null | undefined): void;
+declare function distinctNonNull(rows: readonly Row[], col: string): number;
+declare function nullRateOf(rows: readonly Row[], col: string): number;
+
+/**
+ * Stage registry — register and retrieve stages.
+ * Port of goldenpipe/engine/registry.py.
+ *
+ * Unlike the Python version, which discovers stages via importlib entry points,
+ * the TS registry is STATIC: built-in stages (load, goldencheck.scan,
+ * goldenflow.transform, goldenmatch.dedupe) are registered explicitly in
+ * `defaultRegistry()`. Custom stages can be added with `register()`.
+ *
+ * Edge-safe: no `node:` imports.
+ */
+
+declare class StageRegistry {
+    private readonly stages;
+    /** Register a stage under its `info.name`. */
+    register(stage: Stage): void;
+    /** Retrieve a stage by name. Throws if not found. */
+    get(name: string): Stage;
+    /** True when a stage with this name is registered. */
+    has(name: string): boolean;
+    /** Return `{ name: StageInfo }` for all registered stages. */
+    listAll(): Record<string, StageInfo>;
+}
+/**
+ * Build a registry with the built-in suite stages registered. This is the TS
+ * analogue of Python's entry-point discovery — wiring goldencheck.scan,
+ * goldenflow.transform, and goldenmatch.dedupe (+ the built-in `load` stage).
+ */
+declare function defaultRegistry(): StageRegistry;
+
+/**
+ * Pipeline resolver — build an ExecutionPlan and validate wiring.
+ * Port of goldenpipe/engine/resolver.py.
+ *
+ * Edge-safe: no `node:` imports.
+ */
+
+/** Raised when a stage's `consumes` can't be satisfied by prior `produces`. */
+declare class WiringError extends Error {
+    constructor(message: string);
+}
+interface PlannedStage {
+    name: string;
+    stage: Stage;
+    spec: StageSpec;
+    config: Record<string, unknown>;
+}
+interface ExecutionPlan {
+    stages: PlannedStage[];
+}
+declare const Resolver: {
+    /**
+     * Resolve a config + registry into an ordered ExecutionPlan. Auto-prepends
+     * the built-in `load` stage when available and validates that every stage's
+     * `consumes` is produced by an earlier stage.
+     */
+    resolve(config: PipelineConfig, registry: StageRegistry): ExecutionPlan;
+};
+
+/**
+ * Decision router — apply routing decisions to the remaining execution plan.
+ * Port of goldenpipe/engine/router.py.
+ *
+ * Edge-safe: no `node:` imports.
+ */
+
+declare const Router: {
+    /**
+     * Apply a Decision (skip / abort / insert) to the remaining stages and
+     * return the new remaining list. Records `decision.reason` in
+     * `ctx.reasoning._router`.
+     */
+    apply(decision: Decision, remaining: PlannedStage[], ctx: PipeContext, registry: StageRegistry): PlannedStage[];
+};
+
+/**
+ * Pipeline runner — execute stages with error handling and routing.
+ * Port of goldenpipe/engine/runner.py.
+ *
+ * ASYNC: stage execution awaits each stage's `run`, because the GoldenMatch
+ * `dedupe` adapter is async.
+ *
+ * Edge-safe: no `node:` imports.
+ */
+
+declare class Runner {
+    private readonly registry;
+    constructor(registry: StageRegistry);
+    /** Execute an ExecutionPlan against a PipeContext, returning per-stage results. */
+    run(plan: ExecutionPlan, ctx: PipeContext): Promise<Record<string, StageResult>>;
+}
+
+/**
+ * Reporter — build a PipeResult from a PipeContext after execution.
+ * Port of goldenpipe/engine/reporter.py.
+ *
+ * Edge-safe: no `node:` imports.
+ */
+
+declare const Reporter: {
+    build(ctx: PipeContext, stages: Record<string, StageResult>): PipeResult;
+};
+
+/**
+ * Built-in decision functions for pipeline routing.
+ * Port of goldenpipe/decisions.py.
+ *
+ * Edge-safe: no `node:` imports.
+ *
+ * NOTE on TS sibling skew: GoldenCheck-JS `Finding.severity` is a numeric enum
+ * (INFO=1, WARNING=2, ERROR=3) and has no `"critical"` level, and there is no
+ * `"pii_detection"` check. The check adapter normalizes findings to a string
+ * `severity` label ("info"/"warning"/"error") and a `check` string. So in
+ * practice `severityGate` and `piiRouter` are no-ops against current
+ * GoldenCheck-JS output — they are ported for structural parity and so that
+ * custom stages emitting `"critical"` / `"pii_detection"` findings still route.
+ */
+
+/** Abort the pipeline if any finding has `critical` severity. */
+declare function severityGate(ctx: PipeContext): Decision | null;
+/** Route to PPRL matching if PII is detected. */
+declare function piiRouter(ctx: PipeContext): Decision | null;
+/** Skip matching if fewer than 2 rows. */
+declare function rowCountGate(ctx: PipeContext): Decision | null;
+
+/**
+ * Built-in LoadStage — marks `df` as available. The actual data loading is
+ * handled by the Pipeline (file read) or supplied by the caller (runDf).
+ * Port of goldenpipe/adapters/__init__.py LoadStage.
+ *
+ * Edge-safe: no `node:` imports.
+ */
+
+declare const LoadStage: Stage;
+
+/**
+ * GoldenCheck adapter — wraps GoldenCheck-JS `scanData`.
+ * Port of goldenpipe/adapters/check.py.
+ *
+ * Shape divergence vs Python: the Python adapter calls `scan_file(path)` and
+ * reads `ctx.metadata["source"]`. GoldenCheck-JS's edge-safe `scanData` instead
+ * operates on a `TabularData` built from rows, so the TS adapter scans
+ * `ctx.df` directly. This means `goldencheck.scan` succeeds in the in-memory
+ * (`runDf`) path here, whereas the Python `run_df` path fails the scan stage
+ * (it has no file). Use `run(source)` for cross-language parity.
+ *
+ * Edge-safe: no `node:` imports (GoldenCheck-JS core is edge-safe).
+ */
+
+declare const ScanStage: Stage;
+
+/**
+ * GoldenFlow adapter — wraps GoldenFlow-JS `TransformEngine.transformDf`.
+ * Port of goldenpipe/adapters/flow.py.
+ *
+ * Shape note: GoldenFlow-JS exposes `new TransformEngine(config).transformDf(rows)`
+ * which returns `{ rows, columns, manifest }` (the Python sibling's
+ * `transform_df(df)` returns an object with `.df` + `.manifest`). We read
+ * `.rows` back into `ctx.df` and surface `.manifest` as an artifact.
+ *
+ * Edge-safe: no `node:` imports (GoldenFlow-JS core is edge-safe).
+ */
+
+declare const TransformStage: Stage;
+
+/**
+ * GoldenMatch adapter — wraps GoldenMatch-JS `dedupe`.
+ * Port of goldenpipe/adapters/match.py (+ `_build_config_from_contexts`).
+ *
+ * `dedupe` is ASYNC, so this stage's `run` awaits it (and the whole runner is
+ * async). Config selection priority mirrors Python:
+ *   1. explicit stage config (from YAML / PipelineConfig.config)
+ *   2. config built from upstream column contexts
+ *   3. GoldenMatch auto-configure (no config → shorthand path)
+ *
+ * Shape divergences from the Python sibling, surfaced as artifacts:
+ *   - GoldenMatch-JS `DedupeResult` exposes `.goldenRecords` (not `.golden`),
+ *     `.dupes`, `.unique`, `.stats`, `.scoredPairs`. We map `goldenRecords` to
+ *     the `golden` artifact for parity with the Python pipeline's artifact name.
+ *   - `matchkey_used` is derived from the built config's first matchkey (the
+ *     JS result does not carry the resolved matchkey list back).
+ *
+ * Edge-safe: no `node:` imports (GoldenMatch-JS core is edge-safe).
+ */
+
+declare const DedupeStage: Stage;
+/**
+ * Build a GoldenMatchConfig from pipeline column contexts. Returns `null` if no
+ * usable matchkeys can be built (caller then falls back to auto-configure).
+ * Port of `_build_config_from_contexts`.
+ */
+declare function buildConfigFromContexts(contexts: readonly ColumnContext[], rows: readonly Row[]): GoldenMatchConfig | null;
+
+/**
+ * Adapters index — built-in suite stages + registry wiring.
+ * Replaces Python's importlib entry-point discovery with a STATIC registry.
+ *
+ * Edge-safe: no `node:` imports.
+ */
+
+/**
+ * Build a registry with all built-in suite stages registered:
+ *   - `load` (built-in)
+ *   - `goldencheck.scan`
+ *   - `goldenflow.transform`
+ *   - `goldenmatch.dedupe`
+ *
+ * This is the TS analogue of Python's `StageRegistry.discover()`.
+ */
+declare function buildDefaultRegistry(): StageRegistry;
+
+/**
+ * Pipeline — high-level orchestrator + programmatic run helpers.
+ * Port of goldenpipe/pipeline.py + goldenpipe/_api.py (the DataFrame paths).
+ *
+ * Operates on `Row[]`. The file-loading `run(source)` entry point lives in
+ * `node/` (it needs `node:fs`); the edge-safe core only exposes the in-memory
+ * `runDf` / `runStages` paths.
+ *
+ * Edge-safe: no `node:` imports.
+ */
+
+interface PipelineOptions {
+    config?: PipelineConfig | undefined;
+    registry?: StageRegistry | undefined;
+}
+declare class Pipeline {
+    private readonly config;
+    private readonly registry;
+    constructor(options?: PipelineOptions);
+    /** Run the pipeline on an array of rows. */
+    run(rows: readonly Row[], source?: string): Promise<PipeResult>;
+    /** Build the default check→flow→dedupe config from the available stages. */
+    private autoConfig;
+}
+/**
+ * Run a pipeline on an array of rows. Zero-config (default suite chain) or with
+ * an explicit PipelineConfig. Port of `_api.run_df`.
+ */
+declare function runDf(rows: readonly Row[], config?: PipelineConfig, source?: string): Promise<PipeResult>;
+/**
+ * Run specific stages programmatically against rows. Port of `_api.run_stages`.
+ * The auto-prepended `load` stage is removed since rows are already supplied.
+ */
+declare function runStages(stages: readonly Stage[], rows: readonly Row[]): Promise<PipeResult>;
+
+export { CardinalityBand, type ColumnContext, type ColumnProfileLike, ColumnType, type Decision, DedupeStage, type ExecutionPlan, type FindingLike, LoadStage, MIN_CONFIDENCE, type ManifestRecordLike, type OnError, type PipeContext, type PipeResult, PipeStatus, Pipeline, type PipelineConfig, type PipelineOptions, type PlannedStage, Reporter, Resolver, Router, type Row, Runner, ScanStage, type Stage, type StageInfo, StageRegistry, type StageResult, type StageSpec, StageStatus, TransformStage, WiringError, buildConfigFromContexts, buildContextsFromCheck, buildDefaultRegistry, classifyByName, defaultRegistry, distinctNonNull, enrichContextsFromFlow, makeColumnContext, makeDecision, makePipeContext, makePipelineConfig, makeStageSpec, normalizeDtype, nullRateOf, piiRouter, rowCountGate, runDf, runStages, severityGate, stage };