goldenflow 0.1.0

package/dist/core/index.d.ts

@@ -0,0 +1,360 @@
+/**
+ * Core types for GoldenFlow JS — ported from Python dataclasses and Pydantic models.
+ * All types are readonly. Use factory functions for construction.
+ */
+type TransformMode = "expr" | "series" | "dataframe";
+/** Metadata for a registered transform. */
+interface TransformInfo {
+    readonly name: string;
+    readonly func: TransformFunction;
+    readonly inputTypes: readonly string[];
+    readonly autoApply: boolean;
+    readonly priority: number;
+    readonly mode: TransformMode;
+}
+/**
+ * A transform function. Shape depends on mode:
+ * - expr:      (column: string, ...params: unknown[]) => ExprTransform
+ * - series:    (values: readonly ColumnValue[], ...params: unknown[]) => ColumnValue[] | [ColumnValue[], number[]]
+ * - dataframe: (rows: readonly Row[], column: string) => Row[]
+ */
+type TransformFunction = (...args: any[]) => any;
+type ColumnValue = string | number | boolean | null;
+type Row = Readonly<Record<string, unknown>>;
+type Dtype = "string" | "integer" | "float" | "boolean" | "date" | "datetime" | "null";
+interface TransformRecord {
+    readonly column: string;
+    readonly transform: string;
+    readonly affectedRows: number;
+    readonly totalRows: number;
+    readonly sampleBefore: readonly string[];
+    readonly sampleAfter: readonly string[];
+}
+declare function makeTransformRecord(input: Pick<TransformRecord, "column" | "transform" | "affectedRows" | "totalRows"> & Partial<Pick<TransformRecord, "sampleBefore" | "sampleAfter">>): TransformRecord;
+interface TransformError {
+    readonly column: string;
+    readonly transform: string;
+    readonly row: number;
+    readonly error: string;
+}
+interface Manifest {
+    readonly source: string;
+    readonly records: readonly TransformRecord[];
+    readonly errors: readonly TransformError[];
+    readonly createdAt: string;
+}
+declare function makeManifest(source: string): MutableManifest;
+/** Mutable manifest used during transform pipeline, then frozen. */
+declare class MutableManifest implements Manifest {
+    readonly source: string;
+    readonly records: TransformRecord[];
+    readonly errors: TransformError[];
+    readonly createdAt: string;
+    constructor(source: string);
+    addRecord(record: TransformRecord): void;
+    addError(column: string, transform: string, row: number, error: string): void;
+    toDict(): Record<string, unknown>;
+}
+interface TransformResult {
+    readonly rows: readonly Row[];
+    readonly columns: readonly string[];
+    readonly manifest: Manifest;
+}
+interface ColumnProfile {
+    readonly name: string;
+    readonly inferredType: string;
+    readonly rowCount: number;
+    readonly nullCount: number;
+    readonly nullPct: number;
+    readonly uniqueCount: number;
+    readonly uniquePct: number;
+    readonly sampleValues: readonly string[];
+    readonly detectedFormat: string | null;
+}
+declare function makeColumnProfile(input: Pick<ColumnProfile, "name" | "inferredType" | "rowCount" | "nullCount" | "nullPct" | "uniqueCount" | "uniquePct"> & Partial<Pick<ColumnProfile, "sampleValues" | "detectedFormat">>): ColumnProfile;
+interface DatasetProfile {
+    readonly filePath: string;
+    readonly rowCount: number;
+    readonly columnCount: number;
+    readonly columns: readonly ColumnProfile[];
+}
+interface ColumnDiffDetail {
+    readonly changedRows: number;
+}
+interface DiffResult {
+    readonly totalChanges: number;
+    readonly changedColumns: readonly string[];
+    readonly addedColumns: readonly string[];
+    readonly removedColumns: readonly string[];
+    readonly rowCountBefore: number;
+    readonly rowCountAfter: number;
+    readonly columnDetails: Readonly<Record<string, ColumnDiffDetail>>;
+}
+interface TransformSpec {
+    readonly column: string;
+    readonly ops: readonly string[];
+}
+interface SplitSpec {
+    readonly source: string;
+    readonly target: readonly string[];
+    readonly method: string;
+}
+interface FilterSpec {
+    readonly column: string;
+    readonly condition: string;
+}
+interface DedupSpec {
+    readonly columns: readonly string[];
+    readonly keep: "first" | "last";
+}
+interface MappingSpec {
+    readonly source: string;
+    readonly target: string | readonly string[];
+    readonly transform: string | readonly string[] | null;
+}
+interface GoldenFlowConfig {
+    readonly source: string | null;
+    readonly output: string | null;
+    readonly transforms: readonly TransformSpec[];
+    readonly splits: readonly SplitSpec[];
+    readonly renames: Readonly<Record<string, string>>;
+    readonly drop: readonly string[];
+    readonly filters: readonly FilterSpec[];
+    readonly dedup: DedupSpec | null;
+    readonly mappings: readonly MappingSpec[];
+}
+declare function makeConfig(input?: Partial<GoldenFlowConfig>): GoldenFlowConfig;
+interface ColumnMapping {
+    readonly source: string;
+    readonly target: string;
+    readonly confidence: number;
+    readonly transform: string | null;
+}
+interface DomainPack {
+    readonly name: string;
+    readonly description: string;
+    readonly transforms: readonly string[];
+    readonly defaultConfig: GoldenFlowConfig;
+}
+interface RunRecord {
+    readonly runId: string;
+    readonly source: string;
+    readonly timestamp: string;
+    readonly rows: number;
+    readonly columns: number;
+    readonly transformsApplied: number;
+    readonly errors: number;
+    readonly durationSeconds: number;
+    readonly configHash: string | null;
+    readonly manifestPath: string | null;
+}
+
+/**
+ * TabularData — edge-safe Polars replacement.
+ * Wraps Record<string, unknown>[] and provides column operations.
+ */
+
+declare function isNullish(v: unknown): v is null | undefined;
+declare function toColumnValue(v: unknown): ColumnValue;
+declare class TabularData {
+    private readonly _rows;
+    private _columnCache;
+    constructor(rows: readonly Row[]);
+    get rows(): readonly Row[];
+    get columns(): readonly string[];
+    get rowCount(): number;
+    column(name: string): readonly ColumnValue[];
+    /** Raw column access — preserves original values without null coercion.
+     *  Use for profiling where "N/A" should remain a string, not become null. */
+    rawColumn(name: string): readonly ColumnValue[];
+    nullCount(col: string): number;
+    dropNulls(col: string): ColumnValue[];
+    dtype(col: string): Dtype;
+    nUnique(col: string): number;
+    valueCounts(col: string): Map<ColumnValue, number>;
+    /** MUST use loop — Math.min(...array) crashes on >65K elements. */
+    min(col: string): number | null;
+    /** MUST use loop — Math.max(...array) crashes on >65K elements. */
+    max(col: string): number | null;
+    mean(col: string): number | null;
+    std(col: string): number | null;
+    filter(predicate: (row: Row) => boolean): TabularData;
+    head(n: number): TabularData;
+    sample(n: number, seed?: number): TabularData;
+    strContains(col: string, pattern: RegExp): boolean[];
+    strLengths(col: string): number[];
+    castFloat(col: string): (number | null)[];
+    castInt(col: string): (number | null)[];
+    numericValues(col: string): number[];
+    stringValues(col: string): string[];
+    sortedNumeric(col: string): number[];
+    isSorted(col: string, descending?: boolean): boolean;
+}
+
+/**
+ * Transform registry — TS equivalent of goldenflow/transforms/__init__.py.
+ * Transforms self-register via registerTransform().
+ */
+
+interface RegisterOptions {
+    name: string;
+    inputTypes: readonly string[];
+    autoApply?: boolean;
+    priority?: number;
+    mode?: TransformMode;
+}
+declare function registerTransform(opts: RegisterOptions, func: TransformFunction): void;
+declare function getTransform(name: string): TransformInfo | undefined;
+declare function listTransforms(): TransformInfo[];
+declare function parseTransformName(raw: string): [string, string[]];
+declare function registry(): ReadonlyMap<string, TransformInfo>;
+
+/**
+ * TransformEngine — the main orchestrator.
+ * Dispatches transforms by mode (expr/series/dataframe) on Row[] data.
+ */
+
+declare class TransformEngine {
+    readonly config: GoldenFlowConfig;
+    constructor(config?: Partial<GoldenFlowConfig>);
+    transformDf(rows: readonly Row[], source?: string): TransformResult;
+    private _applyConfigTransforms;
+    private _applyAutoTransforms;
+    private _applySingleTransform;
+    private _applyFilter;
+}
+
+/**
+ * Profiler bridge — infer column types from data using regex heuristics.
+ * Edge-safe (no Node dependencies).
+ */
+
+declare function profileDataframe(rows: readonly Row[], filePath?: string): DatasetProfile;
+
+/**
+ * Selector — pick auto-applicable transforms for a column based on its profile.
+ */
+
+/** GoldenCheck finding check → transform name mapping. */
+declare const FINDING_TRANSFORM_MAP: Readonly<Record<string, readonly string[]>>;
+declare function selectTransforms(profile: ColumnProfile, _confidenceThreshold?: number): TransformInfo[];
+declare function selectFromFindings(findings: readonly Record<string, unknown>[]): Record<string, string[]>;
+
+/**
+ * Differ — compare two row arrays and report differences.
+ */
+
+declare function diffDataframes(before: readonly Row[], after: readonly Row[]): DiffResult;
+
+/**
+ * StreamProcessor — incremental transform processing.
+ */
+
+declare class StreamProcessor {
+    private readonly engine;
+    private _batchCount;
+    constructor(config?: Partial<GoldenFlowConfig>);
+    /** Transform a single record. */
+    transformOne(record: Record<string, unknown>): TransformResult;
+    /** Transform a batch of rows. */
+    transformBatch(rows: readonly Row[]): TransformResult;
+    /** Process rows in chunks, yielding TransformResult per chunk. */
+    streamRows(rows: readonly Row[], chunkSize?: number): Generator<TransformResult>;
+    get batchesProcessed(): number;
+}
+
+/**
+ * Config schema — re-exports types and provides runtime validation.
+ */
+
+/** Basic runtime validation (not as strict as Pydantic, but catches common errors). */
+declare function validateConfig(raw: Record<string, unknown>): GoldenFlowConfig;
+
+/**
+ * Config loader — YAML load/save/merge.
+ * YAML is an optional peer dependency.
+ */
+
+declare function loadConfigFromString(content: string): GoldenFlowConfig;
+declare function saveConfigToString(config: GoldenFlowConfig): string;
+declare function mergeConfigs(fileConfig: GoldenFlowConfig, cliOverrides: Partial<GoldenFlowConfig>): GoldenFlowConfig;
+
+/**
+ * Config learner — generate a config from data profiles.
+ */
+
+declare function learnConfig(rows: readonly Row[], source?: string): GoldenFlowConfig;
+
+/**
+ * Name similarity — fuzzy column name matching with alias support.
+ */
+declare function nameSimilarity(source: string, target: string): number;
+
+/**
+ * Profile similarity — score how similar two column profiles are.
+ */
+
+declare function profileSimilarity(source: ColumnProfile, target: ColumnProfile): number;
+
+/**
+ * SchemaMapper — auto-map source columns to target columns.
+ */
+
+declare class SchemaMapper {
+    readonly autoThreshold: number;
+    readonly suggestThreshold: number;
+    constructor(autoThreshold?: number, suggestThreshold?: number);
+    map(sourceRows: readonly Row[], targetRows: readonly Row[]): ColumnMapping[];
+    toConfig(mappings: readonly ColumnMapping[]): GoldenFlowConfig;
+}
+
+declare function loadDomain(name: string): Promise<DomainPack | null>;
+declare function listDomains(): string[];
+
+/**
+ * JSON reporter — serialize manifest to JSON.
+ */
+
+declare function manifestToJson(manifest: Manifest): string;
+
+declare function printProfile(profile: DatasetProfile): void;
+declare function printManifest(manifest: Manifest): void;
+declare function printDiff(diff: DiffResult): void;
+
+/**
+ * LLM-assisted categorical correction -- edge-safe (raw fetch, no SDK imports).
+ *
+ * Registers `category_llm_correct` as a passthrough transform. The actual LLM
+ * interaction happens via the async helper `applyLlmCorrections()`, which the
+ * engine or CLI should call explicitly before/after the sync transform pipeline.
+ */
+
+/**
+ * Fetch LLM-based corrections for a column's values and cache them.
+ *
+ * Usage:
+ * ```ts
+ * await prepareLlmCorrections("status", rows.map(r => r.status));
+ * // ... then run the sync transform pipeline which includes category_llm_correct
+ * ```
+ *
+ * Returns the corrections map (also cached internally).
+ */
+declare function prepareLlmCorrections(columnName: string, values: readonly ColumnValue[]): Promise<Record<string, string>>;
+/**
+ * High-level async helper: fetch corrections then apply them to `values`.
+ *
+ * Returns corrected values array (same length as input). Skips the LLM call
+ * if corrections are already cached for `columnName`.
+ */
+declare function applyLlmCorrections(columnName: string, values: readonly ColumnValue[]): Promise<ColumnValue[]>;
+
+/**
+ * Notebook HTML rendering — generates HTML tables for TransformResult, Manifest, DatasetProfile.
+ */
+
+declare function transformResultToHtml(result: TransformResult): string;
+declare function manifestToHtml(manifest: Manifest): string;
+declare function profileToHtml(profile: DatasetProfile): string;
+
+export { type ColumnDiffDetail, type ColumnMapping, type ColumnProfile, type ColumnValue, type DatasetProfile, type DedupSpec, type DiffResult, type DomainPack, type Dtype, FINDING_TRANSFORM_MAP, type FilterSpec, type GoldenFlowConfig, type Manifest, type MappingSpec, MutableManifest, type Row, type RunRecord, SchemaMapper, type SplitSpec, StreamProcessor, TabularData, TransformEngine, type TransformError, type TransformFunction, type TransformInfo, type TransformMode, type TransformRecord, type TransformResult, type TransformSpec, applyLlmCorrections, diffDataframes, getTransform, isNullish, learnConfig, listDomains, listTransforms, loadConfigFromString, loadDomain, makeColumnProfile, makeConfig, makeManifest, makeTransformRecord, manifestToHtml, manifestToJson, mergeConfigs, nameSimilarity, parseTransformName, prepareLlmCorrections, printDiff, printManifest, printProfile, profileDataframe, profileSimilarity, profileToHtml, registerTransform, registry, saveConfigToString, selectFromFindings, selectTransforms, toColumnValue, transformResultToHtml, validateConfig };