@nice2dev/ui-tools 1.0.10

package/dist/data-pipeline.d.ts

@@ -0,0 +1,1105 @@
+/**
+ * @file data-pipeline.ts
+ * @module @nice2dev/ui-tools
+ * @description PRO-15.3 — Data Pipeline (ETL)
+ *
+ * Types for visual ETL designer: source connectors, transformations, data cleansing,
+ * data quality, scheduling, incremental load, error handling, lineage tracking.
+ */
+/** Pipeline status */
+export type PipelineStatus = 'draft' | 'scheduled' | 'running' | 'paused' | 'completed' | 'failed' | 'cancelled';
+/** Node status */
+export type NodeStatus = 'pending' | 'running' | 'completed' | 'failed' | 'skipped' | 'warning';
+/** Data type */
+export type DataType = 'string' | 'number' | 'integer' | 'float' | 'boolean' | 'date' | 'datetime' | 'timestamp' | 'time' | 'binary' | 'array' | 'object' | 'json' | 'uuid' | 'null';
+/** Pipeline designer configuration */
+export interface PipelineDesignerConfig {
+    /** Initial pipeline */
+    pipeline?: Pipeline;
+    /** Editor mode */
+    mode?: PipelineEditorMode;
+    /** Theme */
+    theme?: PipelineTheme;
+    /** Locale */
+    locale?: string;
+    /** Available source connectors */
+    sourceConnectors?: SourceConnectorDefinition[];
+    /** Available transformations */
+    transformations?: TransformationDefinition[];
+    /** Available destination writers */
+    destinationWriters?: DestinationWriterDefinition[];
+    /** Custom nodes */
+    customNodes?: CustomNodeDefinition[];
+    /** Enable preview */
+    preview?: boolean;
+    /** Enable debugging */
+    debug?: boolean;
+    /** Toolbar configuration */
+    toolbar?: PipelineToolbarConfig;
+    /** Sidebar configuration */
+    sidebar?: PipelineSidebarConfig;
+    /** Canvas configuration */
+    canvas?: CanvasConfig;
+    /** Validation rules */
+    validationRules?: ValidationRule[];
+}
+/** Pipeline editor mode */
+export type PipelineEditorMode = 'visual' | 'code' | 'split';
+/** Pipeline theme */
+export interface PipelineTheme {
+    name?: string;
+    background?: string;
+    canvas?: string;
+    grid?: string;
+    node?: NodeColors;
+    edge?: EdgeColors;
+    text?: string;
+}
+/** Node colors */
+export interface NodeColors {
+    source?: string;
+    transform?: string;
+    destination?: string;
+    custom?: string;
+    error?: string;
+    warning?: string;
+    success?: string;
+    running?: string;
+}
+/** Edge colors */
+export interface EdgeColors {
+    default?: string;
+    active?: string;
+    error?: string;
+    warning?: string;
+}
+/** Pipeline toolbar configuration */
+export interface PipelineToolbarConfig {
+    visible?: boolean;
+    items?: PipelineToolbarItem[];
+    customItems?: CustomToolbarItem[];
+}
+/** Pipeline toolbar item */
+export type PipelineToolbarItem = 'save' | 'run' | 'stop' | 'schedule' | 'debug' | 'preview' | 'validate' | 'undo' | 'redo' | 'zoomIn' | 'zoomOut' | 'fitView' | 'export' | 'import' | 'settings' | 'separator';
+/** Custom toolbar item */
+export interface CustomToolbarItem {
+    id: string;
+    label?: string;
+    icon?: string;
+    tooltip?: string;
+    handler: () => void;
+    disabled?: boolean;
+    position?: number;
+}
+/** Pipeline sidebar configuration */
+export interface PipelineSidebarConfig {
+    visible?: boolean;
+    position?: 'left' | 'right';
+    width?: number;
+    tabs?: ('nodes' | 'properties' | 'data' | 'logs' | 'lineage')[];
+}
+/** Canvas configuration */
+export interface CanvasConfig {
+    width?: number;
+    height?: number;
+    minZoom?: number;
+    maxZoom?: number;
+    gridSize?: number;
+    showGrid?: boolean;
+    snapToGrid?: boolean;
+    connectionType?: 'bezier' | 'straight' | 'step' | 'smoothstep';
+    nodeSpacing?: number;
+    layerSpacing?: number;
+}
+/** Validation rule */
+export interface ValidationRule {
+    id: string;
+    name: string;
+    type: 'error' | 'warning' | 'info';
+    check: (pipeline: Pipeline) => ValidationResult[];
+}
+/** Validation result */
+export interface ValidationResult {
+    nodeId?: string;
+    edgeId?: string;
+    type: 'error' | 'warning' | 'info';
+    message: string;
+    details?: string;
+}
+/** Pipeline */
+export interface Pipeline {
+    id: string;
+    name: string;
+    description?: string;
+    nodes: PipelineNode[];
+    edges: PipelineEdge[];
+    parameters?: PipelineParameter[];
+    variables?: PipelineVariable[];
+    schedule?: PipelineSchedule;
+    settings?: PipelineSettings;
+    metadata?: PipelineMetadata;
+    version?: string;
+}
+/** Pipeline node */
+export interface PipelineNode {
+    id: string;
+    type: NodeType;
+    name: string;
+    description?: string;
+    position: NodePosition;
+    config: NodeConfig;
+    schema?: NodeSchema;
+    validations?: NodeValidation[];
+    status?: NodeStatus;
+    metrics?: NodeMetrics;
+}
+/** Node type */
+export type NodeType = 'source' | 'transform' | 'destination' | 'junction' | 'filter' | 'aggregate' | 'join' | 'union' | 'lookup' | 'script' | 'branch' | 'merge' | 'custom';
+/** Node position */
+export interface NodePosition {
+    x: number;
+    y: number;
+}
+/** Node configuration */
+export type NodeConfig = SourceNodeConfig | TransformNodeConfig | DestinationNodeConfig | JunctionNodeConfig | FilterNodeConfig | AggregateNodeConfig | JoinNodeConfig | UnionNodeConfig | LookupNodeConfig | ScriptNodeConfig | BranchNodeConfig | MergeNodeConfig | CustomNodeConfig;
+/** Node schema */
+export interface NodeSchema {
+    fields: SchemaField[];
+    partitionBy?: string[];
+    sortBy?: SortField[];
+}
+/** Schema field */
+export interface SchemaField {
+    name: string;
+    type: DataType;
+    nullable?: boolean;
+    metadata?: Record<string, unknown>;
+    source?: string;
+    expression?: string;
+}
+/** Sort field */
+export interface SortField {
+    field: string;
+    direction: 'asc' | 'desc';
+    nullsFirst?: boolean;
+}
+/** Node validation */
+export interface NodeValidation {
+    field?: string;
+    rule: ValidationRuleType;
+    params?: Record<string, unknown>;
+    errorAction?: 'reject' | 'default' | 'null' | 'skip';
+    defaultValue?: unknown;
+}
+/** Validation rule type */
+export type ValidationRuleType = 'notNull' | 'unique' | 'type' | 'range' | 'regex' | 'enum' | 'length' | 'email' | 'url' | 'date' | 'custom';
+/** Node metrics */
+export interface NodeMetrics {
+    rowsRead?: number;
+    rowsWritten?: number;
+    rowsRejected?: number;
+    bytesRead?: number;
+    bytesWritten?: number;
+    startTime?: Date;
+    endTime?: Date;
+    duration?: number;
+    errors?: number;
+    warnings?: number;
+}
+/** Pipeline edge */
+export interface PipelineEdge {
+    id: string;
+    source: string;
+    sourceHandle?: string;
+    target: string;
+    targetHandle?: string;
+    label?: string;
+    type?: EdgeType;
+    condition?: EdgeCondition;
+    style?: EdgeStyle;
+}
+/** Edge type */
+export type EdgeType = 'default' | 'conditional' | 'error' | 'loop';
+/** Edge condition */
+export interface EdgeCondition {
+    type: 'expression' | 'success' | 'failure' | 'always';
+    expression?: string;
+}
+/** Edge style */
+export interface EdgeStyle {
+    color?: string;
+    strokeWidth?: number;
+    animated?: boolean;
+    dashArray?: string;
+}
+/** Pipeline parameter */
+export interface PipelineParameter {
+    name: string;
+    type: DataType;
+    defaultValue?: unknown;
+    required?: boolean;
+    description?: string;
+    options?: ParameterOption[];
+    validation?: ParameterValidation;
+}
+/** Parameter option */
+export interface ParameterOption {
+    value: unknown;
+    label: string;
+}
+/** Parameter validation */
+export interface ParameterValidation {
+    min?: number;
+    max?: number;
+    pattern?: string;
+    custom?: string;
+}
+/** Pipeline variable */
+export interface PipelineVariable {
+    name: string;
+    type: DataType;
+    value?: unknown;
+    expression?: string;
+    scope?: 'pipeline' | 'node';
+}
+/** Pipeline schedule */
+export interface PipelineSchedule {
+    enabled: boolean;
+    type: 'cron' | 'interval' | 'event' | 'manual';
+    cron?: string;
+    interval?: number;
+    intervalUnit?: 'minutes' | 'hours' | 'days';
+    timezone?: string;
+    startDate?: Date;
+    endDate?: Date;
+    retryPolicy?: RetryPolicy;
+    concurrency?: ConcurrencyPolicy;
+}
+/** Retry policy */
+export interface RetryPolicy {
+    maxRetries: number;
+    retryDelay: number;
+    backoffMultiplier?: number;
+    maxDelay?: number;
+    retryableErrors?: string[];
+}
+/** Concurrency policy */
+export interface ConcurrencyPolicy {
+    maxConcurrent: number;
+    queueOverflow: 'skip' | 'queue' | 'replace';
+}
+/** Pipeline settings */
+export interface PipelineSettings {
+    executionMode?: 'batch' | 'streaming' | 'micro-batch';
+    batchSize?: number;
+    parallelism?: number;
+    timeout?: number;
+    memoryLimit?: number;
+    checkpointing?: CheckpointConfig;
+    logging?: LoggingConfig;
+    notifications?: NotificationConfig[];
+    incrementalLoad?: IncrementalLoadConfig;
+}
+/** Checkpoint config */
+export interface CheckpointConfig {
+    enabled: boolean;
+    interval?: number;
+    storage?: string;
+    maxCheckpoints?: number;
+}
+/** Logging config */
+export interface LoggingConfig {
+    level: 'debug' | 'info' | 'warn' | 'error';
+    destination?: string;
+    retentionDays?: number;
+    includeData?: boolean;
+    sampleRate?: number;
+}
+/** Notification config */
+export interface NotificationConfig {
+    event: 'start' | 'success' | 'failure' | 'warning' | 'sla';
+    channel: 'email' | 'slack' | 'teams' | 'webhook' | 'pager';
+    recipients?: string[];
+    config?: Record<string, unknown>;
+}
+/** Incremental load config */
+export interface IncrementalLoadConfig {
+    enabled: boolean;
+    strategy: 'timestamp' | 'id' | 'cdc' | 'hash';
+    watermarkColumn?: string;
+    initialLoad?: 'full' | 'none' | 'delta';
+    lookbackWindow?: number;
+}
+/** Pipeline metadata */
+export interface PipelineMetadata {
+    created?: Date;
+    createdBy?: string;
+    modified?: Date;
+    modifiedBy?: string;
+    status?: PipelineStatus;
+    lastRun?: Date;
+    lastRunStatus?: 'success' | 'failure' | 'warning';
+    nextRun?: Date;
+    runCount?: number;
+    successCount?: number;
+    failureCount?: number;
+    tags?: string[];
+    folder?: string;
+    owner?: string;
+    team?: string;
+}
+/** Source node config */
+export interface SourceNodeConfig {
+    connector: string;
+    connection: ConnectionConfig;
+    extraction: ExtractionConfig;
+    sampling?: SamplingConfig;
+    caching?: CachingConfig;
+}
+/** Connection config */
+export interface ConnectionConfig {
+    type: ConnectorType;
+    host?: string;
+    port?: number;
+    database?: string;
+    schema?: string;
+    username?: string;
+    password?: string;
+    connectionString?: string;
+    sslEnabled?: boolean;
+    sslConfig?: SSLConfig;
+    timeout?: number;
+    poolSize?: number;
+    options?: Record<string, unknown>;
+}
+/** Connector type */
+export type ConnectorType = 'postgresql' | 'mysql' | 'sqlserver' | 'oracle' | 'mongodb' | 'elasticsearch' | 'redis' | 'dynamodb' | 's3' | 'gcs' | 'azure-blob' | 'hdfs' | 'kafka' | 'kinesis' | 'pubsub' | 'rabbitmq' | 'rest-api' | 'graphql' | 'sftp' | 'ftp' | 'file' | 'csv' | 'json' | 'parquet' | 'avro' | 'orc' | 'xml' | 'excel' | 'salesforce' | 'hubspot' | 'stripe' | 'shopify' | 'google-sheets' | 'airtable' | 'snowflake' | 'bigquery' | 'redshift' | 'databricks' | 'custom';
+/** SSL config */
+export interface SSLConfig {
+    mode: 'disable' | 'require' | 'verify-ca' | 'verify-full';
+    certPath?: string;
+    keyPath?: string;
+    caPath?: string;
+    serverName?: string;
+}
+/** Extraction config */
+export interface ExtractionConfig {
+    method: 'table' | 'query' | 'file' | 'api' | 'stream';
+    table?: string;
+    query?: string;
+    filePath?: string;
+    filePattern?: string;
+    apiEndpoint?: string;
+    apiMethod?: 'GET' | 'POST';
+    apiHeaders?: Record<string, string>;
+    apiBody?: unknown;
+    apiPagination?: PaginationConfig;
+    streamTopic?: string;
+    streamPartitions?: number[];
+    streamOffset?: 'earliest' | 'latest' | 'timestamp';
+    streamTimestamp?: Date;
+    columns?: string[];
+    excludeColumns?: string[];
+    whereClause?: string;
+    orderBy?: SortField[];
+    limit?: number;
+}
+/** Pagination config */
+export interface PaginationConfig {
+    type: 'offset' | 'cursor' | 'page' | 'link';
+    pageSize?: number;
+    offsetParam?: string;
+    limitParam?: string;
+    cursorParam?: string;
+    cursorPath?: string;
+    pageParam?: string;
+    totalPath?: string;
+    hasMorePath?: string;
+    nextLinkPath?: string;
+}
+/** Sampling config */
+export interface SamplingConfig {
+    enabled: boolean;
+    method: 'random' | 'first' | 'stratified';
+    size: number;
+    seed?: number;
+    stratifyBy?: string;
+}
+/** Caching config */
+export interface CachingConfig {
+    enabled: boolean;
+    ttl?: number;
+    storage?: 'memory' | 'disk' | 'redis';
+    key?: string;
+}
+/** Transform node config */
+export interface TransformNodeConfig {
+    transformType: TransformType;
+    config: TransformSpecificConfig;
+}
+/** Transform type */
+export type TransformType = 'select' | 'filter' | 'rename' | 'cast' | 'derive' | 'aggregate' | 'join' | 'union' | 'lookup' | 'pivot' | 'unpivot' | 'window' | 'rank' | 'deduplicate' | 'sort' | 'limit' | 'sample' | 'split' | 'normalize' | 'denormalize' | 'flatten' | 'explode' | 'fill' | 'replace' | 'hash' | 'encrypt' | 'decrypt' | 'mask' | 'validate' | 'standardize' | 'geocode' | 'sentiment' | 'ml-predict' | 'custom';
+/** Transform specific configuration */
+export type TransformSpecificConfig = SelectTransformConfig | FilterTransformConfig | RenameTransformConfig | CastTransformConfig | DeriveTransformConfig | AggregateTransformConfig | JoinTransformConfig | UnionTransformConfig | LookupTransformConfig | PivotTransformConfig | UnpivotTransformConfig | WindowTransformConfig | RankTransformConfig | DeduplicateTransformConfig | SortTransformConfig | LimitTransformConfig | SampleTransformConfig | SplitTransformConfig | FillTransformConfig | ReplaceTransformConfig | HashTransformConfig | MaskTransformConfig | ValidateTransformConfig | StandardizeTransformConfig | CustomTransformConfig;
+/** Select transform config */
+export interface SelectTransformConfig {
+    columns: ColumnSelection[];
+}
+/** Column selection */
+export interface ColumnSelection {
+    source: string;
+    alias?: string;
+    expression?: string;
+}
+/** Filter transform config */
+export interface FilterTransformConfig {
+    conditions: FilterCondition[];
+    logic: 'and' | 'or';
+}
+/** Filter condition */
+export interface FilterCondition {
+    field: string;
+    operator: FilterOperator;
+    value: unknown;
+    caseSensitive?: boolean;
+}
+/** Filter operator */
+export type FilterOperator = 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte' | 'in' | 'notIn' | 'contains' | 'notContains' | 'startsWith' | 'endsWith' | 'matches' | 'isNull' | 'isNotNull' | 'between';
+/** Rename transform config */
+export interface RenameTransformConfig {
+    mappings: RenameMapping[];
+}
+/** Rename mapping */
+export interface RenameMapping {
+    from: string;
+    to: string;
+}
+/** Cast transform config */
+export interface CastTransformConfig {
+    casts: CastOperation[];
+}
+/** Cast operation */
+export interface CastOperation {
+    field: string;
+    targetType: DataType;
+    format?: string;
+    errorHandling?: 'fail' | 'null' | 'default';
+    defaultValue?: unknown;
+}
+/** Derive transform config */
+export interface DeriveTransformConfig {
+    derivations: Derivation[];
+}
+/** Derivation */
+export interface Derivation {
+    name: string;
+    expression: string;
+    type?: DataType;
+}
+/** Aggregate transform config */
+export interface AggregateTransformConfig {
+    groupBy: string[];
+    aggregations: Aggregation[];
+    having?: FilterCondition[];
+}
+/** Aggregation */
+export interface Aggregation {
+    field: string;
+    function: AggregateFunction;
+    alias: string;
+    distinct?: boolean;
+    filter?: FilterCondition;
+}
+/** Aggregate function */
+export type AggregateFunction = 'count' | 'countDistinct' | 'sum' | 'avg' | 'min' | 'max' | 'first' | 'last' | 'median' | 'mode' | 'stddev' | 'variance' | 'percentile' | 'collect' | 'collectSet';
+/** Join transform config */
+export interface JoinTransformConfig {
+    joinType: JoinType;
+    leftKey: string[];
+    rightKey: string[];
+    condition?: string;
+    selectColumns?: JoinColumnSelection[];
+    conflictResolution?: 'left' | 'right' | 'both' | 'coalesce';
+}
+/** Join type */
+export type JoinType = 'inner' | 'left' | 'right' | 'full' | 'cross' | 'semi' | 'anti';
+/** Join column selection */
+export interface JoinColumnSelection {
+    source: 'left' | 'right';
+    column: string;
+    alias?: string;
+}
+/** Union transform config */
+export interface UnionTransformConfig {
+    unionType: 'all' | 'distinct';
+    positionBased?: boolean;
+    mappings?: UnionMapping[];
+}
+/** Union mapping */
+export interface UnionMapping {
+    target: string;
+    sources: {
+        input: number;
+        column: string;
+    }[];
+}
+/** Lookup transform config */
+export interface LookupTransformConfig {
+    lookupSource: string;
+    lookupKey: string[];
+    sourceKey: string[];
+    returnColumns: LookupReturnColumn[];
+    matchType: 'first' | 'last' | 'all';
+    noMatchAction: 'null' | 'default' | 'reject';
+    defaultValues?: Record<string, unknown>;
+}
+/** Lookup return column */
+export interface LookupReturnColumn {
+    source: string;
+    alias?: string;
+}
+/** Pivot transform config */
+export interface PivotTransformConfig {
+    groupBy: string[];
+    pivotColumn: string;
+    valueColumn: string;
+    aggregation: AggregateFunction;
+    pivotValues?: unknown[];
+}
+/** Unpivot transform config */
+export interface UnpivotTransformConfig {
+    idColumns: string[];
+    valueColumns: string[];
+    variableName: string;
+    valueName: string;
+}
+/** Window transform config */
+export interface WindowTransformConfig {
+    windows: WindowDefinition[];
+}
+/** Window definition */
+export interface WindowDefinition {
+    name: string;
+    function: WindowFunction;
+    field?: string;
+    partitionBy?: string[];
+    orderBy?: SortField[];
+    frameType?: 'rows' | 'range';
+    frameStart?: FrameBound;
+    frameEnd?: FrameBound;
+}
+/** Window function */
+export type WindowFunction = 'row_number' | 'rank' | 'dense_rank' | 'ntile' | 'lag' | 'lead' | 'first_value' | 'last_value' | 'nth_value' | 'sum' | 'avg' | 'count' | 'min' | 'max' | 'stddev' | 'variance' | 'percent_rank' | 'cume_dist';
+/** Frame bound */
+export interface FrameBound {
+    type: 'unbounded' | 'current' | 'preceding' | 'following';
+    offset?: number;
+}
+/** Rank transform config */
+export interface RankTransformConfig {
+    partitionBy?: string[];
+    orderBy: SortField[];
+    rankType: 'rank' | 'dense_rank' | 'row_number' | 'percent_rank';
+    outputColumn: string;
+}
+/** Deduplicate transform config */
+export interface DeduplicateTransformConfig {
+    keys: string[];
+    keepStrategy: 'first' | 'last' | 'any';
+    orderBy?: SortField[];
+    windowDuration?: number;
+}
+/** Sort transform config */
+export interface SortTransformConfig {
+    sortFields: SortField[];
+    nullsPosition?: 'first' | 'last';
+}
+/** Limit transform config */
+export interface LimitTransformConfig {
+    limit: number;
+    offset?: number;
+}
+/** Sample transform config */
+export interface SampleTransformConfig {
+    method: 'random' | 'systematic' | 'stratified';
+    size: number;
+    fraction?: number;
+    seed?: number;
+    stratifyColumn?: string;
+    withReplacement?: boolean;
+}
+/** Split transform config */
+export interface SplitTransformConfig {
+    column: string;
+    delimiter?: string;
+    pattern?: string;
+    outputColumns: string[];
+    expandRows?: boolean;
+}
+/** Fill transform config */
+export interface FillTransformConfig {
+    columns: FillColumn[];
+}
+/** Fill column */
+export interface FillColumn {
+    column: string;
+    method: 'value' | 'forward' | 'backward' | 'mean' | 'median' | 'mode' | 'interpolate';
+    value?: unknown;
+    limit?: number;
+}
+/** Replace transform config */
+export interface ReplaceTransformConfig {
+    replacements: ReplacementRule[];
+}
+/** Replacement rule */
+export interface ReplacementRule {
+    column: string;
+    find: unknown;
+    replace: unknown;
+    regex?: boolean;
+    caseSensitive?: boolean;
+}
+/** Hash transform config */
+export interface HashTransformConfig {
+    columns: HashColumn[];
+}
+/** Hash column */
+export interface HashColumn {
+    source: string;
+    output: string;
+    algorithm: 'md5' | 'sha1' | 'sha256' | 'sha512' | 'murmur3';
+    salt?: string;
+}
+/** Mask transform config */
+export interface MaskTransformConfig {
+    masks: MaskRule[];
+}
+/** Mask rule */
+export interface MaskRule {
+    column: string;
+    type: 'partial' | 'full' | 'hash' | 'random' | 'custom';
+    showFirst?: number;
+    showLast?: number;
+    maskChar?: string;
+    pattern?: string;
+    replacement?: string;
+}
+/** Validate transform config */
+export interface ValidateTransformConfig {
+    validations: FieldValidation[];
+    onError: 'reject' | 'flag' | 'fix';
+    errorColumn?: string;
+}
+/** Field validation */
+export interface FieldValidation {
+    field: string;
+    rules: ValidationRuleConfig[];
+}
+/** Validation rule config */
+export interface ValidationRuleConfig {
+    type: ValidationRuleType;
+    params?: Record<string, unknown>;
+    message?: string;
+    fixAction?: 'null' | 'default' | 'truncate' | 'clamp';
+    fixValue?: unknown;
+}
+/** Standardize transform config */
+export interface StandardizeTransformConfig {
+    standardizations: StandardizationRule[];
+}
+/** Standardization rule */
+export interface StandardizationRule {
+    column: string;
+    type: StandardizationType;
+    options?: Record<string, unknown>;
+}
+/** Standardization type */
+export type StandardizationType = 'trim' | 'lowercase' | 'uppercase' | 'titlecase' | 'removeAccents' | 'removeSpecialChars' | 'normalizeWhitespace' | 'phoneNumber' | 'email' | 'address' | 'date' | 'currency' | 'unit' | 'custom';
+/** Custom transform config */
+export interface CustomTransformConfig {
+    language: 'sql' | 'python' | 'javascript' | 'scala';
+    code: string;
+    inputs?: string[];
+    outputs?: SchemaField[];
+    dependencies?: string[];
+}
+/** Destination node config */
+export interface DestinationNodeConfig {
+    connector: string;
+    connection: ConnectionConfig;
+    loading: LoadingConfig;
+    errorHandling?: ErrorHandlingConfig;
+}
+/** Loading config */
+export interface LoadingConfig {
+    method: 'insert' | 'upsert' | 'merge' | 'replace' | 'append' | 'overwrite';
+    target: TargetConfig;
+    batchSize?: number;
+    parallelism?: number;
+    preLoad?: PrePostLoadAction[];
+    postLoad?: PrePostLoadAction[];
+}
+/** Target config */
+export interface TargetConfig {
+    type: 'table' | 'file' | 'api' | 'stream';
+    table?: string;
+    schema?: string;
+    filePath?: string;
+    fileFormat?: FileFormat;
+    apiEndpoint?: string;
+    apiMethod?: 'POST' | 'PUT' | 'PATCH';
+    apiHeaders?: Record<string, string>;
+    streamTopic?: string;
+    partitionBy?: string[];
+    createIfNotExists?: boolean;
+    truncateBeforeLoad?: boolean;
+}
+/** File format */
+export interface FileFormat {
+    type: 'csv' | 'json' | 'parquet' | 'avro' | 'orc' | 'xml' | 'excel';
+    delimiter?: string;
+    quote?: string;
+    escape?: string;
+    header?: boolean;
+    encoding?: string;
+    compression?: 'none' | 'gzip' | 'snappy' | 'lz4' | 'zstd';
+    dateFormat?: string;
+    timestampFormat?: string;
+    nullValue?: string;
+    schema?: string;
+}
+/** Pre/Post load action */
+export interface PrePostLoadAction {
+    type: 'sql' | 'api' | 'script';
+    content: string;
+    parameters?: Record<string, unknown>;
+}
+/** Error handling config */
+export interface ErrorHandlingConfig {
+    maxErrors?: number;
+    errorThreshold?: number;
+    onError: 'fail' | 'skip' | 'log';
+    errorTable?: string;
+    deadLetterQueue?: string;
+}
+/** Junction node config */
+export interface JunctionNodeConfig {
+    type: 'split' | 'merge';
+}
+/** Filter node config */
+export interface FilterNodeConfig {
+    conditions: FilterCondition[];
+    logic: 'and' | 'or';
+}
+/** Aggregate node config */
+export interface AggregateNodeConfig {
+    groupBy: string[];
+    aggregations: Aggregation[];
+    having?: FilterCondition[];
+}
+/** Join node config */
+export interface JoinNodeConfig {
+    joinType: JoinType;
+    leftKey: string[];
+    rightKey: string[];
+    condition?: string;
+}
+/** Union node config */
+export interface UnionNodeConfig {
+    unionType: 'all' | 'distinct';
+}
+/** Lookup node config */
+export interface LookupNodeConfig {
+    lookupSource: string;
+    lookupKey: string[];
+    sourceKey: string[];
+    returnColumns: string[];
+}
+/** Script node config */
+export interface ScriptNodeConfig {
+    language: 'sql' | 'python' | 'javascript' | 'scala';
+    code: string;
+    timeout?: number;
+}
+/** Branch node config */
+export interface BranchNodeConfig {
+    conditions: BranchCondition[];
+    defaultBranch?: string;
+}
+/** Branch condition */
+export interface BranchCondition {
+    name: string;
+    expression: string;
+    outputHandle: string;
+}
+/** Merge node config */
+export interface MergeNodeConfig {
+    mergeType: 'union' | 'intersect' | 'except';
+    distinct?: boolean;
+}
+/** Custom node config */
+export interface CustomNodeConfig {
+    nodeType: string;
+    options: Record<string, unknown>;
+}
+/** Source connector definition */
+export interface SourceConnectorDefinition {
+    id: string;
+    name: string;
+    type: ConnectorType;
+    icon?: string;
+    category?: string;
+    description?: string;
+    configSchema: ConnectorConfigSchema;
+    authTypes?: AuthType[];
+    capabilities?: ConnectorCapabilities;
+}
+/** Connector config schema */
+export interface ConnectorConfigSchema {
+    properties: ConnectorProperty[];
+    groups?: PropertyGroup[];
+}
+/** Connector property */
+export interface ConnectorProperty {
+    name: string;
+    label: string;
+    type: 'string' | 'number' | 'boolean' | 'select' | 'password' | 'json' | 'code';
+    required?: boolean;
+    defaultValue?: unknown;
+    options?: PropertyOption[];
+    description?: string;
+    group?: string;
+    advanced?: boolean;
+    dependsOn?: PropertyDependency;
+}
+/** Property option */
+export interface PropertyOption {
+    value: unknown;
+    label: string;
+}
+/** Property group */
+export interface PropertyGroup {
+    id: string;
+    label: string;
+    collapsed?: boolean;
+}
+/** Property dependency */
+export interface PropertyDependency {
+    property: string;
+    operator: 'eq' | 'neq' | 'in';
+    value: unknown;
+}
+/** Auth type */
+export type AuthType = 'none' | 'basic' | 'apiKey' | 'bearer' | 'oauth2' | 'aws' | 'gcp' | 'azure';
+/** Connector capabilities */
+export interface ConnectorCapabilities {
+    incrementalRead?: boolean;
+    fullRefresh?: boolean;
+    cdc?: boolean;
+    streaming?: boolean;
+    preview?: boolean;
+    schemaDiscovery?: boolean;
+    pushDown?: boolean;
+}
+/** Transformation definition */
+export interface TransformationDefinition {
+    id: string;
+    name: string;
+    type: TransformType;
+    icon?: string;
+    category?: string;
+    description?: string;
+    configSchema: TransformConfigSchema;
+    inputPorts?: PortDefinition[];
+    outputPorts?: PortDefinition[];
+}
+/** Transform config schema */
+export interface TransformConfigSchema {
+    properties: TransformProperty[];
+    groups?: PropertyGroup[];
+}
+/** Transform property */
+export interface TransformProperty {
+    name: string;
+    label: string;
+    type: 'string' | 'number' | 'boolean' | 'select' | 'multiSelect' | 'field' | 'fieldList' | 'expression' | 'json' | 'code';
+    required?: boolean;
+    defaultValue?: unknown;
+    options?: PropertyOption[];
+    description?: string;
+    group?: string;
+    advanced?: boolean;
+    dependsOn?: PropertyDependency;
+}
+/** Port definition */
+export interface PortDefinition {
+    id: string;
+    label?: string;
+    maxConnections?: number;
+}
+/** Destination writer definition */
+export interface DestinationWriterDefinition {
+    id: string;
+    name: string;
+    type: ConnectorType;
+    icon?: string;
+    category?: string;
+    description?: string;
+    configSchema: ConnectorConfigSchema;
+    loadMethods?: LoadMethod[];
+    capabilities?: WriterCapabilities;
+}
+/** Load method */
+export type LoadMethod = 'insert' | 'upsert' | 'merge' | 'replace' | 'append' | 'overwrite';
+/** Writer capabilities */
+export interface WriterCapabilities {
+    batch?: boolean;
+    streaming?: boolean;
+    transactions?: boolean;
+    parallelWrite?: boolean;
+    schemaEvolution?: boolean;
+}
+/** Custom node definition */
+export interface CustomNodeDefinition {
+    id: string;
+    name: string;
+    icon?: string;
+    category?: string;
+    description?: string;
+    configSchema: TransformConfigSchema;
+    inputPorts?: PortDefinition[];
+    outputPorts?: PortDefinition[];
+    execute?: (inputs: unknown[], config: Record<string, unknown>) => unknown;
+}
+/** Data lineage */
+export interface DataLineage {
+    nodes: LineageNode[];
+    edges: LineageEdge[];
+    metadata?: LineageMetadata;
+}
+/** Lineage node */
+export interface LineageNode {
+    id: string;
+    type: 'dataset' | 'pipeline' | 'column' | 'transformation';
+    name: string;
+    qualifiedName?: string;
+    description?: string;
+    metadata?: Record<string, unknown>;
+}
+/** Lineage edge */
+export interface LineageEdge {
+    id: string;
+    source: string;
+    target: string;
+    type: 'dataFlow' | 'derivation' | 'dependency';
+    transformation?: string;
+}
+/** Lineage metadata */
+export interface LineageMetadata {
+    captureTime?: Date;
+    pipelineId?: string;
+    runId?: string;
+    version?: string;
+}
+/** Pipeline run */
+export interface PipelineRun {
+    id: string;
+    pipelineId: string;
+    pipelineVersion?: string;
+    status: PipelineStatus;
+    startTime?: Date;
+    endTime?: Date;
+    duration?: number;
+    parameters?: Record<string, unknown>;
+    nodeRuns: NodeRun[];
+    metrics?: PipelineRunMetrics;
+    logs?: RunLog[];
+    checkpoints?: Checkpoint[];
+    error?: RunError;
+}
+/** Node run */
+export interface NodeRun {
+    nodeId: string;
+    status: NodeStatus;
+    startTime?: Date;
+    endTime?: Date;
+    duration?: number;
+    metrics?: NodeMetrics;
+    error?: RunError;
+    retries?: number;
+}
+/** Pipeline run metrics */
+export interface PipelineRunMetrics {
+    totalRows?: number;
+    totalBytes?: number;
+    totalErrors?: number;
+    totalWarnings?: number;
+    nodesExecuted?: number;
+    nodesSkipped?: number;
+    nodesFailed?: number;
+}
+/** Run log */
+export interface RunLog {
+    timestamp: Date;
+    level: 'debug' | 'info' | 'warn' | 'error';
+    nodeId?: string;
+    message: string;
+    details?: Record<string, unknown>;
+}
+/** Checkpoint */
+export interface Checkpoint {
+    id: string;
+    timestamp: Date;
+    nodeId: string;
+    state: Record<string, unknown>;
+}
+/** Run error */
+export interface RunError {
+    code?: string;
+    message: string;
+    stack?: string;
+    nodeId?: string;
+    details?: Record<string, unknown>;
+}
+/** Data quality check */
+export interface DataQualityCheck {
+    id: string;
+    name: string;
+    type: QualityCheckType;
+    config: QualityCheckConfig;
+    severity: 'info' | 'warning' | 'error' | 'critical';
+    enabled: boolean;
+}
+/** Quality check type */
+export type QualityCheckType = 'completeness' | 'uniqueness' | 'validity' | 'accuracy' | 'consistency' | 'timeliness' | 'custom';
+/** Quality check config */
+export interface QualityCheckConfig {
+    column?: string;
+    expression?: string;
+    threshold?: number;
+    referenceValue?: unknown;
+    referenceDataset?: string;
+    customQuery?: string;
+}
+/** Quality result */
+export interface QualityResult {
+    checkId: string;
+    passed: boolean;
+    score: number;
+    details?: QualityDetails;
+    timestamp: Date;
+}
+/** Quality details */
+export interface QualityDetails {
+    totalRecords?: number;
+    passedRecords?: number;
+    failedRecords?: number;
+    nullCount?: number;
+    distinctCount?: number;
+    minValue?: unknown;
+    maxValue?: unknown;
+    avgValue?: number;
+    stddev?: number;
+    samples?: unknown[];
+}
+//# sourceMappingURL=data-pipeline.d.ts.map