@oronts/vendure-data-hub-plugin 0.1.0

package/shared/types/runtime-config.types.ts

@@ -0,0 +1,171 @@
+/**
+ * Runtime Configuration Types - Shared type definitions for plugin configuration
+ *
+ * These types define runtime configuration options that can be shared
+ * between frontend and backend.
+ */
+
+import type { PipelineDefinition } from './pipeline.types';
+import type { JsonObject } from './json.types';
+
+/**
+ * Runtime configuration for batch processing
+ */
+export interface BatchConfig {
+    /** Default batch size for processing (default: 50) */
+    size?: number;
+    /** Bulk operation size (default: 100) */
+    bulkSize?: number;
+    /** Maximum concurrent operations (default: 5) */
+    maxInFlight?: number;
+    /** Rate limit - requests per second (default: 10) */
+    rateLimitRps?: number;
+}
+
+/**
+ * Runtime configuration for HTTP operations
+ */
+export interface HttpConfig {
+    /** Request timeout in milliseconds (default: 30000) */
+    timeoutMs?: number;
+    /** Maximum retry attempts (default: 3) */
+    maxRetries?: number;
+    /** Initial retry delay in milliseconds (default: 1000) */
+    retryDelayMs?: number;
+    /** Maximum retry delay in milliseconds (default: 30000) */
+    retryMaxDelayMs?: number;
+    /** Enable exponential backoff (default: true) */
+    exponentialBackoff?: boolean;
+    /** Backoff multiplier (default: 2) */
+    backoffMultiplier?: number;
+}
+
+/**
+ * Circuit breaker configuration for external calls
+ */
+export interface CircuitBreakerConfig {
+    /** Enable circuit breaker (default: true) */
+    enabled?: boolean;
+    /** Failure threshold before opening circuit (default: 5) */
+    failureThreshold?: number;
+    /** Success threshold to close circuit (default: 3) */
+    successThreshold?: number;
+    /** Time in ms before attempting reset (default: 30000) */
+    resetTimeoutMs?: number;
+    /** Time window for counting failures in ms (default: 60000) */
+    failureWindowMs?: number;
+}
+
+/**
+ * Connection pool configuration
+ */
+export interface ConnectionPoolConfig {
+    /** Minimum connections (default: 1) */
+    min?: number;
+    /** Maximum connections (default: 10) */
+    max?: number;
+    /** Idle timeout in milliseconds (default: 30000) */
+    idleTimeoutMs?: number;
+    /** Acquire timeout in milliseconds (default: 10000) */
+    acquireTimeoutMs?: number;
+}
+
+/**
+ * Runtime pagination configuration
+ */
+export interface RuntimePaginationConfig {
+    /** Maximum pages to fetch from paginated APIs (default: 100) */
+    maxPages?: number;
+    /** Default page size for data extraction (default: 100) */
+    pageSize?: number;
+    /** Database page size (default: 1000) */
+    databasePageSize?: number;
+}
+
+/**
+ * Scheduler configuration for pipeline scheduling
+ */
+export interface SchedulerConfig {
+    /** Interval for checking cron schedules in milliseconds (default: 30000) */
+    checkIntervalMs?: number;
+    /** Interval for refreshing schedule cache in milliseconds (default: 60000) */
+    refreshIntervalMs?: number;
+    /** Minimum allowed interval in milliseconds - safety limit (default: 1000) */
+    minIntervalMs?: number;
+}
+
+/**
+ * Event trigger service configuration for pipeline cache management
+ */
+export interface EventTriggerServiceConfig {
+    /** Interval for refreshing pipeline cache in milliseconds (default: 60000) */
+    cacheRefreshIntervalMs?: number;
+}
+
+/**
+ * Runtime limits configuration - all configurable via plugin options
+ */
+export interface RuntimeLimitsConfig {
+    /** Batch processing configuration */
+    batch?: BatchConfig;
+    /** HTTP configuration */
+    http?: HttpConfig;
+    /** Circuit breaker configuration */
+    circuitBreaker?: CircuitBreakerConfig;
+    /** Connection pool configuration */
+    connectionPool?: ConnectionPoolConfig;
+    /** Pagination configuration */
+    pagination?: RuntimePaginationConfig;
+    /** Scheduler configuration for pipeline scheduling */
+    scheduler?: SchedulerConfig;
+    /** Event trigger service configuration */
+    eventTrigger?: EventTriggerServiceConfig;
+}
+
+/**
+ * Code-first pipeline configuration
+ */
+export interface CodeFirstPipeline {
+    /** Unique pipeline identifier */
+    code: string;
+    /** Human-readable name */
+    name: string;
+    /** Optional description */
+    description?: string;
+    /** Whether pipeline is active (enabled) */
+    enabled?: boolean;
+    /** The pipeline definition */
+    definition: PipelineDefinition;
+    /** Tags for organization */
+    tags?: string[];
+}
+
+/**
+ * Code-first secret configuration.
+ * Define secrets in code (values can reference env vars).
+ */
+export interface CodeFirstSecret {
+    /** Unique secret identifier */
+    code: string;
+    /** Provider: 'INLINE' for direct value, 'ENV' to read from environment */
+    provider: 'INLINE' | 'ENV';
+    /** Value or env variable name */
+    value: string;
+    /** Optional metadata */
+    metadata?: JsonObject;
+}
+
+/**
+ * Code-first connection configuration.
+ * Define external connections (databases, APIs, etc.) in code.
+ */
+export interface CodeFirstConnection {
+    /** Unique connection identifier */
+    code: string;
+    /** Connection type (e.g., 'postgres', 'mysql', 'rest', 's3') */
+    type: string;
+    /** Human-readable name */
+    name: string;
+    /** Connection settings - supports env var references like ${DB_HOST} */
+    settings: JsonObject;
+}

package/shared/types/schema.types.ts

@@ -0,0 +1,238 @@
+/**
+ * Schema Types - Combined schema validation, field, and definition types
+ *
+ * Typed schema system for defining complex data structures with validation,
+ * transformations, and UI hints.
+ */
+
+// ============================================================================
+// VALIDATION TYPES
+// ============================================================================
+
+export interface StringValidation {
+    minLength?: number;
+    maxLength?: number;
+    pattern?: string;           // Regex pattern
+    patternFlags?: string;      // Regex flags (i, g, m)
+    patternMessage?: string;    // Custom error message
+    format?: 'email' | 'url' | 'uuid' | 'slug' | 'phone' | 'ip' | 'ipv4' | 'ipv6' | 'hostname' | 'uri';
+    trim?: boolean;             // Auto-trim whitespace
+    lowercase?: boolean;        // Auto-lowercase
+    uppercase?: boolean;        // Auto-uppercase
+}
+
+export interface NumberValidation {
+    min?: number;
+    max?: number;
+    exclusiveMin?: number;
+    exclusiveMax?: number;
+    multipleOf?: number;        // Must be divisible by
+    precision?: number;         // Decimal places
+    positive?: boolean;
+    negative?: boolean;
+    integer?: boolean;
+}
+
+export interface ArrayValidation {
+    minItems?: number;
+    maxItems?: number;
+    uniqueItems?: boolean;      // No duplicates
+    uniqueBy?: string;          // Unique by field (for object arrays)
+}
+
+export interface ObjectValidation {
+    minProperties?: number;
+    maxProperties?: number;
+    additionalProperties?: boolean;  // Allow extra fields
+}
+
+export interface DateValidation {
+    before?: string | 'now';
+    after?: string | 'now';
+    format?: string;            // Input format for parsing
+}
+
+// ============================================================================
+// FIELD TYPES
+// ============================================================================
+
+// PRIMITIVE TYPES
+
+export type PrimitiveType =
+    | 'string'
+    | 'text'          // Long text / textarea
+    | 'number'
+    | 'integer'
+    | 'float'
+    | 'decimal'       // Exact decimal (for prices)
+    | 'boolean'
+    | 'date'          // Date only (YYYY-MM-DD)
+    | 'datetime'      // Full ISO datetime
+    | 'time'          // Time only (HH:mm:ss)
+    | 'timestamp'     // Unix timestamp
+    | 'email'
+    | 'url'
+    | 'uuid'
+    | 'slug'
+    | 'phone'
+    | 'currency'      // Currency code (USD, EUR)
+    | 'locale'        // Locale code (en-US)
+    | 'country'       // ISO country code
+    | 'json'          // Raw JSON (any structure)
+    | 'binary'        // Base64 encoded
+    | 'null';
+
+// COMPLEX TYPES
+
+export type ComplexType =
+    | 'object'        // Nested object with sub-fields
+    | 'array'         // Array with item type
+    | 'map'           // Key-value map (Record<string, T>)
+    | 'tuple'         // Fixed-length array with specific types
+    | 'union'         // One of multiple types
+    | 'enum'          // Enumeration of values
+    | 'ref';          // Reference to another schema
+
+export type FieldType = PrimitiveType | ComplexType;
+
+// FIELD DEFINITION
+
+export interface EnhancedFieldDefinition {
+    // Core
+    type: FieldType;
+    required?: boolean;
+    nullable?: boolean;
+    readonly?: boolean;
+
+    // Documentation
+    label?: string;
+    description?: string;
+    example?: unknown;
+    deprecated?: boolean;
+    deprecatedMessage?: string;
+
+    // Default & Transform
+    default?: unknown;
+    defaultExpression?: string;     // Dynamic default: "now()", "uuid()"
+    transform?: SchemaFieldTransform[];
+
+    // Validation (type-specific)
+    validation?: StringValidation | NumberValidation | ArrayValidation | ObjectValidation | DateValidation;
+
+    // For 'object' type - nested fields
+    fields?: Record<string, EnhancedFieldDefinition>;
+
+    // For 'array' type - item definition
+    items?: EnhancedFieldDefinition;
+
+    // For 'map' type - value definition (keys are always strings)
+    values?: EnhancedFieldDefinition;
+
+    // For 'tuple' type - ordered item definitions
+    elements?: EnhancedFieldDefinition[];
+
+    // For 'union' type - allowed types
+    oneOf?: EnhancedFieldDefinition[];
+
+    // For 'enum' type - allowed values
+    enum?: Array<string | number | boolean>;
+    enumLabels?: Record<string, string>;  // Human-readable labels
+
+    // For 'ref' type - reference to another schema
+    ref?: string;  // Schema code
+
+    // For 'decimal'/'currency' type
+    precision?: number;
+    scale?: number;
+    currencyField?: string;  // Field containing currency code
+
+    // Conditional
+    dependsOn?: FieldDependency[];
+
+    // UI hints
+    ui?: FieldUIHints;
+}
+
+export interface SchemaFieldTransform {
+    type: 'trim' | 'lowercase' | 'uppercase' | 'slugify' | 'normalize' | 'round' | 'floor' | 'ceil' | 'abs' | 'parseNumber' | 'parseDate' | 'parseJson' | 'stringify';
+    options?: Record<string, unknown>;
+}
+
+export interface FieldDependency {
+    field: string;
+    operator: 'eq' | 'ne' | 'in' | 'notIn' | 'exists' | 'notExists' | 'gt' | 'gte' | 'lt' | 'lte';
+    value?: unknown;
+}
+
+export interface FieldUIHints {
+    widget?: 'input' | 'textarea' | 'select' | 'multiselect' | 'checkbox' | 'radio' | 'switch' | 'datepicker' | 'colorpicker' | 'slider' | 'code' | 'json' | 'password' | 'file' | 'image' | 'rich-text';
+    placeholder?: string;
+    helpText?: string;
+    group?: string;
+    order?: number;
+    hidden?: boolean;
+    disabled?: boolean;
+    width?: 'full' | 'half' | 'third' | 'quarter';
+    rows?: number;          // For textarea
+    language?: string;      // For code editor
+}
+
+// ============================================================================
+// DEFINITION TYPES
+// ============================================================================
+
+export interface EnhancedSchemaDefinition {
+    // Meta
+    $version?: string;          // Schema version (semver)
+    $id?: string;               // Unique schema ID
+    $extends?: string;          // Inherit from another schema
+
+    // Documentation
+    label?: string;             // Human-readable entity name (e.g., "Product Variant")
+    description?: string;       // Short description of the entity
+
+    // Fields
+    fields: Record<string, EnhancedFieldDefinition>;
+
+    // Field groups for UI organization
+    groups?: SchemaFieldGroup[];
+
+    // Primary key field(s) for deduplication/updates
+    primaryKey?: string | string[];
+
+    // Indexes for lookup optimization
+    indexes?: SchemaIndex[];
+
+    // Cross-field validation rules
+    rules?: SchemaValidationRule[];
+
+    // Computed/virtual fields
+    computed?: Record<string, ComputedField>;
+}
+
+export interface SchemaFieldGroup {
+    id: string;
+    label: string;
+    description?: string;
+    collapsed?: boolean;
+    fields: string[];
+}
+
+export interface SchemaIndex {
+    fields: string[];
+    unique?: boolean;
+    sparse?: boolean;       // Only index non-null values
+}
+
+export interface SchemaValidationRule {
+    id: string;
+    expression: string;     // e.g., "endDate > startDate"
+    message: string;
+    severity?: 'error' | 'warning';
+}
+
+export interface ComputedField {
+    type: FieldType;
+    expression: string;     // e.g., "price * quantity"
+    dependencies: string[]; // Fields this depends on
+}

package/shared/types/step.types.ts

@@ -0,0 +1,290 @@
+/**
+ * Step Types
+ *
+ * Types for pipeline steps including step definitions, execution results,
+ * and routing configuration.
+ */
+
+import { JsonObject, JsonValue } from './json.types';
+
+/**
+ * Type of pipeline step.
+ *
+ * String literal union (not an enum) for JSON serialization compatibility.
+ * The backend StepType enum in src/constants/enums.ts uses the same values.
+ */
+export type StepType =
+    | 'TRIGGER'
+    | 'EXTRACT'
+    | 'TRANSFORM'
+    | 'VALIDATE'
+    | 'ENRICH'
+    | 'ROUTE'
+    | 'LOAD'
+    | 'EXPORT'
+    | 'FEED'
+    | 'SINK'
+    | 'GATE';
+
+/** Execution status of a pipeline node/step */
+export type NodeStatus =
+    | 'IDLE'
+    | 'PENDING'
+    | 'RUNNING'
+    | 'SUCCESS'
+    | 'ERROR'
+    | 'SKIPPED'
+    | 'CANCELLED';
+
+/**
+ * Drain strategies - matches src/constants/enums.ts DrainStrategy values
+ */
+export type DrainStrategy = 'BACKOFF' | 'SHED' | 'QUEUE';
+
+/**
+ * Channel strategies - matches src/constants/enums.ts ChannelStrategy values
+ */
+export type ChannelStrategy = 'EXPLICIT' | 'INHERIT' | 'MULTI';
+
+/**
+ * Validation modes - matches src/constants/enums.ts ValidationMode values
+ */
+export type ValidationModeType = 'STRICT' | 'LENIENT';
+
+// Note: RunModeValue in pipeline.types.ts is the canonical exported type with
+// identical values. This file uses it inline to avoid circular imports.
+
+/**
+ * Route condition operators - matches src/constants/enums.ts RouteConditionOperator values
+ */
+export type RouteConditionOp =
+    | 'eq' | 'ne' | 'gt' | 'lt' | 'gte' | 'lte'
+    | 'in' | 'notIn'
+    | 'contains' | 'notContains' | 'startsWith' | 'endsWith'
+    | 'matches' | 'regex' | 'exists' | 'isNull';
+
+/** Domain that a pipeline operates on */
+export type PipelineCapabilityDomain =
+    | 'CATALOG'
+    | 'CUSTOMERS'
+    | 'ORDERS'
+    | 'PROMOTIONS'
+    | 'INVENTORY'
+    | 'CUSTOM';
+
+/**
+ * Result of executing a pipeline step
+ */
+export interface StepResult {
+    /** Unique key identifying the step */
+    stepKey: string;
+    /** Final status of the step */
+    status: NodeStatus;
+    /** Number of input records */
+    inputCount: number;
+    /** Number of output records */
+    outputCount: number;
+    /** Number of records with errors */
+    errorCount: number;
+    /** Number of records dropped */
+    droppedCount: number;
+    /** Duration in milliseconds */
+    duration: number;
+    /** Errors that occurred */
+    errors?: StepError[];
+    /** Additional metadata */
+    meta?: JsonObject;
+}
+
+/**
+ * Error that occurred during step execution
+ */
+export interface StepError {
+    /** Human-readable error message */
+    message: string;
+    /** Error code for programmatic handling */
+    code?: string;
+    /** Field that caused the error */
+    field?: string;
+    /** Index of the record that caused the error */
+    recordIndex?: number;
+    /** ID of the record that caused the error */
+    recordId?: string;
+    /** Stack trace if available */
+    stack?: string;
+    /** Whether the operation can be retried */
+    retryable?: boolean;
+}
+
+/**
+ * Throughput configuration for controlling processing rate
+ */
+export interface Throughput {
+    rateLimitRps?: number;
+    concurrency?: number;
+    batchSize?: number;
+    pauseOnErrorRate?: {
+        threshold: number;
+        intervalSec: number;
+    };
+    drainStrategy?: DrainStrategy;
+}
+
+/**
+ * Definition of a pipeline step
+ */
+export interface PipelineStepDefinition {
+    /** Unique key identifying the step */
+    key: string;
+    /** Type of step */
+    type: StepType;
+    /** Code of the adapter to use */
+    adapterCode?: string;
+    /** Human-readable name */
+    name?: string;
+    /** Display label */
+    label?: string;
+    /** Description of what the step does */
+    description?: string;
+    /** Configuration for the adapter */
+    config: JsonObject;
+    /** Execution order (for sequential steps) */
+    order?: number;
+    /** Whether the step is disabled */
+    disabled?: boolean;
+    /** Whether to run in parallel with other steps */
+    parallel?: boolean;
+    /** Whether to run asynchronously */
+    async?: boolean;
+    /** Number of concurrent operations */
+    concurrency?: number;
+    /** Throughput/rate limiting configuration */
+    throughput?: Throughput;
+    /** Number of retry attempts */
+    retries?: number;
+    /** Delay between retries in milliseconds */
+    retryDelayMs?: number;
+    /** Timeout in milliseconds */
+    timeoutMs?: number;
+    /** Whether to continue pipeline on step error */
+    continueOnError?: boolean;
+    /** Condition expression to evaluate */
+    condition?: string;
+    /** Input step keys this step depends on */
+    inputs?: string[];
+    /** Output step keys this step feeds into */
+    outputs?: string[];
+}
+
+/**
+ * Edge connecting two steps in the pipeline graph
+ */
+export interface PipelineEdge {
+    /** Unique identifier for the edge */
+    id?: string;
+    /** Source step key */
+    from: string;
+    /** Target step key */
+    to: string;
+    /** Branch name (for routing steps) */
+    branch?: string;
+    /** Condition expression for conditional edges */
+    condition?: string;
+    /** Display label for the edge */
+    label?: string;
+}
+
+/**
+ * Metrics collected during step execution
+ */
+export interface StepMetrics {
+    /** Number of input records */
+    inputRecords: number;
+    /** Number of output records */
+    outputRecords: number;
+    /** Number of records with errors */
+    errorRecords: number;
+    /** Number of dropped records */
+    droppedRecords: number;
+    /** Duration in milliseconds */
+    duration: number;
+    /** Records processed per second */
+    throughput: number;
+    /** Memory used in bytes */
+    memoryUsed?: number;
+}
+
+/**
+ * Execution state of a pipeline step
+ */
+export interface StepExecution {
+    /** Unique key identifying the step */
+    stepKey: string;
+    /** Current execution status */
+    status: NodeStatus;
+    /** When execution started */
+    startedAt?: Date;
+    /** When execution completed */
+    completedAt?: Date;
+    /** Execution metrics */
+    metrics?: StepMetrics;
+    /** Errors that occurred */
+    errors?: StepError[];
+    /** Checkpoint data for resumable steps */
+    checkpoint?: JsonObject;
+}
+
+/**
+ * Step context override - per-step configuration overrides
+ */
+export interface StepContextOverride {
+    contentLanguage?: string;
+    channelStrategy?: ChannelStrategy;
+    channelIds?: string[];
+    validationMode?: ValidationModeType;
+    runMode?: 'SYNC' | 'ASYNC' | 'BATCH' | 'STREAM';
+}
+
+/**
+ * Route condition definition
+ */
+export interface RouteCondition {
+    field: string;
+    cmp: RouteConditionOp;
+    value: JsonValue;
+}
+
+/**
+ * Route branch definition
+ */
+export interface RouteBranch {
+    name: string;
+    when: RouteCondition[];
+}
+
+/**
+ * Route step configuration
+ */
+export interface RouteStepConfig {
+    branches: RouteBranch[];
+}
+
+/**
+ * Configuration for a single operator within a transform step.
+ *
+ * Intentional variant: src/services/validation/step-validation.ts defines
+ * a similar interface using `params` instead of `args` for its own validation needs.
+ */
+export interface OperatorConfig {
+    op: string;
+    args?: Record<string, unknown>;
+}
+
+/**
+ * Pipeline capabilities declaration
+ */
+export interface PipelineCapabilities {
+    writes?: PipelineCapabilityDomain[];
+    requires?: string[];
+    streamSafe?: boolean;
+}