@jackchuka/gql-ingest 2.2.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +181 -1
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +237 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1379 -0
- package/dist/index.js.map +7 -0
- package/dist/{config.d.ts → lib/config.d.ts} +3 -2
- package/dist/lib/config.d.ts.map +1 -0
- package/dist/lib/dependency-resolver.d.ts.map +1 -0
- package/dist/lib/events.d.ts +167 -0
- package/dist/lib/events.d.ts.map +1 -0
- package/dist/lib/gql-ingest.d.ts +155 -0
- package/dist/lib/gql-ingest.d.ts.map +1 -0
- package/dist/{graphql-client.d.ts → lib/graphql-client.d.ts} +8 -4
- package/dist/lib/graphql-client.d.ts.map +1 -0
- package/dist/lib/index.d.ts +8 -0
- package/dist/lib/index.d.ts.map +1 -0
- package/dist/lib/logger.d.ts +27 -0
- package/dist/lib/logger.d.ts.map +1 -0
- package/dist/lib/mapper.d.ts +49 -0
- package/dist/lib/mapper.d.ts.map +1 -0
- package/dist/{metrics.d.ts → lib/metrics.d.ts} +22 -8
- package/dist/lib/metrics.d.ts.map +1 -0
- package/dist/readers/csv.d.ts +0 -4
- package/dist/readers/csv.d.ts.map +1 -1
- package/dist/readers/index.d.ts +1 -1
- package/dist/readers/index.d.ts.map +1 -1
- package/package.json +17 -7
- package/dist/cli.d.ts +0 -2
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js +0 -237
- package/dist/config.d.ts.map +0 -1
- package/dist/config.test.d.ts +0 -2
- package/dist/config.test.d.ts.map +0 -1
- package/dist/dependency-resolver.d.ts.map +0 -1
- package/dist/dependency-resolver.test.d.ts +0 -2
- package/dist/dependency-resolver.test.d.ts.map +0 -1
- package/dist/graphql-client.d.ts.map +0 -1
- package/dist/graphql-client.test.d.ts +0 -2
- package/dist/graphql-client.test.d.ts.map +0 -1
- package/dist/mapper.d.ts +0 -31
- package/dist/mapper.d.ts.map +0 -1
- package/dist/mapper.test.d.ts +0 -2
- package/dist/mapper.test.d.ts.map +0 -1
- package/dist/metrics.d.ts.map +0 -1
- package/dist/metrics.test.d.ts +0 -2
- package/dist/metrics.test.d.ts.map +0 -1
- package/dist/readers/csv.test.d.ts +0 -2
- package/dist/readers/csv.test.d.ts.map +0 -1
- package/dist/readers/data-reader.test.d.ts +0 -2
- package/dist/readers/data-reader.test.d.ts.map +0 -1
- package/dist/readers/json.test.d.ts +0 -2
- package/dist/readers/json.test.d.ts.map +0 -1
- package/dist/readers/jsonl.test.d.ts +0 -2
- package/dist/readers/jsonl.test.d.ts.map +0 -1
- package/dist/readers/yaml.test.d.ts +0 -2
- package/dist/readers/yaml.test.d.ts.map +0 -1
- /package/dist/{dependency-resolver.d.ts → lib/dependency-resolver.d.ts} +0 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/lib/gql-ingest.ts", "../src/lib/graphql-client.ts", "../src/lib/config.ts", "../src/lib/logger.ts", "../src/lib/mapper.ts", "../src/readers/data-reader.ts", "../src/readers/csv.ts", "../src/readers/json.ts", "../src/readers/yaml.ts", "../src/readers/jsonl.ts", "../src/readers/index.ts", "../src/lib/metrics.ts", "../src/lib/dependency-resolver.ts", "../src/lib/events.ts"],
|
|
4
|
+
"sourcesContent": ["import { EventEmitter } from \"events\";\nimport { GraphQLClientWrapper } from \"./graphql-client\";\nimport { DataMapper } from \"./mapper\";\nimport { MetricsCollector, ProcessingMetrics } from \"./metrics\";\nimport { DependencyResolver } from \"./dependency-resolver\";\nimport { loadConfig, getEntityConfig, getRetryConfig, ProcessingConfig } from \"./config\";\nimport { Logger, noopLogger } from \"./logger\";\nimport { basename } from \"path\";\nimport {\n GQLIngestEventMap,\n EventOptions,\n DEFAULT_EVENT_OPTIONS,\n StartedEventPayload,\n ProgressEventPayload,\n CancelledEventPayload,\n FinishedEventPayload,\n ErroredEventPayload,\n} from \"./events\";\n\n/**\n * Options for initializing GQLIngest client\n */\nexport interface GQLIngestOptions {\n /** GraphQL endpoint URL */\n endpoint: string;\n /** Optional headers to include in GraphQL requests */\n headers?: Record<string, string>;\n /** Logger instance. Defaults to silent no-op logger. */\n logger?: Logger;\n /** Override data format detection (csv, json, yaml, jsonl) */\n formatOverride?: string;\n /** Event emission options */\n eventOptions?: EventOptions;\n}\n\n/**\n * Options for ingesting data\n */\nexport interface IngestOptions {\n /** Comma-separated list or array of specific entities to process */\n entities?: string[] | string;\n /** Override data format detection for this operation */\n format?: string;\n /** AbortSignal for external cancellation */\n signal?: AbortSignal;\n}\n\n/**\n * Result of an ingestion operation\n */\nexport interface IngestResult {\n /** Processing metrics */\n metrics: ProcessingMetrics;\n /** Whether the operation was successful */\n success: boolean;\n /** Any errors that occurred */\n errors?: string[];\n /** Whether the operation was cancelled */\n cancelled?: boolean;\n}\n\n/**\n * Main class for programmatic access to gql-ingest functionality.\n * Extends EventEmitter for progress monitoring and cancellation support.\n */\nexport class GQLIngest extends EventEmitter {\n private endpoint: string;\n private headers: Record<string, string>;\n private logger: Logger;\n private formatOverride?: string;\n private metrics: MetricsCollector;\n private client: GraphQLClientWrapper;\n private mapper: DataMapper;\n private eventOptions: Required<EventOptions>;\n\n // Cancellation state\n private abortController: AbortController | null = null;\n private isProcessing = false;\n private startTime = 0;\n private progressIntervalId: ReturnType<typeof setInterval> | null = null;\n\n // Processing state for progress tracking\n private currentWave = 0;\n private totalWaves = 0;\n private entitiesCompleted = 0;\n private totalEntities = 0;\n private currentEntity: string | undefined;\n\n constructor(options: GQLIngestOptions) {\n super();\n this.endpoint = options.endpoint;\n this.headers = options.headers || {};\n this.logger = options.logger ?? noopLogger;\n this.formatOverride = options.formatOverride;\n this.eventOptions = { ...DEFAULT_EVENT_OPTIONS, ...options.eventOptions };\n\n // Initialize components\n this.metrics = new MetricsCollector();\n this.client = new GraphQLClientWrapper(this.endpoint, this.headers, this.metrics, this.logger);\n this.mapper = new DataMapper(\n this.client,\n process.cwd(),\n this.metrics,\n this.logger,\n this.formatOverride,\n );\n }\n\n /**\n * Cancel the current ingestion process\n * @param reason Optional reason for cancellation\n */\n cancel(reason = \"User requested cancellation\"): void {\n if (this.abortController && this.isProcessing) {\n this.abortController.abort(reason);\n }\n }\n\n /**\n * Check if processing is currently in progress\n */\n get processing(): boolean {\n return this.isProcessing;\n }\n\n /**\n * Safely emit an event, catching any errors from listeners\n */\n private safeEmit<K extends keyof GQLIngestEventMap>(\n event: K,\n payload: GQLIngestEventMap[K],\n ): boolean {\n try {\n return this.emit(event, payload);\n } catch (error) {\n this.logger.error(`Error in event listener for '${String(event)}':`, error);\n return false;\n }\n }\n\n /**\n * Start the progress interval timer\n */\n private startProgressInterval(): void {\n if (!this.eventOptions.emitProgressEvents) return;\n\n this.progressIntervalId = setInterval(() => {\n this.emitProgressEvent();\n }, this.eventOptions.progressInterval);\n }\n\n /**\n * Stop the progress interval timer\n */\n private stopProgressInterval(): void {\n if (this.progressIntervalId) {\n clearInterval(this.progressIntervalId);\n this.progressIntervalId = null;\n }\n }\n\n /**\n * Emit a progress event with current state\n */\n private emitProgressEvent(): void {\n const metrics = this.metrics.getMetrics();\n const payload: ProgressEventPayload = {\n currentWave: this.currentWave,\n totalWaves: this.totalWaves,\n entitiesCompleted: this.entitiesCompleted,\n totalEntities: this.totalEntities,\n rowsProcessed: metrics.totalRows,\n successfulRows: metrics.successfulOperations,\n failedRows: metrics.failedOperations,\n progressPercent:\n this.totalEntities > 0 ? (this.entitiesCompleted / this.totalEntities) * 100 : 0,\n elapsedMs: Date.now() - this.startTime,\n };\n this.safeEmit(\"progress\", payload);\n }\n\n /**\n * Handle cancellation and emit event\n */\n private handleCancellation(reason: string): IngestResult {\n this.stopProgressInterval();\n\n const payload: CancelledEventPayload = {\n reason: reason || \"Cancelled\",\n metrics: this.metrics.getMetrics(),\n currentEntity: this.currentEntity,\n elapsedMs: Date.now() - this.startTime,\n };\n this.safeEmit(\"cancelled\", payload);\n\n return {\n metrics: this.metrics.getMetrics(),\n success: false,\n cancelled: true,\n errors: [`Operation cancelled: ${reason}`],\n };\n }\n\n /**\n * Combine multiple AbortSignals into one\n */\n private combineSignals(...signals: AbortSignal[]): AbortSignal {\n const controller = new AbortController();\n\n for (const signal of signals) {\n if (signal.aborted) {\n controller.abort(signal.reason);\n break;\n }\n signal.addEventListener(\"abort\", () => controller.abort(signal.reason), {\n once: true,\n });\n }\n\n return controller.signal;\n }\n\n /**\n * Ingest data from a configuration directory\n * @param configPath Path to configuration directory (containing data/, graphql/, mappings/ subdirectories)\n * @param options Optional ingestion options\n * @returns Promise with ingestion result\n */\n async ingest(configPath: string, options?: IngestOptions): Promise<IngestResult> {\n const errors: string[] = [];\n\n // Setup cancellation\n this.abortController = new AbortController();\n const signal = options?.signal\n ? this.combineSignals(options.signal, this.abortController.signal)\n : this.abortController.signal;\n\n this.isProcessing = true;\n this.startTime = Date.now();\n this.entitiesCompleted = 0;\n this.currentWave = 0;\n this.currentEntity = undefined;\n\n try {\n // Check for pre-cancelled signal\n if (signal.aborted) {\n return this.handleCancellation(signal.reason);\n }\n\n // Reset metrics for new operation\n this.metrics = new MetricsCollector();\n\n // Update client and mapper with new metrics and logger\n this.client = new GraphQLClientWrapper(\n this.endpoint,\n this.headers,\n this.metrics,\n this.logger,\n );\n\n this.mapper = new DataMapper(\n this.client,\n process.cwd(),\n this.metrics,\n this.logger,\n options?.format ?? this.formatOverride,\n );\n\n // Load configuration\n const config = loadConfig(configPath, this.logger);\n\n // Parse entities filter if provided\n let entityFilter: string[] | undefined;\n if (options?.entities) {\n if (typeof options.entities === \"string\") {\n entityFilter = options.entities.split(\",\").map((e: string) => e.trim());\n } else {\n entityFilter = options.entities;\n }\n }\n\n // Discover all mapping files dynamically\n const mappingPaths = this.mapper.discoverMappings(configPath, entityFilter);\n\n if (mappingPaths.length === 0) {\n const filterMsg = entityFilter ? ` matching entities: ${entityFilter.join(\", \")}` : \"\";\n const warning = `No mapping files found in ${configPath}/mappings${filterMsg}`;\n this.logger.warn(warning);\n return {\n metrics: this.metrics.getMetrics(),\n success: false,\n errors: [warning],\n };\n }\n\n // Extract entity names from mapping paths\n const entityNames = mappingPaths.map((path) => basename(path, \".json\"));\n this.totalEntities = entityNames.length;\n\n // Filter dependencies to only include those relevant to selected entities\n const relevantDependencies: Record<string, string[]> = {};\n if (config.entityDependencies) {\n for (const entity of entityNames) {\n if (config.entityDependencies[entity]) {\n relevantDependencies[entity] = config.entityDependencies[entity];\n }\n }\n }\n\n // Setup dependency resolver with filtered dependencies\n const resolver = new DependencyResolver(\n entityNames,\n relevantDependencies,\n !!entityFilter, // Allow partial resolution when using --entities\n );\n\n // Validate dependencies\n const validationErrors = resolver.validateDependencies();\n if (validationErrors.length > 0) {\n if (entityFilter) {\n // When using entities filter, show warnings instead of errors\n this.logger.warn(\"\\n\u26A0\uFE0F Warning: Dependency validation issues:\");\n validationErrors.forEach((error) => this.logger.warn(` - ${error}`));\n this.logger.warn(\"This may cause errors if the dependent data doesn't already exist.\\n\");\n } else {\n // Strict validation when processing all entities\n this.logger.error(\"Dependency validation errors:\");\n validationErrors.forEach((error) => {\n this.logger.error(` - ${error}`);\n errors.push(error);\n });\n return {\n metrics: this.metrics.getMetrics(),\n success: false,\n errors,\n };\n }\n }\n\n const waves = resolver.resolveExecutionOrder();\n this.totalWaves = waves.length;\n\n // Emit started event\n const startedPayload: StartedEventPayload = {\n configPath,\n totalEntities: entityNames.length,\n entityNames,\n totalWaves: waves.length,\n startTime: this.startTime,\n };\n this.safeEmit(\"started\", startedPayload);\n\n // Start progress interval\n this.startProgressInterval();\n\n // Process entities with abort checking\n await this.processEntitiesInWaves(\n mappingPaths,\n resolver,\n this.mapper,\n config,\n this.logger,\n signal,\n );\n\n // Check if cancelled during processing\n if (signal.aborted) {\n return this.handleCancellation(signal.reason);\n }\n\n this.metrics.finishProcessing();\n this.stopProgressInterval();\n\n const finalMetrics = this.metrics.getMetrics();\n const allSuccessful = finalMetrics.failedOperations === 0;\n\n // Emit finished event\n const finishedPayload: FinishedEventPayload = {\n metrics: finalMetrics,\n durationMs: Date.now() - this.startTime,\n allSuccessful,\n };\n this.safeEmit(\"finished\", finishedPayload);\n\n return {\n metrics: finalMetrics,\n success: true,\n };\n } catch (error) {\n this.stopProgressInterval();\n\n if (signal.aborted) {\n return this.handleCancellation(signal.reason);\n }\n\n const errorMessage = error instanceof Error ? error.message : String(error);\n this.logger.error(`Error: ${errorMessage}`);\n errors.push(errorMessage);\n\n // Emit errored event\n const erroredPayload: ErroredEventPayload = {\n error: error instanceof Error ? error : new Error(String(error)),\n metrics: this.metrics.getMetrics(),\n currentEntity: this.currentEntity,\n elapsedMs: Date.now() - this.startTime,\n };\n this.safeEmit(\"errored\", erroredPayload);\n\n return {\n metrics: this.metrics.getMetrics(),\n success: false,\n errors,\n };\n } finally {\n this.isProcessing = false;\n this.abortController = null;\n this.stopProgressInterval();\n }\n }\n\n /**\n * Ingest specific entities from a configuration directory\n * @param configPath Path to configuration directory\n * @param entities Array of entity names to process\n * @returns Promise with ingestion result\n */\n async ingestEntities(configPath: string, entities: string[]): Promise<IngestResult> {\n return this.ingest(configPath, { entities });\n }\n\n /**\n * Get current processing metrics\n * @returns Current metrics\n */\n getMetrics(): ProcessingMetrics {\n return this.metrics.getMetrics();\n }\n\n /**\n * Get a summary of the current metrics\n * @returns Formatted summary string\n */\n getMetricsSummary(): string {\n return this.metrics.generateSummary();\n }\n\n /**\n * Get the GraphQL client for advanced usage\n * @returns GraphQL client wrapper\n */\n getClient(): GraphQLClientWrapper {\n return this.client;\n }\n\n /**\n * Get the data mapper for advanced usage\n * @returns Data mapper\n */\n getMapper(): DataMapper {\n return this.mapper;\n }\n\n /**\n * Set the logger instance\n * @param logger Logger instance to use\n */\n setLogger(logger: Logger): void {\n this.logger = logger;\n this.client = new GraphQLClientWrapper(this.endpoint, this.headers, this.metrics, logger);\n this.mapper = new DataMapper(\n this.client,\n process.cwd(),\n this.metrics,\n logger,\n this.formatOverride,\n );\n }\n\n /**\n * Update headers for GraphQL requests\n * @param headers New headers to use\n */\n setHeaders(headers: Record<string, string>): void {\n this.headers = headers;\n this.client = new GraphQLClientWrapper(this.endpoint, headers, this.metrics, this.logger);\n this.mapper = new DataMapper(\n this.client,\n process.cwd(),\n this.metrics,\n this.logger,\n this.formatOverride,\n );\n }\n\n /**\n * Process entities in dependency-aware waves with abort support\n */\n private async processEntitiesInWaves(\n mappingPaths: string[],\n resolver: DependencyResolver,\n mapper: DataMapper,\n config: ProcessingConfig,\n logger: Logger,\n signal?: AbortSignal,\n ): Promise<void> {\n const waves = resolver.resolveExecutionOrder();\n const pathMap = new Map(mappingPaths.map((path) => [basename(path, \".json\"), path]));\n\n logger.info(`Processing ${waves.length} dependency waves...`);\n\n for (const wave of waves) {\n // Check for cancellation before each wave\n if (signal?.aborted) {\n return;\n }\n\n this.currentWave = wave.wave;\n logger.info(`Wave ${wave.wave + 1}: Processing entities [${wave.entities.join(\", \")}]`);\n\n // Process entities in controlled batches based on entityConcurrency\n const entityConcurrency = config.parallelProcessing.entityConcurrency;\n const chunks = this.chunkArray(wave.entities, entityConcurrency);\n\n for (const chunk of chunks) {\n // Check for cancellation before each chunk\n if (signal?.aborted) {\n return;\n }\n\n const entityPromises = chunk.map(async (entityName) => {\n // Check for cancellation before processing each entity\n if (signal?.aborted) {\n return;\n }\n\n const configPath = pathMap.get(entityName);\n if (configPath) {\n this.currentEntity = entityName;\n try {\n const entityConfig = getEntityConfig(entityName, config, logger);\n const retryConfig = getRetryConfig(entityName, config);\n\n // Process entity with event callbacks\n await mapper.processEntityWithEvents(configPath, entityConfig, retryConfig, signal, {\n onEntityStart: (payload) =>\n this.safeEmit(\"entityStart\", {\n ...payload,\n waveIndex: wave.wave,\n }),\n onEntityComplete: (payload) => {\n this.entitiesCompleted++;\n this.safeEmit(\"entityComplete\", payload);\n },\n onRowSuccess: this.eventOptions.emitRowEvents\n ? (payload) => this.safeEmit(\"rowSuccess\", payload)\n : undefined,\n onRowFailure: this.eventOptions.emitRowEvents\n ? (payload) => this.safeEmit(\"rowFailure\", payload)\n : undefined,\n });\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n logger.warn(`Warning: Could not process ${configPath}: ${message}`);\n }\n }\n });\n\n await Promise.allSettled(entityPromises);\n }\n }\n }\n\n /**\n * Utility function to chunk array into smaller arrays\n */\n private chunkArray<T>(array: T[], chunkSize: number): T[][] {\n if (chunkSize <= 0) return [array];\n const chunks: T[][] = [];\n for (let i = 0; i < array.length; i += chunkSize) {\n chunks.push(array.slice(i, i + chunkSize));\n }\n return chunks;\n }\n}\n", "import { GraphQLClient } from \"graphql-request\";\nimport { MetricsCollector } from \"./metrics\";\nimport { DEFAULT_RETRY_CONFIG, RetryConfig } from \"./config\";\nimport { Logger, noopLogger } from \"./logger\";\n\nexport class GraphQLClientWrapper {\n private client: GraphQLClient;\n private metrics?: MetricsCollector;\n private logger: Logger;\n\n constructor(\n endpoint: string,\n headers?: Record<string, string>,\n metrics?: MetricsCollector,\n logger: Logger = noopLogger,\n ) {\n this.client = new GraphQLClient(endpoint, {\n headers: headers || {},\n });\n this.metrics = metrics;\n this.logger = logger;\n }\n\n async executeMutation(\n mutation: string,\n variables: Record<string, any>,\n retryConfig?: RetryConfig,\n signal?: AbortSignal,\n ): Promise<any> {\n const config = retryConfig || DEFAULT_RETRY_CONFIG;\n\n let lastError: any;\n const totalStartTime = Date.now();\n\n for (let attempt = 0; attempt < config.maxAttempts; attempt++) {\n // Check for abort before each attempt\n if (signal?.aborted) {\n throw new Error(`Request aborted: ${signal.reason || \"cancelled\"}`);\n }\n\n const attemptStartTime = Date.now();\n\n try {\n const result = await this.client.request(mutation, variables);\n\n if (this.metrics) {\n const duration = Date.now() - attemptStartTime;\n this.metrics.recordRequestDuration(duration);\n if (attempt > 0) {\n this.metrics.recordRetrySuccess(attempt);\n }\n }\n\n const totalDuration = Date.now() - totalStartTime;\n const retryInfo = attempt > 0 ? ` (succeeded on attempt ${attempt + 1})` : \"\";\n this.logger.debug(`\u2713 GraphQL request completed in ${totalDuration}ms${retryInfo}`, result);\n\n return result;\n } catch (error: any) {\n lastError = error;\n const duration = Date.now() - attemptStartTime;\n\n if (this.metrics) {\n this.metrics.recordRequestDuration(duration);\n }\n\n // Check if this is the last attempt\n if (attempt === config.maxAttempts - 1) {\n if (this.metrics && attempt > 0) {\n this.metrics.recordRetryFailure(attempt);\n }\n break;\n }\n\n // Check if error is retryable\n if (!this.isRetryableError(error, config)) {\n this.logger.error(\n `\u2717 GraphQL request failed with non-retryable error in ${duration}ms`,\n error,\n );\n throw error;\n }\n\n // Calculate delay\n const delay = this.calculateDelay(attempt, config);\n this.logger.debug(\n `\u23F3 GraphQL request failed (attempt ${attempt + 1}/${config.maxAttempts}), retrying in ${delay}ms...`,\n );\n\n // Wait before retry (interruptible by abort signal)\n await this.sleepWithSignal(delay, signal);\n }\n }\n\n // All retries exhausted\n const totalDuration = Date.now() - totalStartTime;\n this.logger.error(\n `\u2717 GraphQL request failed after ${config.maxAttempts} attempts in ${totalDuration}ms`,\n lastError,\n );\n\n throw lastError;\n }\n\n private isRetryableError(error: any, config: RetryConfig): boolean {\n // Network errors (no response)\n if (!error.response) {\n return true;\n }\n\n // Check HTTP status codes\n const status = error.response.status;\n return config.retryableStatusCodes.includes(status);\n }\n\n private calculateDelay(attempt: number, config: RetryConfig): number {\n if (!config.exponentialBackoff) {\n return config.baseDelay;\n }\n\n const exponentialDelay = config.baseDelay * Math.pow(2, attempt);\n const cappedDelay = Math.min(exponentialDelay, config.maxDelay);\n\n // Add jitter (\u00B120% randomization)\n const jitter = cappedDelay * 0.2 * (Math.random() - 0.5);\n return Math.max(0, cappedDelay + jitter);\n }\n\n /**\n * Sleep that can be interrupted by abort signal\n */\n private sleepWithSignal(ms: number, signal?: AbortSignal): Promise<void> {\n return new Promise((resolve, reject) => {\n if (signal?.aborted) {\n reject(new Error(`Request aborted: ${signal.reason || \"cancelled\"}`));\n return;\n }\n\n const timeoutId = setTimeout(resolve, ms);\n\n if (signal) {\n const abortHandler = () => {\n clearTimeout(timeoutId);\n reject(new Error(`Request aborted: ${signal.reason || \"cancelled\"}`));\n };\n signal.addEventListener(\"abort\", abortHandler, { once: true });\n }\n });\n }\n\n setHeaders(headers: Record<string, string>) {\n this.client.setHeaders(headers);\n }\n}\n", "import fs from \"fs\";\nimport path from \"path\";\nimport * as yaml from \"js-yaml\";\nimport { Logger, noopLogger } from \"./logger\";\n\nexport interface ParallelProcessingConfig {\n concurrency: number;\n entityConcurrency: number;\n preserveRowOrder: boolean;\n}\n\nexport interface RetryConfig {\n maxAttempts: number;\n baseDelay: number;\n maxDelay: number;\n exponentialBackoff: boolean;\n retryableStatusCodes: number[];\n}\n\nexport interface EntityConfig {\n concurrency?: number;\n preserveRowOrder?: boolean;\n retry?: Partial<RetryConfig>;\n}\n\nexport interface ProcessingConfig {\n retry: RetryConfig;\n parallelProcessing: ParallelProcessingConfig;\n entityConfig: Record<string, EntityConfig>;\n entityDependencies: Record<string, string[]>;\n}\n\nexport interface FullConfig extends ProcessingConfig {\n // Future: additional config sections can be added here\n}\n\nexport const DEFAULT_RETRY_CONFIG: RetryConfig = {\n maxAttempts: 3,\n baseDelay: 1000,\n maxDelay: 30000,\n exponentialBackoff: true,\n retryableStatusCodes: [408, 429, 500, 502, 503, 504],\n};\n\nexport const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig = {\n concurrency: 1,\n entityConcurrency: 1,\n preserveRowOrder: true,\n};\n\nexport const DEFAULT_CONFIG: ProcessingConfig = {\n retry: DEFAULT_RETRY_CONFIG,\n parallelProcessing: DEFAULT_PARALLEL_CONFIG,\n entityConfig: {},\n entityDependencies: {},\n};\n\nexport function loadConfig(configDir: string, logger: Logger = noopLogger): ProcessingConfig {\n const configPath = path.join(configDir, \"config.yaml\");\n\n try {\n if (!fs.existsSync(configPath)) {\n logger.info(\"No config.yaml found, using default sequential processing\");\n return DEFAULT_CONFIG;\n }\n\n const fileContents = fs.readFileSync(configPath, \"utf8\");\n const yamlConfig = yaml.load(fileContents) as Partial<FullConfig>;\n\n return mergeWithDefaults(yamlConfig);\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n logger.warn(`Warning: Failed to parse config.yaml: ${errorMessage}. Using defaults.`);\n return DEFAULT_CONFIG;\n }\n}\n\nfunction mergeWithDefaults(yamlConfig: Partial<FullConfig>): ProcessingConfig {\n return {\n retry: {\n ...DEFAULT_RETRY_CONFIG,\n ...yamlConfig.retry,\n },\n parallelProcessing: {\n ...DEFAULT_PARALLEL_CONFIG,\n ...yamlConfig.parallelProcessing,\n },\n entityConfig: yamlConfig.entityConfig || {},\n entityDependencies: yamlConfig.entityDependencies || {},\n };\n}\n\nexport function getEntityConfig(\n entityName: string,\n globalConfig: ProcessingConfig,\n logger: Logger = noopLogger,\n): ParallelProcessingConfig {\n const entityOverrides = globalConfig.entityConfig[entityName] || {};\n\n let finalConfig = {\n ...globalConfig.parallelProcessing,\n ...entityOverrides,\n };\n\n // Apply constraint: preserveRowOrder forces concurrency = 1\n if (finalConfig.preserveRowOrder && finalConfig.concurrency > 1) {\n logger.warn(\n `Entity '${entityName}': preserveRowOrder=true forces concurrency=1 (was ${finalConfig.concurrency})`,\n );\n finalConfig.concurrency = 1;\n }\n\n return finalConfig;\n}\n\nexport function getRetryConfig(entityName: string, globalConfig: ProcessingConfig): RetryConfig {\n const entityOverrides = globalConfig.entityConfig[entityName]?.retry || {};\n\n return {\n ...globalConfig.retry,\n ...entityOverrides,\n };\n}\n", "/**\n * Logger interface for configurable logging\n *\n * Libraries should be quiet by default unless explicitly configured.\n */\nexport interface Logger {\n debug: (message: string, ...args: unknown[]) => void;\n info: (message: string, ...args: unknown[]) => void;\n warn: (message: string, ...args: unknown[]) => void;\n error: (message: string, ...args: unknown[]) => void;\n}\n\n/**\n * No-op logger that silently discards all log messages.\n * Used as the default logger to keep the library quiet.\n */\nexport const noopLogger: Logger = {\n debug: () => {},\n info: () => {},\n warn: () => {},\n error: () => {},\n};\n\n/**\n * Creates a console logger with [gql-ingest] prefix.\n * Used when verbose mode is enabled.\n */\nexport function createConsoleLogger(): Logger {\n return {\n debug: (msg, ...args) => console.debug(`[gql-ingest] ${msg}`, ...args),\n info: (msg, ...args) => console.info(`[gql-ingest] ${msg}`, ...args),\n warn: (msg, ...args) => console.warn(`[gql-ingest] ${msg}`, ...args),\n error: (msg, ...args) => console.error(`[gql-ingest] ${msg}`, ...args),\n };\n}\n\n/**\n * Creates a default logger based on the verbose flag.\n * Returns no-op logger by default, console logger when verbose is true.\n */\nexport function createDefaultLogger(verbose = false): Logger {\n return verbose ? createConsoleLogger() : noopLogger;\n}\n", "import fs from \"fs\";\nimport path from \"path\";\nimport { parse, DocumentNode, VariableDefinitionNode } from \"graphql\";\nimport { DataReaderFactory, DataRow } from \"../readers\";\nimport { GraphQLClientWrapper } from \"./graphql-client\";\nimport { MetricsCollector } from \"./metrics\";\nimport { ParallelProcessingConfig, RetryConfig } from \"./config\";\nimport { Logger, noopLogger } from \"./logger\";\nimport {\n EntityStartEventPayload,\n EntityCompleteEventPayload,\n RowSuccessEventPayload,\n RowFailureEventPayload,\n} from \"./events\";\n\nexport interface MappingConfig {\n // Legacy CSV support\n csvFile?: string;\n // New flexible data file support\n dataFile?: string;\n dataFormat?: string;\n graphqlFile: string;\n mapping: Record<string, unknown>;\n}\n\n/**\n * Callbacks for entity processing events\n */\nexport interface EntityProcessingCallbacks {\n onEntityStart?: (payload: Omit<EntityStartEventPayload, \"waveIndex\">) => void;\n onEntityComplete?: (payload: EntityCompleteEventPayload) => void;\n onRowSuccess?: (payload: RowSuccessEventPayload) => void;\n onRowFailure?: (payload: RowFailureEventPayload) => void;\n}\n\nexport class DataMapper {\n private client: GraphQLClientWrapper;\n private basePath: string;\n private metrics: MetricsCollector;\n private logger: Logger;\n private formatOverride?: string;\n\n constructor(\n client: GraphQLClientWrapper,\n basePath: string = process.cwd(),\n metrics?: MetricsCollector,\n logger: Logger = noopLogger,\n formatOverride?: string,\n ) {\n this.client = client;\n this.basePath = basePath;\n this.metrics = metrics || new MetricsCollector();\n this.logger = logger;\n this.formatOverride = formatOverride;\n }\n\n discoverMappings(configDir: string, entityFilter?: string[]): string[] {\n const mappingsPath = path.resolve(this.basePath, configDir, \"mappings\");\n\n try {\n const files = fs.readdirSync(mappingsPath);\n let jsonFiles = files.filter((file) => file.endsWith(\".json\"));\n\n // Apply entity filter if provided\n if (entityFilter && entityFilter.length > 0) {\n const requestedEntities = new Set(entityFilter);\n const foundEntities = new Set<string>();\n\n jsonFiles = jsonFiles.filter((file) => {\n const entityName = path.basename(file, \".json\");\n if (requestedEntities.has(entityName)) {\n foundEntities.add(entityName);\n return true;\n }\n return false;\n });\n\n // Check for requested entities that were not found\n const notFound = entityFilter.filter((e) => !foundEntities.has(e));\n if (notFound.length > 0) {\n this.logger.warn(\n `Warning: The following entities were not found in mappings: ${notFound.join(\", \")}`,\n );\n }\n }\n\n jsonFiles.sort(); // Alphabetical order for consistent processing\n\n this.logger.info(`Discovered ${jsonFiles.length} mapping files: ${jsonFiles.join(\", \")}`);\n return jsonFiles.map((file) => path.join(configDir, \"mappings\", file));\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n this.logger.error(`Error reading mappings directory ${mappingsPath}: ${message}`);\n return [];\n }\n }\n\n /**\n * Process an entity (backward-compatible method)\n */\n async processEntity(\n configPath: string,\n parallelConfig?: ParallelProcessingConfig,\n retryConfig?: RetryConfig,\n ): Promise<void> {\n return this.processEntityWithEvents(configPath, parallelConfig, retryConfig);\n }\n\n /**\n * Process an entity with event callbacks and abort support\n */\n async processEntityWithEvents(\n configPath: string,\n parallelConfig?: ParallelProcessingConfig,\n retryConfig?: RetryConfig,\n signal?: AbortSignal,\n callbacks?: EntityProcessingCallbacks,\n ): Promise<void> {\n const entityName = path.basename(configPath, \".json\");\n const entityStartTime = Date.now();\n\n this.logger.info(`Processing entity: ${configPath}`);\n this.metrics.startEntityProcessing(entityName);\n\n // Read mapping configuration\n const configFullPath = path.resolve(this.basePath, configPath);\n const config: MappingConfig = JSON.parse(fs.readFileSync(configFullPath, \"utf8\"));\n\n // Extract config directory (parent of mappings directory)\n const configDir = path.dirname(path.dirname(configFullPath));\n\n // Determine data file path (support both legacy csvFile and new dataFile)\n const dataFile = config.dataFile || config.csvFile;\n if (!dataFile) {\n throw new Error(`No data file specified in mapping config: ${configPath}`);\n }\n\n const dataPath = path.resolve(configDir, dataFile);\n\n // Get appropriate reader (prioritize CLI format override, then config format)\n const format = this.formatOverride || config.dataFormat;\n const reader = DataReaderFactory.getReader(dataPath, format);\n const data = await reader.readFile(dataPath);\n\n // Read GraphQL mutation (relative to config directory)\n const graphqlPath = path.resolve(configDir, config.graphqlFile);\n const mutation = fs.readFileSync(graphqlPath, \"utf8\");\n\n // Emit entityStart event\n callbacks?.onEntityStart?.({\n entityName,\n mappingPath: configPath,\n totalRows: data.length,\n });\n\n // Process rows with optional parallelization\n if (parallelConfig && parallelConfig.concurrency > 1) {\n await this.processRowsConcurrentlyWithEvents(\n data,\n mutation,\n config.mapping,\n entityName,\n parallelConfig,\n retryConfig,\n signal,\n callbacks,\n );\n } else {\n await this.processRowsSequentiallyWithEvents(\n data,\n mutation,\n config.mapping,\n entityName,\n retryConfig,\n signal,\n callbacks,\n );\n }\n\n this.metrics.finishEntityProcessing(entityName);\n\n // Emit entityComplete event - convert internal metrics to EntityMetrics format\n const internalMetrics = this.metrics.getEntityMetrics(entityName);\n const duration = Date.now() - entityStartTime;\n const entityMetrics = internalMetrics\n ? {\n rowsProcessed: internalMetrics.successCount + internalMetrics.failureCount,\n successfulRows: internalMetrics.successCount,\n failedRows: internalMetrics.failureCount,\n duration,\n }\n : {\n rowsProcessed: 0,\n successfulRows: 0,\n failedRows: 0,\n duration,\n };\n\n callbacks?.onEntityComplete?.({\n entityName,\n metrics: entityMetrics,\n success: entityMetrics.failedRows === 0,\n durationMs: duration,\n });\n }\n\n private async processRowsSequentiallyWithEvents(\n data: DataRow[],\n mutation: string,\n mapping: Record<string, unknown>,\n entityName: string,\n retryConfig?: RetryConfig,\n signal?: AbortSignal,\n callbacks?: EntityProcessingCallbacks,\n ): Promise<void> {\n const totalRows = data.length;\n const variableTypes = this.extractVariableTypes(mutation);\n\n for (let i = 0; i < data.length; i++) {\n // Check for abort signal\n if (signal?.aborted) {\n this.logger.info(`Processing cancelled at row ${i + 1}/${totalRows}`);\n return;\n }\n\n const row = data[i];\n const variables = this.mapRowToVariables(row, mapping, variableTypes);\n const rowStartTime = Date.now();\n\n try {\n const result = await this.client.executeMutation(mutation, variables, retryConfig, signal);\n this.metrics.recordSuccess(entityName);\n\n // Emit row success event\n callbacks?.onRowSuccess?.({\n entityName,\n rowIndex: i,\n row,\n result,\n durationMs: Date.now() - rowStartTime,\n });\n\n // Show progress every 10% or at the end\n if ((i + 1) % Math.max(1, Math.floor(totalRows / 10)) === 0 || i === totalRows - 1) {\n const progress = (((i + 1) / totalRows) * 100).toFixed(1);\n this.logger.info(`\uD83D\uDCCA Progress: ${i + 1}/${totalRows} (${progress}%) \u2713`);\n }\n } catch (error) {\n // Check if error is due to abort\n if (signal?.aborted) {\n this.logger.info(`Processing cancelled at row ${i + 1}/${totalRows}`);\n return;\n }\n\n this.metrics.recordFailure(entityName);\n this.logger.error(`\u2717 Failed to create entity for row ${i + 1}`, row, error);\n\n // Emit row failure event\n callbacks?.onRowFailure?.({\n entityName,\n rowIndex: i,\n row,\n error: error instanceof Error ? error : new Error(String(error)),\n retryAttempts: retryConfig?.maxAttempts ?? 3,\n });\n }\n }\n }\n\n private async processRowsConcurrentlyWithEvents(\n data: DataRow[],\n mutation: string,\n mapping: Record<string, unknown>,\n entityName: string,\n parallelConfig: ParallelProcessingConfig,\n retryConfig?: RetryConfig,\n signal?: AbortSignal,\n callbacks?: EntityProcessingCallbacks,\n ): Promise<void> {\n const concurrency = parallelConfig.concurrency;\n this.logger.info(`Processing ${data.length} rows with concurrency: ${concurrency}`);\n\n // Extract variable types once for all rows\n const variableTypes = this.extractVariableTypes(mutation);\n\n // Split data into chunks for concurrent processing\n const chunks = this.chunkArray(data, concurrency);\n let processedCount = 0;\n const totalRows = data.length;\n\n for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {\n // Check for abort signal before each chunk\n if (signal?.aborted) {\n this.logger.info(`Processing cancelled at chunk ${chunkIndex + 1}/${chunks.length}`);\n return;\n }\n\n const chunk = chunks[chunkIndex];\n const chunkStartIndex = chunkIndex * concurrency;\n\n const promises = chunk.map(async (row, index) => {\n const rowIndex = chunkStartIndex + index;\n const variables = this.mapRowToVariables(row, mapping, variableTypes);\n const rowStartTime = Date.now();\n\n try {\n const result = await this.client.executeMutation(\n mutation,\n variables,\n retryConfig,\n signal,\n );\n this.metrics.recordSuccess(entityName);\n\n // Emit row success event\n callbacks?.onRowSuccess?.({\n entityName,\n rowIndex,\n row,\n result,\n durationMs: Date.now() - rowStartTime,\n });\n\n return { success: true, result, row, rowIndex };\n } catch (error) {\n this.metrics.recordFailure(entityName);\n\n // Emit row failure event\n callbacks?.onRowFailure?.({\n entityName,\n rowIndex,\n row,\n error: error instanceof Error ? error : new Error(String(error)),\n retryAttempts: retryConfig?.maxAttempts ?? 3,\n });\n\n return { success: false, error, row, rowIndex };\n }\n });\n\n const results = await Promise.allSettled(promises);\n processedCount += chunk.length;\n\n // Check if cancelled during chunk processing\n if (signal?.aborted) {\n this.logger.info(`Processing cancelled after chunk ${chunkIndex + 1}/${chunks.length}`);\n return;\n }\n\n // Count successes and failures in this chunk\n let chunkSuccesses = 0;\n let chunkFailures = 0;\n\n results.forEach((result) => {\n if (result.status === \"fulfilled\") {\n const { success, error, row } = result.value;\n if (success) {\n chunkSuccesses++;\n } else {\n chunkFailures++;\n this.logger.error(`\u2717 Failed to create entity for row`, row, error);\n }\n } else {\n chunkFailures++;\n this.logger.error(`\u2717 Promise rejected: ${result.reason}`);\n }\n });\n\n // Show progress update\n const progress = ((processedCount / totalRows) * 100).toFixed(1);\n this.logger.info(\n `\uD83D\uDCCA Progress: ${processedCount}/${totalRows} (${progress}%) - Chunk ${\n chunkIndex + 1\n }: ${chunkSuccesses} \u2713, ${chunkFailures} \u2717`,\n );\n }\n }\n\n private chunkArray<T>(array: T[], chunkSize: number): T[][] {\n const chunks: T[][] = [];\n for (let i = 0; i < array.length; i += chunkSize) {\n chunks.push(array.slice(i, i + chunkSize));\n }\n return chunks;\n }\n\n private mapRowToVariables(\n row: DataRow,\n mapping: Record<string, unknown>,\n variableTypes: Record<string, string>,\n ): Record<string, any> {\n const variables: Record<string, any> = {};\n\n for (const [graphqlVar, mappingValue] of Object.entries(mapping)) {\n // Handle direct mapping for nested data (e.g., \"input\": \"$\")\n if (mappingValue === \"$\") {\n // Use the entire row as the variable value\n variables[graphqlVar] = row;\n }\n // Handle path-based mapping for nested data (e.g., \"input.name\": \"$.product.name\")\n else if (typeof mappingValue === \"string\" && mappingValue.startsWith(\"$.\")) {\n const path = mappingValue.substring(2); // Remove '$.'\n const value = this.getValueByPath(row, path);\n if (value !== undefined) {\n const type = variableTypes[graphqlVar];\n variables[graphqlVar] = this.convertValue(value, type, graphqlVar);\n }\n }\n // Handle traditional flat mapping (e.g., \"name\": \"product_name\")\n else if (typeof mappingValue === \"string\" && row[mappingValue] !== undefined) {\n const rawValue = row[mappingValue];\n const type = variableTypes[graphqlVar];\n variables[graphqlVar] = this.convertValue(rawValue, type, graphqlVar);\n }\n // Handle complex mapping object\n else if (typeof mappingValue === \"object\" && mappingValue !== null) {\n variables[graphqlVar] = this.mapNestedObject(row, mappingValue, variableTypes);\n }\n }\n\n return variables;\n }\n\n private getValueByPath(obj: any, path: string): any {\n const parts = path.split(\".\");\n let current = obj;\n\n for (const part of parts) {\n if (current && typeof current === \"object\" && part in current) {\n current = current[part];\n } else {\n return undefined;\n }\n }\n\n return current;\n }\n\n private mapNestedObject(\n row: DataRow,\n mappingObj: any,\n variableTypes: Record<string, string>,\n ): any {\n if (Array.isArray(mappingObj)) {\n return mappingObj.map((item) => this.mapNestedObject(row, item, variableTypes));\n }\n\n if (typeof mappingObj === \"object\" && mappingObj !== null) {\n const result: any = {};\n for (const [key, value] of Object.entries(mappingObj)) {\n if (typeof value === \"string\" && value.startsWith(\"$.\")) {\n const path = value.substring(2);\n let fieldValue = this.getValueByPath(row, path);\n\n // Handle special case for array fields (e.g., comma-separated values)\n if (key === \"values\" && typeof fieldValue === \"string\" && fieldValue.includes(\",\")) {\n fieldValue = fieldValue.split(\",\").map((v) => v.trim());\n }\n\n result[key] = fieldValue;\n } else if (typeof value === \"string\" && row[value] !== undefined) {\n result[key] = row[value];\n } else if (typeof value === \"object\") {\n result[key] = this.mapNestedObject(row, value, variableTypes);\n } else {\n result[key] = value;\n }\n }\n return result;\n }\n\n return mappingObj;\n }\n\n private extractVariableTypes(mutation: string): Record<string, string> {\n const types: Record<string, string> = {};\n\n try {\n const document: DocumentNode = parse(mutation);\n\n // Find the operation (mutation/query) and extract variable definitions\n for (const definition of document.definitions) {\n if (definition.kind === \"OperationDefinition\" && definition.variableDefinitions) {\n for (const variableDef of definition.variableDefinitions) {\n const varName = variableDef.variable.name.value;\n const typeName = this.extractTypeName(variableDef);\n if (typeName) {\n types[varName] = typeName;\n }\n }\n }\n }\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n this.logger.error(`Error parsing GraphQL mutation: ${message}`);\n }\n\n return types;\n }\n\n private extractTypeName(variableDef: VariableDefinitionNode): string | null {\n const type = variableDef.type;\n\n if (type.kind === \"NonNullType\") {\n // Handle non-null types like String!\n if (type.type.kind === \"NamedType\") {\n return type.type.name.value;\n }\n } else if (type.kind === \"NamedType\") {\n // Handle nullable types like String\n return type.name.value;\n }\n\n return null;\n }\n\n private convertValue(value: any, type: string | undefined, varName: string): any {\n if (!type) {\n // No type information available, keep as is\n return value;\n }\n\n // For non-string values (objects, arrays), return as is\n if (typeof value !== \"string\") {\n return value;\n }\n\n const trimmedValue = value.trim();\n\n switch (type) {\n case \"Int\":\n const intValue = Number(trimmedValue);\n // Validate that it's a valid integer (no decimals, NaN, or Infinity)\n if (isNaN(intValue) || !isFinite(intValue) || !Number.isInteger(intValue)) {\n this.logger.warn(\n `Warning: Cannot convert \"${value}\" to Int for variable $${varName}. Expected a valid integer. Using original value.`,\n );\n return value;\n }\n return intValue;\n\n case \"Float\":\n const floatValue = Number(trimmedValue);\n // Number() is more strict than parseFloat() - it requires the entire string to be valid\n if (isNaN(floatValue) || !isFinite(floatValue)) {\n this.logger.warn(\n `Warning: Cannot convert \"${value}\" to Float for variable $${varName}. Expected a valid number. Using original value.`,\n );\n return value;\n }\n return floatValue;\n\n case \"Boolean\":\n const lowerValue = trimmedValue.toLowerCase();\n if (lowerValue === \"true\" || lowerValue === \"1\") return true;\n if (lowerValue === \"false\" || lowerValue === \"0\") return false;\n this.logger.warn(\n `Warning: Cannot convert \"${value}\" to Boolean for variable $${varName}. Expected \"true\", \"false\", \"1\", or \"0\". Using original value.`,\n );\n return value;\n\n case \"String\":\n return value;\n\n default:\n // Unknown scalar type - keep as string for safety\n this.logger.debug(\n `Unknown GraphQL type \"${type}\" for variable $${varName}. Keeping value as string.`,\n );\n return value;\n }\n }\n\n getMetrics(): MetricsCollector {\n return this.metrics;\n }\n}\n", "export interface DataRow {\n [key: string]: any;\n}\n\nexport abstract class DataReader {\n abstract readFile(filePath: string): Promise<DataRow[]>;\n\n /**\n * Get the supported file extensions for this reader\n */\n abstract getSupportedExtensions(): string[];\n\n /**\n * Check if this reader can handle the given file\n */\n canHandle(filePath: string): boolean {\n const extension = filePath.split(\".\").pop()?.toLowerCase();\n return extension ? this.getSupportedExtensions().includes(extension) : false;\n }\n}\n\nexport class DataReaderFactory {\n private static readers: DataReader[] = [];\n\n static registerReader(reader: DataReader): void {\n this.readers.push(reader);\n }\n\n static getReader(filePath: string, format?: string): DataReader {\n // If format is specified, try to find reader by format\n if (format) {\n const reader = this.readers.find((r) =>\n r.getSupportedExtensions().includes(format.toLowerCase()),\n );\n if (reader) return reader;\n }\n\n // Otherwise, try to find reader by file extension\n const reader = this.readers.find((r) => r.canHandle(filePath));\n\n if (!reader) {\n throw new Error(\n `No reader found for file: ${filePath}${format ? ` with format: ${format}` : \"\"}`,\n );\n }\n\n return reader;\n }\n\n static getSupportedFormats(): string[] {\n const formats = new Set<string>();\n this.readers.forEach((reader) => {\n reader.getSupportedExtensions().forEach((ext) => formats.add(ext));\n });\n return Array.from(formats);\n }\n}\n", "import fs from \"fs\";\nimport csv from \"csv-parser\";\nimport { DataReader, DataRow } from \"./data-reader\";\n\nexport class CsvReader extends DataReader {\n getSupportedExtensions(): string[] {\n return [\"csv\"];\n }\n\n async readFile(filePath: string): Promise<DataRow[]> {\n return new Promise((resolve, reject) => {\n const results: DataRow[] = [];\n\n fs.createReadStream(filePath)\n .pipe(csv())\n .on(\"data\", (data) => results.push(data))\n .on(\"end\", () => resolve(results))\n .on(\"error\", (error) => reject(error));\n });\n }\n}\n", "import fs from \"fs/promises\";\nimport { DataReader, DataRow } from \"./data-reader\";\n\nexport class JsonReader extends DataReader {\n getSupportedExtensions(): string[] {\n return [\"json\"];\n }\n\n async readFile(filePath: string): Promise<DataRow[]> {\n const content = await fs.readFile(filePath, \"utf8\");\n const data = JSON.parse(content);\n\n // If the data is already an array, return it\n if (Array.isArray(data)) {\n return data;\n }\n\n // If it's a single object, wrap it in an array\n if (typeof data === \"object\" && data !== null) {\n return [data];\n }\n\n throw new Error(`Invalid JSON data structure in file: ${filePath}. Expected array or object.`);\n }\n}\n", "import fs from \"fs/promises\";\nimport yaml from \"js-yaml\";\nimport { DataReader, DataRow } from \"./data-reader\";\n\nexport class YamlReader extends DataReader {\n getSupportedExtensions(): string[] {\n return [\"yaml\", \"yml\"];\n }\n\n async readFile(filePath: string): Promise<DataRow[]> {\n const content = await fs.readFile(filePath, \"utf8\");\n const data = yaml.load(content);\n\n // If the data is already an array, return it\n if (Array.isArray(data)) {\n return data;\n }\n\n // If it's a single object, wrap it in an array\n if (typeof data === \"object\" && data !== null) {\n return [data];\n }\n\n throw new Error(`Invalid YAML data structure in file: ${filePath}. Expected array or object.`);\n }\n}\n", "import fs from \"fs/promises\";\nimport { DataReader, DataRow } from \"./data-reader\";\n\nexport class JsonlReader extends DataReader {\n getSupportedExtensions(): string[] {\n return [\"jsonl\", \"ndjson\"];\n }\n\n async readFile(filePath: string): Promise<DataRow[]> {\n const content = await fs.readFile(filePath, \"utf8\");\n const lines = content.split(\"\\n\").filter((line) => line.trim());\n\n const results: DataRow[] = [];\n\n for (let i = 0; i < lines.length; i++) {\n try {\n const data = JSON.parse(lines[i]);\n results.push(data);\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n throw new Error(\n `Invalid JSON at line ${i + 1} in file: ${filePath}. Error: ${errorMessage}`,\n );\n }\n }\n\n return results;\n }\n}\n", "export { DataReader, DataRow, DataReaderFactory } from \"./data-reader\";\nexport { CsvReader } from \"./csv\";\nexport { JsonReader } from \"./json\";\nexport { YamlReader } from \"./yaml\";\nexport { JsonlReader } from \"./jsonl\";\n\n// Register all readers\nimport { DataReaderFactory } from \"./data-reader\";\nimport { CsvReader } from \"./csv\";\nimport { JsonReader } from \"./json\";\nimport { YamlReader } from \"./yaml\";\nimport { JsonlReader } from \"./jsonl\";\n\n// Register readers on module load\nDataReaderFactory.registerReader(new CsvReader());\nDataReaderFactory.registerReader(new JsonReader());\nDataReaderFactory.registerReader(new YamlReader());\nDataReaderFactory.registerReader(new JsonlReader());\n", "export interface EntityMetrics {\n rowsProcessed: number;\n successfulRows: number;\n failedRows: number;\n duration: number;\n}\n\nexport interface ProcessingMetrics {\n totalRows: number;\n successfulOperations: number;\n failedOperations: number;\n totalDuration: number;\n entities: Record<string, EntityMetrics>;\n}\n\nexport interface InternalMetrics {\n totalEntities: number;\n totalSuccesses: number;\n totalFailures: number;\n entityMetrics: Map<string, InternalEntityMetrics>;\n requestDurations: number[];\n retryAttempts: number;\n retrySuccesses: number;\n retryFailures: number;\n startTime: number;\n endTime?: number;\n}\n\nexport interface InternalEntityMetrics {\n entityName: string;\n successCount: number;\n failureCount: number;\n startTime: number;\n endTime?: number;\n}\n\nexport class MetricsCollector {\n private metrics: InternalMetrics;\n\n constructor() {\n this.metrics = {\n totalEntities: 0,\n totalSuccesses: 0,\n totalFailures: 0,\n entityMetrics: new Map(),\n requestDurations: [],\n retryAttempts: 0,\n retrySuccesses: 0,\n retryFailures: 0,\n startTime: Date.now(),\n };\n }\n\n startEntityProcessing(entityName: string): void {\n if (!this.metrics.entityMetrics.has(entityName)) {\n this.metrics.entityMetrics.set(entityName, {\n entityName,\n successCount: 0,\n failureCount: 0,\n startTime: Date.now(),\n });\n }\n }\n\n recordSuccess(entityName: string): void {\n const entityMetric = this.metrics.entityMetrics.get(entityName);\n if (entityMetric) {\n entityMetric.successCount++;\n this.metrics.totalSuccesses++;\n this.metrics.totalEntities++;\n }\n }\n\n recordFailure(entityName: string): void {\n const entityMetric = this.metrics.entityMetrics.get(entityName);\n if (entityMetric) {\n entityMetric.failureCount++;\n this.metrics.totalFailures++;\n this.metrics.totalEntities++;\n }\n }\n\n finishEntityProcessing(entityName: string): void {\n const entityMetric = this.metrics.entityMetrics.get(entityName);\n if (entityMetric) {\n entityMetric.endTime = Date.now();\n }\n }\n\n finishProcessing(): InternalMetrics {\n this.metrics.endTime = Date.now();\n return { ...this.metrics };\n }\n\n getEntityMetrics(entityName: string): InternalEntityMetrics | undefined {\n return this.metrics.entityMetrics.get(entityName);\n }\n\n getMetrics(): ProcessingMetrics {\n const endTime = this.metrics.endTime || Date.now();\n const totalDuration = endTime - this.metrics.startTime;\n\n const entities: Record<string, EntityMetrics> = {};\n for (const [name, metrics] of this.metrics.entityMetrics) {\n const entityEndTime = metrics.endTime || Date.now();\n entities[name] = {\n rowsProcessed: metrics.successCount + metrics.failureCount,\n successfulRows: metrics.successCount,\n failedRows: metrics.failureCount,\n duration: entityEndTime - metrics.startTime,\n };\n }\n\n return {\n totalRows: this.metrics.totalEntities,\n successfulOperations: this.metrics.totalSuccesses,\n failedOperations: this.metrics.totalFailures,\n totalDuration,\n entities,\n };\n }\n\n getTotalProcessed(): number {\n return this.metrics.totalEntities;\n }\n\n getSuccessRate(): number {\n if (this.metrics.totalEntities === 0) return 0;\n return (this.metrics.totalSuccesses / this.metrics.totalEntities) * 100;\n }\n\n recordRequestDuration(duration: number): void {\n this.metrics.requestDurations.push(duration);\n }\n\n recordRetrySuccess(attempts: number): void {\n this.metrics.retryAttempts += attempts;\n this.metrics.retrySuccesses++;\n }\n\n recordRetryFailure(attempts: number): void {\n this.metrics.retryAttempts += attempts;\n this.metrics.retryFailures++;\n }\n\n getAverageRequestDuration(): number {\n if (this.metrics.requestDurations.length === 0) return 0;\n const sum = this.metrics.requestDurations.reduce((a, b) => a + b, 0);\n return sum / this.metrics.requestDurations.length;\n }\n\n getDurationMs(): number {\n const endTime = this.metrics.endTime || Date.now();\n return endTime - this.metrics.startTime;\n }\n\n generateSummary(): string {\n const duration = this.getDurationMs();\n const successRate = this.getSuccessRate();\n const avgRequestDuration = this.getAverageRequestDuration();\n\n let summary = `\\n\uD83D\uDCCA Processing Summary:\\n`;\n summary += ` Total Processed: ${this.metrics.totalEntities}\\n`;\n summary += ` \u2713 Successes: ${this.metrics.totalSuccesses}\\n`;\n summary += ` \u2717 Failures: ${this.metrics.totalFailures}\\n`;\n summary += ` Success Rate: ${successRate.toFixed(1)}%\\n`;\n summary += ` Duration: ${(duration / 1000).toFixed(2)}s\\n`;\n\n if (this.metrics.requestDurations.length > 0) {\n summary += ` Avg Request Time: ${avgRequestDuration.toFixed(0)}ms\\n`;\n }\n\n if (this.metrics.retryAttempts > 0) {\n summary += ` Retry Attempts: ${this.metrics.retryAttempts}\\n`;\n summary += ` Retry Successes: ${this.metrics.retrySuccesses}\\n`;\n summary += ` Retry Failures: ${this.metrics.retryFailures}\\n`;\n }\n\n if (this.metrics.entityMetrics.size > 0) {\n summary += `\\n\uD83D\uDCCB Per-Entity Breakdown:\\n`;\n for (const [entityName, entityMetric] of this.metrics.entityMetrics) {\n const entityTotal = entityMetric.successCount + entityMetric.failureCount;\n const entityRate = entityTotal > 0 ? (entityMetric.successCount / entityTotal) * 100 : 0;\n const entityDuration = entityMetric.endTime\n ? entityMetric.endTime - entityMetric.startTime\n : 0;\n\n summary += ` ${entityName}: ${entityTotal} total (${\n entityMetric.successCount\n } \u2713, ${entityMetric.failureCount} \u2717) - ${entityRate.toFixed(\n 1,\n )}% success - ${(entityDuration / 1000).toFixed(2)}s\\n`;\n }\n }\n\n return summary;\n }\n}\n", "export interface DependencyGraph {\n [entityName: string]: string[];\n}\n\nexport interface ExecutionWave {\n entities: string[];\n wave: number;\n}\n\nexport class DependencyResolver {\n private dependencies: DependencyGraph;\n private entities: string[];\n private allowPartialResolution: boolean;\n\n constructor(\n entities: string[],\n dependencies: DependencyGraph = {},\n allowPartialResolution: boolean = false,\n ) {\n this.entities = entities;\n this.dependencies = dependencies;\n this.allowPartialResolution = allowPartialResolution;\n }\n\n resolveExecutionOrder(): ExecutionWave[] {\n const waves: ExecutionWave[] = [];\n const processed = new Set<string>();\n const inProgress = new Set<string>();\n let waveNumber = 0;\n\n while (processed.size < this.entities.length) {\n const currentWave: string[] = [];\n\n for (const entity of this.entities) {\n if (processed.has(entity) || inProgress.has(entity)) {\n continue;\n }\n\n const deps = this.dependencies[entity] || [];\n const canProcess = deps.every(\n (dep) =>\n processed.has(dep) || (this.allowPartialResolution && !this.entities.includes(dep)),\n );\n\n if (canProcess) {\n currentWave.push(entity);\n inProgress.add(entity);\n }\n }\n\n if (currentWave.length === 0) {\n const remaining = this.entities.filter((e) => !processed.has(e));\n throw new Error(\n `Circular dependency detected or missing dependencies for entities: ${remaining.join(\n \", \",\n )}`,\n );\n }\n\n waves.push({\n entities: currentWave,\n wave: waveNumber++,\n });\n\n currentWave.forEach((entity) => {\n processed.add(entity);\n inProgress.delete(entity);\n });\n }\n\n return waves;\n }\n\n validateDependencies(): string[] {\n const errors: string[] = [];\n const entitySet = new Set(this.entities);\n\n for (const [entity, deps] of Object.entries(this.dependencies)) {\n if (!entitySet.has(entity)) {\n errors.push(`Entity '${entity}' has dependencies but is not in the entity list`);\n continue;\n }\n\n for (const dep of deps) {\n if (!entitySet.has(dep)) {\n errors.push(`Entity '${entity}' depends on '${dep}' which does not exist`);\n }\n }\n }\n\n return errors;\n }\n\n getDependents(entityName: string): string[] {\n return Object.entries(this.dependencies)\n .filter(([_, deps]) => deps.includes(entityName))\n .map(([entity, _]) => entity);\n }\n\n getDependencies(entityName: string): string[] {\n return this.dependencies[entityName] || [];\n }\n}\n", "import { ProcessingMetrics, EntityMetrics } from \"./metrics\";\nimport { DataRow } from \"../readers\";\n\n/**\n * Event types emitted by GQLIngest during processing\n */\nexport type GQLIngestEventType =\n | \"started\"\n | \"progress\"\n | \"entityStart\"\n | \"entityComplete\"\n | \"rowSuccess\"\n | \"rowFailure\"\n | \"cancelled\"\n | \"finished\"\n | \"errored\";\n\n/**\n * Payload for 'started' event - emitted when ingestion begins\n */\nexport interface StartedEventPayload {\n /** Path to configuration directory */\n configPath: string;\n /** Total number of entities to process */\n totalEntities: number;\n /** Names of entities that will be processed */\n entityNames: string[];\n /** Number of dependency waves */\n totalWaves: number;\n /** Timestamp when ingestion started */\n startTime: number;\n}\n\n/**\n * Payload for 'progress' event - emitted periodically during processing\n */\nexport interface ProgressEventPayload {\n /** Current wave number (0-indexed) */\n currentWave: number;\n /** Total number of waves */\n totalWaves: number;\n /** Number of entities completed */\n entitiesCompleted: number;\n /** Total number of entities */\n totalEntities: number;\n /** Total rows processed so far */\n rowsProcessed: number;\n /** Number of successful row operations */\n successfulRows: number;\n /** Number of failed row operations */\n failedRows: number;\n /** Progress percentage (0-100) */\n progressPercent: number;\n /** Elapsed time in milliseconds */\n elapsedMs: number;\n}\n\n/**\n * Payload for 'entityStart' event - emitted when an entity begins processing\n */\nexport interface EntityStartEventPayload {\n /** Name of the entity being processed */\n entityName: string;\n /** Path to the entity's mapping configuration */\n mappingPath: string;\n /** Total number of rows to process for this entity */\n totalRows: number;\n /** Wave number this entity belongs to */\n waveIndex: number;\n}\n\n/**\n * Payload for 'entityComplete' event - emitted when an entity finishes processing\n */\nexport interface EntityCompleteEventPayload {\n /** Name of the entity that completed */\n entityName: string;\n /** Metrics for this entity */\n metrics: EntityMetrics;\n /** Whether all rows were successful */\n success: boolean;\n /** Duration in milliseconds */\n durationMs: number;\n}\n\n/**\n * Payload for 'rowSuccess' event - emitted for each successful row mutation\n */\nexport interface RowSuccessEventPayload {\n /** Entity name */\n entityName: string;\n /** Row index (0-indexed) */\n rowIndex: number;\n /** The original row data */\n row: DataRow;\n /** GraphQL mutation result */\n result: unknown;\n /** Duration of the mutation in milliseconds */\n durationMs: number;\n}\n\n/**\n * Payload for 'rowFailure' event - emitted for each failed row mutation\n */\nexport interface RowFailureEventPayload {\n /** Entity name */\n entityName: string;\n /** Row index (0-indexed) */\n rowIndex: number;\n /** The original row data */\n row: DataRow;\n /** Error that occurred */\n error: Error;\n /** Number of retry attempts made */\n retryAttempts: number;\n}\n\n/**\n * Payload for 'cancelled' event - emitted when processing is cancelled\n */\nexport interface CancelledEventPayload {\n /** Reason for cancellation */\n reason: string;\n /** Metrics at time of cancellation */\n metrics: ProcessingMetrics;\n /** Entity being processed when cancelled (if any) */\n currentEntity?: string;\n /** How long processing ran before cancellation */\n elapsedMs: number;\n}\n\n/**\n * Payload for 'finished' event - emitted when processing completes successfully\n */\nexport interface FinishedEventPayload {\n /** Final processing metrics */\n metrics: ProcessingMetrics;\n /** Total duration in milliseconds */\n durationMs: number;\n /** Whether all entities succeeded */\n allSuccessful: boolean;\n}\n\n/**\n * Payload for 'errored' event - emitted when a fatal error occurs\n */\nexport interface ErroredEventPayload {\n /** The error that occurred */\n error: Error;\n /** Metrics at time of error */\n metrics: ProcessingMetrics;\n /** Entity being processed when error occurred (if any) */\n currentEntity?: string;\n /** How long processing ran before error */\n elapsedMs: number;\n}\n\n/**\n * Map of event types to their payload types\n */\nexport interface GQLIngestEventMap {\n started: StartedEventPayload;\n progress: ProgressEventPayload;\n entityStart: EntityStartEventPayload;\n entityComplete: EntityCompleteEventPayload;\n rowSuccess: RowSuccessEventPayload;\n rowFailure: RowFailureEventPayload;\n cancelled: CancelledEventPayload;\n finished: FinishedEventPayload;\n errored: ErroredEventPayload;\n}\n\n/**\n * Options for configuring event emission behavior\n */\nexport interface EventOptions {\n /** Emit rowSuccess/rowFailure events (can be verbose for large datasets). Default: true */\n emitRowEvents?: boolean;\n /** Interval for progress events in milliseconds. Default: 1000 (1 second) */\n progressInterval?: number;\n /** Emit progress events. Default: true */\n emitProgressEvents?: boolean;\n}\n\n/**\n * Default event options\n */\nexport const DEFAULT_EVENT_OPTIONS: Required<EventOptions> = {\n emitRowEvents: true,\n progressInterval: 1000,\n emitProgressEvents: true,\n};\n"],
|
|
5
|
+
"mappings": ";AAAA,SAAS,oBAAoB;;;ACA7B,SAAS,qBAAqB;;;ACA9B,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,YAAY,UAAU;;;ACcf,IAAM,aAAqB;AAAA,EAChC,OAAO,MAAM;AAAA,EAAC;AAAA,EACd,MAAM,MAAM;AAAA,EAAC;AAAA,EACb,MAAM,MAAM;AAAA,EAAC;AAAA,EACb,OAAO,MAAM;AAAA,EAAC;AAChB;AAMO,SAAS,sBAA8B;AAC5C,SAAO;AAAA,IACL,OAAO,CAAC,QAAQ,SAAS,QAAQ,MAAM,gBAAgB,GAAG,IAAI,GAAG,IAAI;AAAA,IACrE,MAAM,CAAC,QAAQ,SAAS,QAAQ,KAAK,gBAAgB,GAAG,IAAI,GAAG,IAAI;AAAA,IACnE,MAAM,CAAC,QAAQ,SAAS,QAAQ,KAAK,gBAAgB,GAAG,IAAI,GAAG,IAAI;AAAA,IACnE,OAAO,CAAC,QAAQ,SAAS,QAAQ,MAAM,gBAAgB,GAAG,IAAI,GAAG,IAAI;AAAA,EACvE;AACF;AAMO,SAAS,oBAAoB,UAAU,OAAe;AAC3D,SAAO,UAAU,oBAAoB,IAAI;AAC3C;;;ADNO,IAAM,uBAAoC;AAAA,EAC/C,aAAa;AAAA,EACb,WAAW;AAAA,EACX,UAAU;AAAA,EACV,oBAAoB;AAAA,EACpB,sBAAsB,CAAC,KAAK,KAAK,KAAK,KAAK,KAAK,GAAG;AACrD;AAEO,IAAM,0BAAoD;AAAA,EAC/D,aAAa;AAAA,EACb,mBAAmB;AAAA,EACnB,kBAAkB;AACpB;AAEO,IAAM,iBAAmC;AAAA,EAC9C,OAAO;AAAA,EACP,oBAAoB;AAAA,EACpB,cAAc,CAAC;AAAA,EACf,oBAAoB,CAAC;AACvB;AAEO,SAAS,WAAW,WAAmB,SAAiB,YAA8B;AAC3F,QAAM,aAAa,KAAK,KAAK,WAAW,aAAa;AAErD,MAAI;AACF,QAAI,CAAC,GAAG,WAAW,UAAU,GAAG;AAC9B,aAAO,KAAK,2DAA2D;AACvE,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,GAAG,aAAa,YAAY,MAAM;AACvD,UAAM,aAAkB,UAAK,YAAY;AAEzC,WAAO,kBAAkB,UAAU;AAAA,EACrC,SAAS,OAAO;AACd,UAAM,eAAe,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC1E,WAAO,KAAK,yCAAyC,YAAY,mBAAmB;AACpF,WAAO;AAAA,EACT;AACF;AAEA,SAAS,kBAAkB,YAAmD;AAC5E,SAAO;AAAA,IACL,OAAO;AAAA,MACL,GAAG;AAAA,MACH,GAAG,WAAW;AAAA,IAChB;AAAA,IACA,oBAAoB;AAAA,MAClB,GAAG;AAAA,MACH,GAAG,WAAW;AAAA,IAChB;AAAA,IACA,cAAc,WAAW,gBAAgB,CAAC;AAAA,IAC1C,oBAAoB,WAAW,sBAAsB,CAAC;AAAA,EACxD;AACF;AAEO,SAAS,gBACd,YACA,cACA,SAAiB,YACS;AAC1B,QAAM,kBAAkB,aAAa,aAAa,UAAU,KAAK,CAAC;AAElE,MAAI,cAAc;AAAA,IAChB,GAAG,aAAa;AAAA,IAChB,GAAG;AAAA,EACL;AAGA,MAAI,YAAY,oBAAoB,YAAY,cAAc,GAAG;AAC/D,WAAO;AAAA,MACL,WAAW,UAAU,sDAAsD,YAAY,WAAW;AAAA,IACpG;AACA,gBAAY,cAAc;AAAA,EAC5B;AAEA,SAAO;AACT;AAEO,SAAS,eAAe,YAAoB,cAA6C;AAC9F,QAAM,kBAAkB,aAAa,aAAa,UAAU,GAAG,SAAS,CAAC;AAEzE,SAAO;AAAA,IACL,GAAG,aAAa;AAAA,IAChB,GAAG;AAAA,EACL;AACF;;;ADrHO,IAAM,uBAAN,MAA2B;AAAA,EAKhC,YACE,UACA,SACA,SACA,SAAiB,YACjB;AACA,SAAK,SAAS,IAAI,cAAc,UAAU;AAAA,MACxC,SAAS,WAAW,CAAC;AAAA,IACvB,CAAC;AACD,SAAK,UAAU;AACf,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,gBACJ,UACA,WACA,aACA,QACc;AACd,UAAM,SAAS,eAAe;AAE9B,QAAI;AACJ,UAAM,iBAAiB,KAAK,IAAI;AAEhC,aAAS,UAAU,GAAG,UAAU,OAAO,aAAa,WAAW;AAE7D,UAAI,QAAQ,SAAS;AACnB,cAAM,IAAI,MAAM,oBAAoB,OAAO,UAAU,WAAW,EAAE;AAAA,MACpE;AAEA,YAAM,mBAAmB,KAAK,IAAI;AAElC,UAAI;AACF,cAAM,SAAS,MAAM,KAAK,OAAO,QAAQ,UAAU,SAAS;AAE5D,YAAI,KAAK,SAAS;AAChB,gBAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,eAAK,QAAQ,sBAAsB,QAAQ;AAC3C,cAAI,UAAU,GAAG;AACf,iBAAK,QAAQ,mBAAmB,OAAO;AAAA,UACzC;AAAA,QACF;AAEA,cAAMA,iBAAgB,KAAK,IAAI,IAAI;AACnC,cAAM,YAAY,UAAU,IAAI,0BAA0B,UAAU,CAAC,MAAM;AAC3E,aAAK,OAAO,MAAM,uCAAkCA,cAAa,KAAK,SAAS,IAAI,MAAM;AAEzF,eAAO;AAAA,MACT,SAAS,OAAY;AACnB,oBAAY;AACZ,cAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,YAAI,KAAK,SAAS;AAChB,eAAK,QAAQ,sBAAsB,QAAQ;AAAA,QAC7C;AAGA,YAAI,YAAY,OAAO,cAAc,GAAG;AACtC,cAAI,KAAK,WAAW,UAAU,GAAG;AAC/B,iBAAK,QAAQ,mBAAmB,OAAO;AAAA,UACzC;AACA;AAAA,QACF;AAGA,YAAI,CAAC,KAAK,iBAAiB,OAAO,MAAM,GAAG;AACzC,eAAK,OAAO;AAAA,YACV,6DAAwD,QAAQ;AAAA,YAChE;AAAA,UACF;AACA,gBAAM;AAAA,QACR;AAGA,cAAM,QAAQ,KAAK,eAAe,SAAS,MAAM;AACjD,aAAK,OAAO;AAAA,UACV,0CAAqC,UAAU,CAAC,IAAI,OAAO,WAAW,kBAAkB,KAAK;AAAA,QAC/F;AAGA,cAAM,KAAK,gBAAgB,OAAO,MAAM;AAAA,MAC1C;AAAA,IACF;AAGA,UAAM,gBAAgB,KAAK,IAAI,IAAI;AACnC,SAAK,OAAO;AAAA,MACV,uCAAkC,OAAO,WAAW,gBAAgB,aAAa;AAAA,MACjF;AAAA,IACF;AAEA,UAAM;AAAA,EACR;AAAA,EAEQ,iBAAiB,OAAY,QAA8B;AAEjE,QAAI,CAAC,MAAM,UAAU;AACnB,aAAO;AAAA,IACT;AAGA,UAAM,SAAS,MAAM,SAAS;AAC9B,WAAO,OAAO,qBAAqB,SAAS,MAAM;AAAA,EACpD;AAAA,EAEQ,eAAe,SAAiB,QAA6B;AACnE,QAAI,CAAC,OAAO,oBAAoB;AAC9B,aAAO,OAAO;AAAA,IAChB;AAEA,UAAM,mBAAmB,OAAO,YAAY,KAAK,IAAI,GAAG,OAAO;AAC/D,UAAM,cAAc,KAAK,IAAI,kBAAkB,OAAO,QAAQ;AAG9D,UAAM,SAAS,cAAc,OAAO,KAAK,OAAO,IAAI;AACpD,WAAO,KAAK,IAAI,GAAG,cAAc,MAAM;AAAA,EACzC;AAAA;AAAA;AAAA;AAAA,EAKQ,gBAAgB,IAAY,QAAqC;AACvE,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAI,QAAQ,SAAS;AACnB,eAAO,IAAI,MAAM,oBAAoB,OAAO,UAAU,WAAW,EAAE,CAAC;AACpE;AAAA,MACF;AAEA,YAAM,YAAY,WAAW,SAAS,EAAE;AAExC,UAAI,QAAQ;AACV,cAAM,eAAe,MAAM;AACzB,uBAAa,SAAS;AACtB,iBAAO,IAAI,MAAM,oBAAoB,OAAO,UAAU,WAAW,EAAE,CAAC;AAAA,QACtE;AACA,eAAO,iBAAiB,SAAS,cAAc,EAAE,MAAM,KAAK,CAAC;AAAA,MAC/D;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,SAAiC;AAC1C,SAAK,OAAO,WAAW,OAAO;AAAA,EAChC;AACF;;;AGzJA,OAAOC,SAAQ;AACf,OAAOC,WAAU;AACjB,SAAS,aAAmD;;;ACErD,IAAe,aAAf,MAA0B;AAAA;AAAA;AAAA;AAAA,EAW/B,UAAU,UAA2B;AACnC,UAAM,YAAY,SAAS,MAAM,GAAG,EAAE,IAAI,GAAG,YAAY;AACzD,WAAO,YAAY,KAAK,uBAAuB,EAAE,SAAS,SAAS,IAAI;AAAA,EACzE;AACF;AAEO,IAAM,oBAAN,MAAwB;AAAA,EAC7B;AAAA,SAAe,UAAwB,CAAC;AAAA;AAAA,EAExC,OAAO,eAAe,QAA0B;AAC9C,SAAK,QAAQ,KAAK,MAAM;AAAA,EAC1B;AAAA,EAEA,OAAO,UAAU,UAAkB,QAA6B;AAE9D,QAAI,QAAQ;AACV,YAAMC,UAAS,KAAK,QAAQ;AAAA,QAAK,CAAC,MAChC,EAAE,uBAAuB,EAAE,SAAS,OAAO,YAAY,CAAC;AAAA,MAC1D;AACA,UAAIA,QAAQ,QAAOA;AAAA,IACrB;AAGA,UAAM,SAAS,KAAK,QAAQ,KAAK,CAAC,MAAM,EAAE,UAAU,QAAQ,CAAC;AAE7D,QAAI,CAAC,QAAQ;AACX,YAAM,IAAI;AAAA,QACR,6BAA6B,QAAQ,GAAG,SAAS,iBAAiB,MAAM,KAAK,EAAE;AAAA,MACjF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,OAAO,sBAAgC;AACrC,UAAM,UAAU,oBAAI,IAAY;AAChC,SAAK,QAAQ,QAAQ,CAAC,WAAW;AAC/B,aAAO,uBAAuB,EAAE,QAAQ,CAAC,QAAQ,QAAQ,IAAI,GAAG,CAAC;AAAA,IACnE,CAAC;AACD,WAAO,MAAM,KAAK,OAAO;AAAA,EAC3B;AACF;;;ACxDA,OAAOC,SAAQ;AACf,OAAO,SAAS;AAGT,IAAM,YAAN,cAAwB,WAAW;AAAA,EACxC,yBAAmC;AACjC,WAAO,CAAC,KAAK;AAAA,EACf;AAAA,EAEA,MAAM,SAAS,UAAsC;AACnD,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,UAAqB,CAAC;AAE5B,MAAAC,IAAG,iBAAiB,QAAQ,EACzB,KAAK,IAAI,CAAC,EACV,GAAG,QAAQ,CAAC,SAAS,QAAQ,KAAK,IAAI,CAAC,EACvC,GAAG,OAAO,MAAM,QAAQ,OAAO,CAAC,EAChC,GAAG,SAAS,CAAC,UAAU,OAAO,KAAK,CAAC;AAAA,IACzC,CAAC;AAAA,EACH;AACF;;;ACpBA,OAAOC,SAAQ;AAGR,IAAM,aAAN,cAAyB,WAAW;AAAA,EACzC,yBAAmC;AACjC,WAAO,CAAC,MAAM;AAAA,EAChB;AAAA,EAEA,MAAM,SAAS,UAAsC;AACnD,UAAM,UAAU,MAAMC,IAAG,SAAS,UAAU,MAAM;AAClD,UAAM,OAAO,KAAK,MAAM,OAAO;AAG/B,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,aAAO;AAAA,IACT;AAGA,QAAI,OAAO,SAAS,YAAY,SAAS,MAAM;AAC7C,aAAO,CAAC,IAAI;AAAA,IACd;AAEA,UAAM,IAAI,MAAM,wCAAwC,QAAQ,6BAA6B;AAAA,EAC/F;AACF;;;ACxBA,OAAOC,SAAQ;AACf,OAAOC,WAAU;AAGV,IAAM,aAAN,cAAyB,WAAW;AAAA,EACzC,yBAAmC;AACjC,WAAO,CAAC,QAAQ,KAAK;AAAA,EACvB;AAAA,EAEA,MAAM,SAAS,UAAsC;AACnD,UAAM,UAAU,MAAMC,IAAG,SAAS,UAAU,MAAM;AAClD,UAAM,OAAOC,MAAK,KAAK,OAAO;AAG9B,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,aAAO;AAAA,IACT;AAGA,QAAI,OAAO,SAAS,YAAY,SAAS,MAAM;AAC7C,aAAO,CAAC,IAAI;AAAA,IACd;AAEA,UAAM,IAAI,MAAM,wCAAwC,QAAQ,6BAA6B;AAAA,EAC/F;AACF;;;ACzBA,OAAOC,SAAQ;AAGR,IAAM,cAAN,cAA0B,WAAW;AAAA,EAC1C,yBAAmC;AACjC,WAAO,CAAC,SAAS,QAAQ;AAAA,EAC3B;AAAA,EAEA,MAAM,SAAS,UAAsC;AACnD,UAAM,UAAU,MAAMC,IAAG,SAAS,UAAU,MAAM;AAClD,UAAM,QAAQ,QAAQ,MAAM,IAAI,EAAE,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC;AAE9D,UAAM,UAAqB,CAAC;AAE5B,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,UAAI;AACF,cAAM,OAAO,KAAK,MAAM,MAAM,CAAC,CAAC;AAChC,gBAAQ,KAAK,IAAI;AAAA,MACnB,SAAS,OAAO;AACd,cAAM,eAAe,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC1E,cAAM,IAAI;AAAA,UACR,wBAAwB,IAAI,CAAC,aAAa,QAAQ,YAAY,YAAY;AAAA,QAC5E;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;;;ACdA,kBAAkB,eAAe,IAAI,UAAU,CAAC;AAChD,kBAAkB,eAAe,IAAI,WAAW,CAAC;AACjD,kBAAkB,eAAe,IAAI,WAAW,CAAC;AACjD,kBAAkB,eAAe,IAAI,YAAY,CAAC;;;ACmB3C,IAAM,mBAAN,MAAuB;AAAA,EAG5B,cAAc;AACZ,SAAK,UAAU;AAAA,MACb,eAAe;AAAA,MACf,gBAAgB;AAAA,MAChB,eAAe;AAAA,MACf,eAAe,oBAAI,IAAI;AAAA,MACvB,kBAAkB,CAAC;AAAA,MACnB,eAAe;AAAA,MACf,gBAAgB;AAAA,MAChB,eAAe;AAAA,MACf,WAAW,KAAK,IAAI;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,sBAAsB,YAA0B;AAC9C,QAAI,CAAC,KAAK,QAAQ,cAAc,IAAI,UAAU,GAAG;AAC/C,WAAK,QAAQ,cAAc,IAAI,YAAY;AAAA,QACzC;AAAA,QACA,cAAc;AAAA,QACd,cAAc;AAAA,QACd,WAAW,KAAK,IAAI;AAAA,MACtB,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,cAAc,YAA0B;AACtC,UAAM,eAAe,KAAK,QAAQ,cAAc,IAAI,UAAU;AAC9D,QAAI,cAAc;AAChB,mBAAa;AACb,WAAK,QAAQ;AACb,WAAK,QAAQ;AAAA,IACf;AAAA,EACF;AAAA,EAEA,cAAc,YAA0B;AACtC,UAAM,eAAe,KAAK,QAAQ,cAAc,IAAI,UAAU;AAC9D,QAAI,cAAc;AAChB,mBAAa;AACb,WAAK,QAAQ;AACb,WAAK,QAAQ;AAAA,IACf;AAAA,EACF;AAAA,EAEA,uBAAuB,YAA0B;AAC/C,UAAM,eAAe,KAAK,QAAQ,cAAc,IAAI,UAAU;AAC9D,QAAI,cAAc;AAChB,mBAAa,UAAU,KAAK,IAAI;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,mBAAoC;AAClC,SAAK,QAAQ,UAAU,KAAK,IAAI;AAChC,WAAO,EAAE,GAAG,KAAK,QAAQ;AAAA,EAC3B;AAAA,EAEA,iBAAiB,YAAuD;AACtE,WAAO,KAAK,QAAQ,cAAc,IAAI,UAAU;AAAA,EAClD;AAAA,EAEA,aAAgC;AAC9B,UAAM,UAAU,KAAK,QAAQ,WAAW,KAAK,IAAI;AACjD,UAAM,gBAAgB,UAAU,KAAK,QAAQ;AAE7C,UAAM,WAA0C,CAAC;AACjD,eAAW,CAAC,MAAM,OAAO,KAAK,KAAK,QAAQ,eAAe;AACxD,YAAM,gBAAgB,QAAQ,WAAW,KAAK,IAAI;AAClD,eAAS,IAAI,IAAI;AAAA,QACf,eAAe,QAAQ,eAAe,QAAQ;AAAA,QAC9C,gBAAgB,QAAQ;AAAA,QACxB,YAAY,QAAQ;AAAA,QACpB,UAAU,gBAAgB,QAAQ;AAAA,MACpC;AAAA,IACF;AAEA,WAAO;AAAA,MACL,WAAW,KAAK,QAAQ;AAAA,MACxB,sBAAsB,KAAK,QAAQ;AAAA,MACnC,kBAAkB,KAAK,QAAQ;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,oBAA4B;AAC1B,WAAO,KAAK,QAAQ;AAAA,EACtB;AAAA,EAEA,iBAAyB;AACvB,QAAI,KAAK,QAAQ,kBAAkB,EAAG,QAAO;AAC7C,WAAQ,KAAK,QAAQ,iBAAiB,KAAK,QAAQ,gBAAiB;AAAA,EACtE;AAAA,EAEA,sBAAsB,UAAwB;AAC5C,SAAK,QAAQ,iBAAiB,KAAK,QAAQ;AAAA,EAC7C;AAAA,EAEA,mBAAmB,UAAwB;AACzC,SAAK,QAAQ,iBAAiB;AAC9B,SAAK,QAAQ;AAAA,EACf;AAAA,EAEA,mBAAmB,UAAwB;AACzC,SAAK,QAAQ,iBAAiB;AAC9B,SAAK,QAAQ;AAAA,EACf;AAAA,EAEA,4BAAoC;AAClC,QAAI,KAAK,QAAQ,iBAAiB,WAAW,EAAG,QAAO;AACvD,UAAM,MAAM,KAAK,QAAQ,iBAAiB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AACnE,WAAO,MAAM,KAAK,QAAQ,iBAAiB;AAAA,EAC7C;AAAA,EAEA,gBAAwB;AACtB,UAAM,UAAU,KAAK,QAAQ,WAAW,KAAK,IAAI;AACjD,WAAO,UAAU,KAAK,QAAQ;AAAA,EAChC;AAAA,EAEA,kBAA0B;AACxB,UAAM,WAAW,KAAK,cAAc;AACpC,UAAM,cAAc,KAAK,eAAe;AACxC,UAAM,qBAAqB,KAAK,0BAA0B;AAE1D,QAAI,UAAU;AAAA;AAAA;AACd,eAAW,uBAAuB,KAAK,QAAQ,aAAa;AAAA;AAC5D,eAAW,wBAAmB,KAAK,QAAQ,cAAc;AAAA;AACzD,eAAW,uBAAkB,KAAK,QAAQ,aAAa;AAAA;AACvD,eAAW,oBAAoB,YAAY,QAAQ,CAAC,CAAC;AAAA;AACrD,eAAW,iBAAiB,WAAW,KAAM,QAAQ,CAAC,CAAC;AAAA;AAEvD,QAAI,KAAK,QAAQ,iBAAiB,SAAS,GAAG;AAC5C,iBAAW,wBAAwB,mBAAmB,QAAQ,CAAC,CAAC;AAAA;AAAA,IAClE;AAEA,QAAI,KAAK,QAAQ,gBAAgB,GAAG;AAClC,iBAAW,sBAAsB,KAAK,QAAQ,aAAa;AAAA;AAC3D,iBAAW,uBAAuB,KAAK,QAAQ,cAAc;AAAA;AAC7D,iBAAW,sBAAsB,KAAK,QAAQ,aAAa;AAAA;AAAA,IAC7D;AAEA,QAAI,KAAK,QAAQ,cAAc,OAAO,GAAG;AACvC,iBAAW;AAAA;AAAA;AACX,iBAAW,CAAC,YAAY,YAAY,KAAK,KAAK,QAAQ,eAAe;AACnE,cAAM,cAAc,aAAa,eAAe,aAAa;AAC7D,cAAM,aAAa,cAAc,IAAK,aAAa,eAAe,cAAe,MAAM;AACvF,cAAM,iBAAiB,aAAa,UAChC,aAAa,UAAU,aAAa,YACpC;AAEJ,mBAAW,MAAM,UAAU,KAAK,WAAW,WACzC,aAAa,YACf,YAAO,aAAa,YAAY,cAAS,WAAW;AAAA,UAClD;AAAA,QACF,CAAC,gBAAgB,iBAAiB,KAAM,QAAQ,CAAC,CAAC;AAAA;AAAA,MACpD;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;;;APlKO,IAAM,aAAN,MAAiB;AAAA,EAOtB,YACE,QACA,WAAmB,QAAQ,IAAI,GAC/B,SACA,SAAiB,YACjB,gBACA;AACA,SAAK,SAAS;AACd,SAAK,WAAW;AAChB,SAAK,UAAU,WAAW,IAAI,iBAAiB;AAC/C,SAAK,SAAS;AACd,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,iBAAiB,WAAmB,cAAmC;AACrE,UAAM,eAAeC,MAAK,QAAQ,KAAK,UAAU,WAAW,UAAU;AAEtE,QAAI;AACF,YAAM,QAAQC,IAAG,YAAY,YAAY;AACzC,UAAI,YAAY,MAAM,OAAO,CAAC,SAAS,KAAK,SAAS,OAAO,CAAC;AAG7D,UAAI,gBAAgB,aAAa,SAAS,GAAG;AAC3C,cAAM,oBAAoB,IAAI,IAAI,YAAY;AAC9C,cAAM,gBAAgB,oBAAI,IAAY;AAEtC,oBAAY,UAAU,OAAO,CAAC,SAAS;AACrC,gBAAM,aAAaD,MAAK,SAAS,MAAM,OAAO;AAC9C,cAAI,kBAAkB,IAAI,UAAU,GAAG;AACrC,0BAAc,IAAI,UAAU;AAC5B,mBAAO;AAAA,UACT;AACA,iBAAO;AAAA,QACT,CAAC;AAGD,cAAM,WAAW,aAAa,OAAO,CAAC,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;AACjE,YAAI,SAAS,SAAS,GAAG;AACvB,eAAK,OAAO;AAAA,YACV,+DAA+D,SAAS,KAAK,IAAI,CAAC;AAAA,UACpF;AAAA,QACF;AAAA,MACF;AAEA,gBAAU,KAAK;AAEf,WAAK,OAAO,KAAK,cAAc,UAAU,MAAM,mBAAmB,UAAU,KAAK,IAAI,CAAC,EAAE;AACxF,aAAO,UAAU,IAAI,CAAC,SAASA,MAAK,KAAK,WAAW,YAAY,IAAI,CAAC;AAAA,IACvE,SAAS,OAAO;AACd,YAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACrE,WAAK,OAAO,MAAM,oCAAoC,YAAY,KAAK,OAAO,EAAE;AAChF,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,cACJ,YACA,gBACA,aACe;AACf,WAAO,KAAK,wBAAwB,YAAY,gBAAgB,WAAW;AAAA,EAC7E;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,wBACJ,YACA,gBACA,aACA,QACA,WACe;AACf,UAAM,aAAaA,MAAK,SAAS,YAAY,OAAO;AACpD,UAAM,kBAAkB,KAAK,IAAI;AAEjC,SAAK,OAAO,KAAK,sBAAsB,UAAU,EAAE;AACnD,SAAK,QAAQ,sBAAsB,UAAU;AAG7C,UAAM,iBAAiBA,MAAK,QAAQ,KAAK,UAAU,UAAU;AAC7D,UAAM,SAAwB,KAAK,MAAMC,IAAG,aAAa,gBAAgB,MAAM,CAAC;AAGhF,UAAM,YAAYD,MAAK,QAAQA,MAAK,QAAQ,cAAc,CAAC;AAG3D,UAAM,WAAW,OAAO,YAAY,OAAO;AAC3C,QAAI,CAAC,UAAU;AACb,YAAM,IAAI,MAAM,6CAA6C,UAAU,EAAE;AAAA,IAC3E;AAEA,UAAM,WAAWA,MAAK,QAAQ,WAAW,QAAQ;AAGjD,UAAM,SAAS,KAAK,kBAAkB,OAAO;AAC7C,UAAM,SAAS,kBAAkB,UAAU,UAAU,MAAM;AAC3D,UAAM,OAAO,MAAM,OAAO,SAAS,QAAQ;AAG3C,UAAM,cAAcA,MAAK,QAAQ,WAAW,OAAO,WAAW;AAC9D,UAAM,WAAWC,IAAG,aAAa,aAAa,MAAM;AAGpD,eAAW,gBAAgB;AAAA,MACzB;AAAA,MACA,aAAa;AAAA,MACb,WAAW,KAAK;AAAA,IAClB,CAAC;AAGD,QAAI,kBAAkB,eAAe,cAAc,GAAG;AACpD,YAAM,KAAK;AAAA,QACT;AAAA,QACA;AAAA,QACA,OAAO;AAAA,QACP;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF,OAAO;AACL,YAAM,KAAK;AAAA,QACT;AAAA,QACA;AAAA,QACA,OAAO;AAAA,QACP;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAEA,SAAK,QAAQ,uBAAuB,UAAU;AAG9C,UAAM,kBAAkB,KAAK,QAAQ,iBAAiB,UAAU;AAChE,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,gBAAgB,kBAClB;AAAA,MACE,eAAe,gBAAgB,eAAe,gBAAgB;AAAA,MAC9D,gBAAgB,gBAAgB;AAAA,MAChC,YAAY,gBAAgB;AAAA,MAC5B;AAAA,IACF,IACA;AAAA,MACE,eAAe;AAAA,MACf,gBAAgB;AAAA,MAChB,YAAY;AAAA,MACZ;AAAA,IACF;AAEJ,eAAW,mBAAmB;AAAA,MAC5B;AAAA,MACA,SAAS;AAAA,MACT,SAAS,cAAc,eAAe;AAAA,MACtC,YAAY;AAAA,IACd,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,kCACZ,MACA,UACA,SACA,YACA,aACA,QACA,WACe;AACf,UAAM,YAAY,KAAK;AACvB,UAAM,gBAAgB,KAAK,qBAAqB,QAAQ;AAExD,aAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AAEpC,UAAI,QAAQ,SAAS;AACnB,aAAK,OAAO,KAAK,+BAA+B,IAAI,CAAC,IAAI,SAAS,EAAE;AACpE;AAAA,MACF;AAEA,YAAM,MAAM,KAAK,CAAC;AAClB,YAAM,YAAY,KAAK,kBAAkB,KAAK,SAAS,aAAa;AACpE,YAAM,eAAe,KAAK,IAAI;AAE9B,UAAI;AACF,cAAM,SAAS,MAAM,KAAK,OAAO,gBAAgB,UAAU,WAAW,aAAa,MAAM;AACzF,aAAK,QAAQ,cAAc,UAAU;AAGrC,mBAAW,eAAe;AAAA,UACxB;AAAA,UACA,UAAU;AAAA,UACV;AAAA,UACA;AAAA,UACA,YAAY,KAAK,IAAI,IAAI;AAAA,QAC3B,CAAC;AAGD,aAAK,IAAI,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,YAAY,EAAE,CAAC,MAAM,KAAK,MAAM,YAAY,GAAG;AAClF,gBAAM,aAAc,IAAI,KAAK,YAAa,KAAK,QAAQ,CAAC;AACxD,eAAK,OAAO,KAAK,uBAAgB,IAAI,CAAC,IAAI,SAAS,KAAK,QAAQ,WAAM;AAAA,QACxE;AAAA,MACF,SAAS,OAAO;AAEd,YAAI,QAAQ,SAAS;AACnB,eAAK,OAAO,KAAK,+BAA+B,IAAI,CAAC,IAAI,SAAS,EAAE;AACpE;AAAA,QACF;AAEA,aAAK,QAAQ,cAAc,UAAU;AACrC,aAAK,OAAO,MAAM,0CAAqC,IAAI,CAAC,IAAI,KAAK,KAAK;AAG1E,mBAAW,eAAe;AAAA,UACxB;AAAA,UACA,UAAU;AAAA,UACV;AAAA,UACA,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AAAA,UAC/D,eAAe,aAAa,eAAe;AAAA,QAC7C,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,kCACZ,MACA,UACA,SACA,YACA,gBACA,aACA,QACA,WACe;AACf,UAAM,cAAc,eAAe;AACnC,SAAK,OAAO,KAAK,cAAc,KAAK,MAAM,2BAA2B,WAAW,EAAE;AAGlF,UAAM,gBAAgB,KAAK,qBAAqB,QAAQ;AAGxD,UAAM,SAAS,KAAK,WAAW,MAAM,WAAW;AAChD,QAAI,iBAAiB;AACrB,UAAM,YAAY,KAAK;AAEvB,aAAS,aAAa,GAAG,aAAa,OAAO,QAAQ,cAAc;AAEjE,UAAI,QAAQ,SAAS;AACnB,aAAK,OAAO,KAAK,iCAAiC,aAAa,CAAC,IAAI,OAAO,MAAM,EAAE;AACnF;AAAA,MACF;AAEA,YAAM,QAAQ,OAAO,UAAU;AAC/B,YAAM,kBAAkB,aAAa;AAErC,YAAM,WAAW,MAAM,IAAI,OAAO,KAAK,UAAU;AAC/C,cAAM,WAAW,kBAAkB;AACnC,cAAM,YAAY,KAAK,kBAAkB,KAAK,SAAS,aAAa;AACpE,cAAM,eAAe,KAAK,IAAI;AAE9B,YAAI;AACF,gBAAM,SAAS,MAAM,KAAK,OAAO;AAAA,YAC/B;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACF;AACA,eAAK,QAAQ,cAAc,UAAU;AAGrC,qBAAW,eAAe;AAAA,YACxB;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,YAAY,KAAK,IAAI,IAAI;AAAA,UAC3B,CAAC;AAED,iBAAO,EAAE,SAAS,MAAM,QAAQ,KAAK,SAAS;AAAA,QAChD,SAAS,OAAO;AACd,eAAK,QAAQ,cAAc,UAAU;AAGrC,qBAAW,eAAe;AAAA,YACxB;AAAA,YACA;AAAA,YACA;AAAA,YACA,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AAAA,YAC/D,eAAe,aAAa,eAAe;AAAA,UAC7C,CAAC;AAED,iBAAO,EAAE,SAAS,OAAO,OAAO,KAAK,SAAS;AAAA,QAChD;AAAA,MACF,CAAC;AAED,YAAM,UAAU,MAAM,QAAQ,WAAW,QAAQ;AACjD,wBAAkB,MAAM;AAGxB,UAAI,QAAQ,SAAS;AACnB,aAAK,OAAO,KAAK,oCAAoC,aAAa,CAAC,IAAI,OAAO,MAAM,EAAE;AACtF;AAAA,MACF;AAGA,UAAI,iBAAiB;AACrB,UAAI,gBAAgB;AAEpB,cAAQ,QAAQ,CAAC,WAAW;AAC1B,YAAI,OAAO,WAAW,aAAa;AACjC,gBAAM,EAAE,SAAS,OAAO,IAAI,IAAI,OAAO;AACvC,cAAI,SAAS;AACX;AAAA,UACF,OAAO;AACL;AACA,iBAAK,OAAO,MAAM,0CAAqC,KAAK,KAAK;AAAA,UACnE;AAAA,QACF,OAAO;AACL;AACA,eAAK,OAAO,MAAM,4BAAuB,OAAO,MAAM,EAAE;AAAA,QAC1D;AAAA,MACF,CAAC;AAGD,YAAM,YAAa,iBAAiB,YAAa,KAAK,QAAQ,CAAC;AAC/D,WAAK,OAAO;AAAA,QACV,uBAAgB,cAAc,IAAI,SAAS,KAAK,QAAQ,cACtD,aAAa,CACf,KAAK,cAAc,YAAO,aAAa;AAAA,MACzC;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,WAAc,OAAY,WAA0B;AAC1D,UAAM,SAAgB,CAAC;AACvB,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,WAAW;AAChD,aAAO,KAAK,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,kBACN,KACA,SACA,eACqB;AACrB,UAAM,YAAiC,CAAC;AAExC,eAAW,CAAC,YAAY,YAAY,KAAK,OAAO,QAAQ,OAAO,GAAG;AAEhE,UAAI,iBAAiB,KAAK;AAExB,kBAAU,UAAU,IAAI;AAAA,MAC1B,WAES,OAAO,iBAAiB,YAAY,aAAa,WAAW,IAAI,GAAG;AAC1E,cAAMD,QAAO,aAAa,UAAU,CAAC;AACrC,cAAM,QAAQ,KAAK,eAAe,KAAKA,KAAI;AAC3C,YAAI,UAAU,QAAW;AACvB,gBAAM,OAAO,cAAc,UAAU;AACrC,oBAAU,UAAU,IAAI,KAAK,aAAa,OAAO,MAAM,UAAU;AAAA,QACnE;AAAA,MACF,WAES,OAAO,iBAAiB,YAAY,IAAI,YAAY,MAAM,QAAW;AAC5E,cAAM,WAAW,IAAI,YAAY;AACjC,cAAM,OAAO,cAAc,UAAU;AACrC,kBAAU,UAAU,IAAI,KAAK,aAAa,UAAU,MAAM,UAAU;AAAA,MACtE,WAES,OAAO,iBAAiB,YAAY,iBAAiB,MAAM;AAClE,kBAAU,UAAU,IAAI,KAAK,gBAAgB,KAAK,cAAc,aAAa;AAAA,MAC/E;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,eAAe,KAAUA,OAAmB;AAClD,UAAM,QAAQA,MAAK,MAAM,GAAG;AAC5B,QAAI,UAAU;AAEd,eAAW,QAAQ,OAAO;AACxB,UAAI,WAAW,OAAO,YAAY,YAAY,QAAQ,SAAS;AAC7D,kBAAU,QAAQ,IAAI;AAAA,MACxB,OAAO;AACL,eAAO;AAAA,MACT;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,gBACN,KACA,YACA,eACK;AACL,QAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,aAAO,WAAW,IAAI,CAAC,SAAS,KAAK,gBAAgB,KAAK,MAAM,aAAa,CAAC;AAAA,IAChF;AAEA,QAAI,OAAO,eAAe,YAAY,eAAe,MAAM;AACzD,YAAM,SAAc,CAAC;AACrB,iBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,UAAU,GAAG;AACrD,YAAI,OAAO,UAAU,YAAY,MAAM,WAAW,IAAI,GAAG;AACvD,gBAAMA,QAAO,MAAM,UAAU,CAAC;AAC9B,cAAI,aAAa,KAAK,eAAe,KAAKA,KAAI;AAG9C,cAAI,QAAQ,YAAY,OAAO,eAAe,YAAY,WAAW,SAAS,GAAG,GAAG;AAClF,yBAAa,WAAW,MAAM,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC;AAAA,UACxD;AAEA,iBAAO,GAAG,IAAI;AAAA,QAChB,WAAW,OAAO,UAAU,YAAY,IAAI,KAAK,MAAM,QAAW;AAChE,iBAAO,GAAG,IAAI,IAAI,KAAK;AAAA,QACzB,WAAW,OAAO,UAAU,UAAU;AACpC,iBAAO,GAAG,IAAI,KAAK,gBAAgB,KAAK,OAAO,aAAa;AAAA,QAC9D,OAAO;AACL,iBAAO,GAAG,IAAI;AAAA,QAChB;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAAqB,UAA0C;AACrE,UAAM,QAAgC,CAAC;AAEvC,QAAI;AACF,YAAM,WAAyB,MAAM,QAAQ;AAG7C,iBAAW,cAAc,SAAS,aAAa;AAC7C,YAAI,WAAW,SAAS,yBAAyB,WAAW,qBAAqB;AAC/E,qBAAW,eAAe,WAAW,qBAAqB;AACxD,kBAAM,UAAU,YAAY,SAAS,KAAK;AAC1C,kBAAM,WAAW,KAAK,gBAAgB,WAAW;AACjD,gBAAI,UAAU;AACZ,oBAAM,OAAO,IAAI;AAAA,YACnB;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,YAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACrE,WAAK,OAAO,MAAM,mCAAmC,OAAO,EAAE;AAAA,IAChE;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,gBAAgB,aAAoD;AAC1E,UAAM,OAAO,YAAY;AAEzB,QAAI,KAAK,SAAS,eAAe;AAE/B,UAAI,KAAK,KAAK,SAAS,aAAa;AAClC,eAAO,KAAK,KAAK,KAAK;AAAA,MACxB;AAAA,IACF,WAAW,KAAK,SAAS,aAAa;AAEpC,aAAO,KAAK,KAAK;AAAA,IACnB;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,OAAY,MAA0B,SAAsB;AAC/E,QAAI,CAAC,MAAM;AAET,aAAO;AAAA,IACT;AAGA,QAAI,OAAO,UAAU,UAAU;AAC7B,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,MAAM,KAAK;AAEhC,YAAQ,MAAM;AAAA,MACZ,KAAK;AACH,cAAM,WAAW,OAAO,YAAY;AAEpC,YAAI,MAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,KAAK,CAAC,OAAO,UAAU,QAAQ,GAAG;AACzE,eAAK,OAAO;AAAA,YACV,4BAA4B,KAAK,0BAA0B,OAAO;AAAA,UACpE;AACA,iBAAO;AAAA,QACT;AACA,eAAO;AAAA,MAET,KAAK;AACH,cAAM,aAAa,OAAO,YAAY;AAEtC,YAAI,MAAM,UAAU,KAAK,CAAC,SAAS,UAAU,GAAG;AAC9C,eAAK,OAAO;AAAA,YACV,4BAA4B,KAAK,4BAA4B,OAAO;AAAA,UACtE;AACA,iBAAO;AAAA,QACT;AACA,eAAO;AAAA,MAET,KAAK;AACH,cAAM,aAAa,aAAa,YAAY;AAC5C,YAAI,eAAe,UAAU,eAAe,IAAK,QAAO;AACxD,YAAI,eAAe,WAAW,eAAe,IAAK,QAAO;AACzD,aAAK,OAAO;AAAA,UACV,4BAA4B,KAAK,8BAA8B,OAAO;AAAA,QACxE;AACA,eAAO;AAAA,MAET,KAAK;AACH,eAAO;AAAA,MAET;AAEE,aAAK,OAAO;AAAA,UACV,yBAAyB,IAAI,mBAAmB,OAAO;AAAA,QACzD;AACA,eAAO;AAAA,IACX;AAAA,EACF;AAAA,EAEA,aAA+B;AAC7B,WAAO,KAAK;AAAA,EACd;AACF;;;AQvjBO,IAAM,qBAAN,MAAyB;AAAA,EAK9B,YACE,UACA,eAAgC,CAAC,GACjC,yBAAkC,OAClC;AACA,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,yBAAyB;AAAA,EAChC;AAAA,EAEA,wBAAyC;AACvC,UAAM,QAAyB,CAAC;AAChC,UAAM,YAAY,oBAAI,IAAY;AAClC,UAAM,aAAa,oBAAI,IAAY;AACnC,QAAI,aAAa;AAEjB,WAAO,UAAU,OAAO,KAAK,SAAS,QAAQ;AAC5C,YAAM,cAAwB,CAAC;AAE/B,iBAAW,UAAU,KAAK,UAAU;AAClC,YAAI,UAAU,IAAI,MAAM,KAAK,WAAW,IAAI,MAAM,GAAG;AACnD;AAAA,QACF;AAEA,cAAM,OAAO,KAAK,aAAa,MAAM,KAAK,CAAC;AAC3C,cAAM,aAAa,KAAK;AAAA,UACtB,CAAC,QACC,UAAU,IAAI,GAAG,KAAM,KAAK,0BAA0B,CAAC,KAAK,SAAS,SAAS,GAAG;AAAA,QACrF;AAEA,YAAI,YAAY;AACd,sBAAY,KAAK,MAAM;AACvB,qBAAW,IAAI,MAAM;AAAA,QACvB;AAAA,MACF;AAEA,UAAI,YAAY,WAAW,GAAG;AAC5B,cAAM,YAAY,KAAK,SAAS,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;AAC/D,cAAM,IAAI;AAAA,UACR,sEAAsE,UAAU;AAAA,YAC9E;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAEA,YAAM,KAAK;AAAA,QACT,UAAU;AAAA,QACV,MAAM;AAAA,MACR,CAAC;AAED,kBAAY,QAAQ,CAAC,WAAW;AAC9B,kBAAU,IAAI,MAAM;AACpB,mBAAW,OAAO,MAAM;AAAA,MAC1B,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,uBAAiC;AAC/B,UAAM,SAAmB,CAAC;AAC1B,UAAM,YAAY,IAAI,IAAI,KAAK,QAAQ;AAEvC,eAAW,CAAC,QAAQ,IAAI,KAAK,OAAO,QAAQ,KAAK,YAAY,GAAG;AAC9D,UAAI,CAAC,UAAU,IAAI,MAAM,GAAG;AAC1B,eAAO,KAAK,WAAW,MAAM,kDAAkD;AAC/E;AAAA,MACF;AAEA,iBAAW,OAAO,MAAM;AACtB,YAAI,CAAC,UAAU,IAAI,GAAG,GAAG;AACvB,iBAAO,KAAK,WAAW,MAAM,iBAAiB,GAAG,wBAAwB;AAAA,QAC3E;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,cAAc,YAA8B;AAC1C,WAAO,OAAO,QAAQ,KAAK,YAAY,EACpC,OAAO,CAAC,CAAC,GAAG,IAAI,MAAM,KAAK,SAAS,UAAU,CAAC,EAC/C,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,MAAM;AAAA,EAChC;AAAA,EAEA,gBAAgB,YAA8B;AAC5C,WAAO,KAAK,aAAa,UAAU,KAAK,CAAC;AAAA,EAC3C;AACF;;;AZ/FA,SAAS,gBAAgB;;;AaoLlB,IAAM,wBAAgD;AAAA,EAC3D,eAAe;AAAA,EACf,kBAAkB;AAAA,EAClB,oBAAoB;AACtB;;;Ab9HO,IAAM,YAAN,cAAwB,aAAa;AAAA,EAuB1C,YAAY,SAA2B;AACrC,UAAM;AAbR;AAAA,SAAQ,kBAA0C;AAClD,SAAQ,eAAe;AACvB,SAAQ,YAAY;AACpB,SAAQ,qBAA4D;AAGpE;AAAA,SAAQ,cAAc;AACtB,SAAQ,aAAa;AACrB,SAAQ,oBAAoB;AAC5B,SAAQ,gBAAgB;AAKtB,SAAK,WAAW,QAAQ;AACxB,SAAK,UAAU,QAAQ,WAAW,CAAC;AACnC,SAAK,SAAS,QAAQ,UAAU;AAChC,SAAK,iBAAiB,QAAQ;AAC9B,SAAK,eAAe,EAAE,GAAG,uBAAuB,GAAG,QAAQ,aAAa;AAGxE,SAAK,UAAU,IAAI,iBAAiB;AACpC,SAAK,SAAS,IAAI,qBAAqB,KAAK,UAAU,KAAK,SAAS,KAAK,SAAS,KAAK,MAAM;AAC7F,SAAK,SAAS,IAAI;AAAA,MAChB,KAAK;AAAA,MACL,QAAQ,IAAI;AAAA,MACZ,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,OAAO,SAAS,+BAAqC;AACnD,QAAI,KAAK,mBAAmB,KAAK,cAAc;AAC7C,WAAK,gBAAgB,MAAM,MAAM;AAAA,IACnC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,IAAI,aAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAKQ,SACN,OACA,SACS;AACT,QAAI;AACF,aAAO,KAAK,KAAK,OAAO,OAAO;AAAA,IACjC,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,gCAAgC,OAAO,KAAK,CAAC,MAAM,KAAK;AAC1E,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,wBAA8B;AACpC,QAAI,CAAC,KAAK,aAAa,mBAAoB;AAE3C,SAAK,qBAAqB,YAAY,MAAM;AAC1C,WAAK,kBAAkB;AAAA,IACzB,GAAG,KAAK,aAAa,gBAAgB;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA,EAKQ,uBAA6B;AACnC,QAAI,KAAK,oBAAoB;AAC3B,oBAAc,KAAK,kBAAkB;AACrC,WAAK,qBAAqB;AAAA,IAC5B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,oBAA0B;AAChC,UAAM,UAAU,KAAK,QAAQ,WAAW;AACxC,UAAM,UAAgC;AAAA,MACpC,aAAa,KAAK;AAAA,MAClB,YAAY,KAAK;AAAA,MACjB,mBAAmB,KAAK;AAAA,MACxB,eAAe,KAAK;AAAA,MACpB,eAAe,QAAQ;AAAA,MACvB,gBAAgB,QAAQ;AAAA,MACxB,YAAY,QAAQ;AAAA,MACpB,iBACE,KAAK,gBAAgB,IAAK,KAAK,oBAAoB,KAAK,gBAAiB,MAAM;AAAA,MACjF,WAAW,KAAK,IAAI,IAAI,KAAK;AAAA,IAC/B;AACA,SAAK,SAAS,YAAY,OAAO;AAAA,EACnC;AAAA;AAAA;AAAA;AAAA,EAKQ,mBAAmB,QAA8B;AACvD,SAAK,qBAAqB;AAE1B,UAAM,UAAiC;AAAA,MACrC,QAAQ,UAAU;AAAA,MAClB,SAAS,KAAK,QAAQ,WAAW;AAAA,MACjC,eAAe,KAAK;AAAA,MACpB,WAAW,KAAK,IAAI,IAAI,KAAK;AAAA,IAC/B;AACA,SAAK,SAAS,aAAa,OAAO;AAElC,WAAO;AAAA,MACL,SAAS,KAAK,QAAQ,WAAW;AAAA,MACjC,SAAS;AAAA,MACT,WAAW;AAAA,MACX,QAAQ,CAAC,wBAAwB,MAAM,EAAE;AAAA,IAC3C;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,kBAAkB,SAAqC;AAC7D,UAAM,aAAa,IAAI,gBAAgB;AAEvC,eAAW,UAAU,SAAS;AAC5B,UAAI,OAAO,SAAS;AAClB,mBAAW,MAAM,OAAO,MAAM;AAC9B;AAAA,MACF;AACA,aAAO,iBAAiB,SAAS,MAAM,WAAW,MAAM,OAAO,MAAM,GAAG;AAAA,QACtE,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAEA,WAAO,WAAW;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,OAAO,YAAoB,SAAgD;AAC/E,UAAM,SAAmB,CAAC;AAG1B,SAAK,kBAAkB,IAAI,gBAAgB;AAC3C,UAAM,SAAS,SAAS,SACpB,KAAK,eAAe,QAAQ,QAAQ,KAAK,gBAAgB,MAAM,IAC/D,KAAK,gBAAgB;AAEzB,SAAK,eAAe;AACpB,SAAK,YAAY,KAAK,IAAI;AAC1B,SAAK,oBAAoB;AACzB,SAAK,cAAc;AACnB,SAAK,gBAAgB;AAErB,QAAI;AAEF,UAAI,OAAO,SAAS;AAClB,eAAO,KAAK,mBAAmB,OAAO,MAAM;AAAA,MAC9C;AAGA,WAAK,UAAU,IAAI,iBAAiB;AAGpC,WAAK,SAAS,IAAI;AAAA,QAChB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,MACP;AAEA,WAAK,SAAS,IAAI;AAAA,QAChB,KAAK;AAAA,QACL,QAAQ,IAAI;AAAA,QACZ,KAAK;AAAA,QACL,KAAK;AAAA,QACL,SAAS,UAAU,KAAK;AAAA,MAC1B;AAGA,YAAM,SAAS,WAAW,YAAY,KAAK,MAAM;AAGjD,UAAI;AACJ,UAAI,SAAS,UAAU;AACrB,YAAI,OAAO,QAAQ,aAAa,UAAU;AACxC,yBAAe,QAAQ,SAAS,MAAM,GAAG,EAAE,IAAI,CAAC,MAAc,EAAE,KAAK,CAAC;AAAA,QACxE,OAAO;AACL,yBAAe,QAAQ;AAAA,QACzB;AAAA,MACF;AAGA,YAAM,eAAe,KAAK,OAAO,iBAAiB,YAAY,YAAY;AAE1E,UAAI,aAAa,WAAW,GAAG;AAC7B,cAAM,YAAY,eAAe,uBAAuB,aAAa,KAAK,IAAI,CAAC,KAAK;AACpF,cAAM,UAAU,6BAA6B,UAAU,YAAY,SAAS;AAC5E,aAAK,OAAO,KAAK,OAAO;AACxB,eAAO;AAAA,UACL,SAAS,KAAK,QAAQ,WAAW;AAAA,UACjC,SAAS;AAAA,UACT,QAAQ,CAAC,OAAO;AAAA,QAClB;AAAA,MACF;AAGA,YAAM,cAAc,aAAa,IAAI,CAACE,UAAS,SAASA,OAAM,OAAO,CAAC;AACtE,WAAK,gBAAgB,YAAY;AAGjC,YAAM,uBAAiD,CAAC;AACxD,UAAI,OAAO,oBAAoB;AAC7B,mBAAW,UAAU,aAAa;AAChC,cAAI,OAAO,mBAAmB,MAAM,GAAG;AACrC,iCAAqB,MAAM,IAAI,OAAO,mBAAmB,MAAM;AAAA,UACjE;AAAA,QACF;AAAA,MACF;AAGA,YAAM,WAAW,IAAI;AAAA,QACnB;AAAA,QACA;AAAA,QACA,CAAC,CAAC;AAAA;AAAA,MACJ;AAGA,YAAM,mBAAmB,SAAS,qBAAqB;AACvD,UAAI,iBAAiB,SAAS,GAAG;AAC/B,YAAI,cAAc;AAEhB,eAAK,OAAO,KAAK,wDAA8C;AAC/D,2BAAiB,QAAQ,CAAC,UAAU,KAAK,OAAO,KAAK,OAAO,KAAK,EAAE,CAAC;AACpE,eAAK,OAAO,KAAK,sEAAsE;AAAA,QACzF,OAAO;AAEL,eAAK,OAAO,MAAM,+BAA+B;AACjD,2BAAiB,QAAQ,CAAC,UAAU;AAClC,iBAAK,OAAO,MAAM,OAAO,KAAK,EAAE;AAChC,mBAAO,KAAK,KAAK;AAAA,UACnB,CAAC;AACD,iBAAO;AAAA,YACL,SAAS,KAAK,QAAQ,WAAW;AAAA,YACjC,SAAS;AAAA,YACT;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAEA,YAAM,QAAQ,SAAS,sBAAsB;AAC7C,WAAK,aAAa,MAAM;AAGxB,YAAM,iBAAsC;AAAA,QAC1C;AAAA,QACA,eAAe,YAAY;AAAA,QAC3B;AAAA,QACA,YAAY,MAAM;AAAA,QAClB,WAAW,KAAK;AAAA,MAClB;AACA,WAAK,SAAS,WAAW,cAAc;AAGvC,WAAK,sBAAsB;AAG3B,YAAM,KAAK;AAAA,QACT;AAAA,QACA;AAAA,QACA,KAAK;AAAA,QACL;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACF;AAGA,UAAI,OAAO,SAAS;AAClB,eAAO,KAAK,mBAAmB,OAAO,MAAM;AAAA,MAC9C;AAEA,WAAK,QAAQ,iBAAiB;AAC9B,WAAK,qBAAqB;AAE1B,YAAM,eAAe,KAAK,QAAQ,WAAW;AAC7C,YAAM,gBAAgB,aAAa,qBAAqB;AAGxD,YAAM,kBAAwC;AAAA,QAC5C,SAAS;AAAA,QACT,YAAY,KAAK,IAAI,IAAI,KAAK;AAAA,QAC9B;AAAA,MACF;AACA,WAAK,SAAS,YAAY,eAAe;AAEzC,aAAO;AAAA,QACL,SAAS;AAAA,QACT,SAAS;AAAA,MACX;AAAA,IACF,SAAS,OAAO;AACd,WAAK,qBAAqB;AAE1B,UAAI,OAAO,SAAS;AAClB,eAAO,KAAK,mBAAmB,OAAO,MAAM;AAAA,MAC9C;AAEA,YAAM,eAAe,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC1E,WAAK,OAAO,MAAM,UAAU,YAAY,EAAE;AAC1C,aAAO,KAAK,YAAY;AAGxB,YAAM,iBAAsC;AAAA,QAC1C,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AAAA,QAC/D,SAAS,KAAK,QAAQ,WAAW;AAAA,QACjC,eAAe,KAAK;AAAA,QACpB,WAAW,KAAK,IAAI,IAAI,KAAK;AAAA,MAC/B;AACA,WAAK,SAAS,WAAW,cAAc;AAEvC,aAAO;AAAA,QACL,SAAS,KAAK,QAAQ,WAAW;AAAA,QACjC,SAAS;AAAA,QACT;AAAA,MACF;AAAA,IACF,UAAE;AACA,WAAK,eAAe;AACpB,WAAK,kBAAkB;AACvB,WAAK,qBAAqB;AAAA,IAC5B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,eAAe,YAAoB,UAA2C;AAClF,WAAO,KAAK,OAAO,YAAY,EAAE,SAAS,CAAC;AAAA,EAC7C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,aAAgC;AAC9B,WAAO,KAAK,QAAQ,WAAW;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,oBAA4B;AAC1B,WAAO,KAAK,QAAQ,gBAAgB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,YAAkC;AAChC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,YAAwB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAU,QAAsB;AAC9B,SAAK,SAAS;AACd,SAAK,SAAS,IAAI,qBAAqB,KAAK,UAAU,KAAK,SAAS,KAAK,SAAS,MAAM;AACxF,SAAK,SAAS,IAAI;AAAA,MAChB,KAAK;AAAA,MACL,QAAQ,IAAI;AAAA,MACZ,KAAK;AAAA,MACL;AAAA,MACA,KAAK;AAAA,IACP;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,WAAW,SAAuC;AAChD,SAAK,UAAU;AACf,SAAK,SAAS,IAAI,qBAAqB,KAAK,UAAU,SAAS,KAAK,SAAS,KAAK,MAAM;AACxF,SAAK,SAAS,IAAI;AAAA,MAChB,KAAK;AAAA,MACL,QAAQ,IAAI;AAAA,MACZ,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,uBACZ,cACA,UACA,QACA,QACA,QACA,QACe;AACf,UAAM,QAAQ,SAAS,sBAAsB;AAC7C,UAAM,UAAU,IAAI,IAAI,aAAa,IAAI,CAACA,UAAS,CAAC,SAASA,OAAM,OAAO,GAAGA,KAAI,CAAC,CAAC;AAEnF,WAAO,KAAK,cAAc,MAAM,MAAM,sBAAsB;AAE5D,eAAW,QAAQ,OAAO;AAExB,UAAI,QAAQ,SAAS;AACnB;AAAA,MACF;AAEA,WAAK,cAAc,KAAK;AACxB,aAAO,KAAK,QAAQ,KAAK,OAAO,CAAC,0BAA0B,KAAK,SAAS,KAAK,IAAI,CAAC,GAAG;AAGtF,YAAM,oBAAoB,OAAO,mBAAmB;AACpD,YAAM,SAAS,KAAK,WAAW,KAAK,UAAU,iBAAiB;AAE/D,iBAAW,SAAS,QAAQ;AAE1B,YAAI,QAAQ,SAAS;AACnB;AAAA,QACF;AAEA,cAAM,iBAAiB,MAAM,IAAI,OAAO,eAAe;AAErD,cAAI,QAAQ,SAAS;AACnB;AAAA,UACF;AAEA,gBAAM,aAAa,QAAQ,IAAI,UAAU;AACzC,cAAI,YAAY;AACd,iBAAK,gBAAgB;AACrB,gBAAI;AACF,oBAAM,eAAe,gBAAgB,YAAY,QAAQ,MAAM;AAC/D,oBAAM,cAAc,eAAe,YAAY,MAAM;AAGrD,oBAAM,OAAO,wBAAwB,YAAY,cAAc,aAAa,QAAQ;AAAA,gBAClF,eAAe,CAAC,YACd,KAAK,SAAS,eAAe;AAAA,kBAC3B,GAAG;AAAA,kBACH,WAAW,KAAK;AAAA,gBAClB,CAAC;AAAA,gBACH,kBAAkB,CAAC,YAAY;AAC7B,uBAAK;AACL,uBAAK,SAAS,kBAAkB,OAAO;AAAA,gBACzC;AAAA,gBACA,cAAc,KAAK,aAAa,gBAC5B,CAAC,YAAY,KAAK,SAAS,cAAc,OAAO,IAChD;AAAA,gBACJ,cAAc,KAAK,aAAa,gBAC5B,CAAC,YAAY,KAAK,SAAS,cAAc,OAAO,IAChD;AAAA,cACN,CAAC;AAAA,YACH,SAAS,OAAO;AACd,oBAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACrE,qBAAO,KAAK,8BAA8B,UAAU,KAAK,OAAO,EAAE;AAAA,YACpE;AAAA,UACF;AAAA,QACF,CAAC;AAED,cAAM,QAAQ,WAAW,cAAc;AAAA,MACzC;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,WAAc,OAAY,WAA0B;AAC1D,QAAI,aAAa,EAAG,QAAO,CAAC,KAAK;AACjC,UAAM,SAAgB,CAAC;AACvB,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,WAAW;AAChD,aAAO,KAAK,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AACF;",
|
|
6
|
+
"names": ["totalDuration", "fs", "path", "reader", "fs", "fs", "fs", "fs", "fs", "yaml", "fs", "yaml", "fs", "fs", "path", "fs", "path"]
|
|
7
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Logger } from "./logger";
|
|
1
2
|
export interface ParallelProcessingConfig {
|
|
2
3
|
concurrency: number;
|
|
3
4
|
entityConcurrency: number;
|
|
@@ -26,7 +27,7 @@ export interface FullConfig extends ProcessingConfig {
|
|
|
26
27
|
export declare const DEFAULT_RETRY_CONFIG: RetryConfig;
|
|
27
28
|
export declare const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig;
|
|
28
29
|
export declare const DEFAULT_CONFIG: ProcessingConfig;
|
|
29
|
-
export declare function loadConfig(configDir: string): ProcessingConfig;
|
|
30
|
-
export declare function getEntityConfig(entityName: string, globalConfig: ProcessingConfig): ParallelProcessingConfig;
|
|
30
|
+
export declare function loadConfig(configDir: string, logger?: Logger): ProcessingConfig;
|
|
31
|
+
export declare function getEntityConfig(entityName: string, globalConfig: ProcessingConfig, logger?: Logger): ParallelProcessingConfig;
|
|
31
32
|
export declare function getRetryConfig(entityName: string, globalConfig: ProcessingConfig): RetryConfig;
|
|
32
33
|
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAE9C,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,oBAAoB,EAAE,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,WAAW,CAAC;IACnB,kBAAkB,EAAE,wBAAwB,CAAC;IAC7C,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC3C,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,UAAW,SAAQ,gBAAgB;CAEnD;AAED,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,wBAIrC,CAAC;AAEF,eAAO,MAAM,cAAc,EAAE,gBAK5B,CAAC;AAEF,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,GAAE,MAAmB,GAAG,gBAAgB,CAkB3F;AAiBD,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,gBAAgB,EAC9B,MAAM,GAAE,MAAmB,GAC1B,wBAAwB,CAiB1B;AAED,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,GAAG,WAAW,CAO9F"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dependency-resolver.d.ts","sourceRoot":"","sources":["../../src/lib/dependency-resolver.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,sBAAsB,CAAU;gBAGtC,QAAQ,EAAE,MAAM,EAAE,EAClB,YAAY,GAAE,eAAoB,EAClC,sBAAsB,GAAE,OAAe;IAOzC,qBAAqB,IAAI,aAAa,EAAE;IAiDxC,oBAAoB,IAAI,MAAM,EAAE;IAoBhC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;IAM3C,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;CAG9C"}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { ProcessingMetrics, EntityMetrics } from "./metrics";
|
|
2
|
+
import { DataRow } from "../readers";
|
|
3
|
+
/**
|
|
4
|
+
* Event types emitted by GQLIngest during processing
|
|
5
|
+
*/
|
|
6
|
+
export type GQLIngestEventType = "started" | "progress" | "entityStart" | "entityComplete" | "rowSuccess" | "rowFailure" | "cancelled" | "finished" | "errored";
|
|
7
|
+
/**
|
|
8
|
+
* Payload for 'started' event - emitted when ingestion begins
|
|
9
|
+
*/
|
|
10
|
+
export interface StartedEventPayload {
|
|
11
|
+
/** Path to configuration directory */
|
|
12
|
+
configPath: string;
|
|
13
|
+
/** Total number of entities to process */
|
|
14
|
+
totalEntities: number;
|
|
15
|
+
/** Names of entities that will be processed */
|
|
16
|
+
entityNames: string[];
|
|
17
|
+
/** Number of dependency waves */
|
|
18
|
+
totalWaves: number;
|
|
19
|
+
/** Timestamp when ingestion started */
|
|
20
|
+
startTime: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Payload for 'progress' event - emitted periodically during processing
|
|
24
|
+
*/
|
|
25
|
+
export interface ProgressEventPayload {
|
|
26
|
+
/** Current wave number (0-indexed) */
|
|
27
|
+
currentWave: number;
|
|
28
|
+
/** Total number of waves */
|
|
29
|
+
totalWaves: number;
|
|
30
|
+
/** Number of entities completed */
|
|
31
|
+
entitiesCompleted: number;
|
|
32
|
+
/** Total number of entities */
|
|
33
|
+
totalEntities: number;
|
|
34
|
+
/** Total rows processed so far */
|
|
35
|
+
rowsProcessed: number;
|
|
36
|
+
/** Number of successful row operations */
|
|
37
|
+
successfulRows: number;
|
|
38
|
+
/** Number of failed row operations */
|
|
39
|
+
failedRows: number;
|
|
40
|
+
/** Progress percentage (0-100) */
|
|
41
|
+
progressPercent: number;
|
|
42
|
+
/** Elapsed time in milliseconds */
|
|
43
|
+
elapsedMs: number;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Payload for 'entityStart' event - emitted when an entity begins processing
|
|
47
|
+
*/
|
|
48
|
+
export interface EntityStartEventPayload {
|
|
49
|
+
/** Name of the entity being processed */
|
|
50
|
+
entityName: string;
|
|
51
|
+
/** Path to the entity's mapping configuration */
|
|
52
|
+
mappingPath: string;
|
|
53
|
+
/** Total number of rows to process for this entity */
|
|
54
|
+
totalRows: number;
|
|
55
|
+
/** Wave number this entity belongs to */
|
|
56
|
+
waveIndex: number;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Payload for 'entityComplete' event - emitted when an entity finishes processing
|
|
60
|
+
*/
|
|
61
|
+
export interface EntityCompleteEventPayload {
|
|
62
|
+
/** Name of the entity that completed */
|
|
63
|
+
entityName: string;
|
|
64
|
+
/** Metrics for this entity */
|
|
65
|
+
metrics: EntityMetrics;
|
|
66
|
+
/** Whether all rows were successful */
|
|
67
|
+
success: boolean;
|
|
68
|
+
/** Duration in milliseconds */
|
|
69
|
+
durationMs: number;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Payload for 'rowSuccess' event - emitted for each successful row mutation
|
|
73
|
+
*/
|
|
74
|
+
export interface RowSuccessEventPayload {
|
|
75
|
+
/** Entity name */
|
|
76
|
+
entityName: string;
|
|
77
|
+
/** Row index (0-indexed) */
|
|
78
|
+
rowIndex: number;
|
|
79
|
+
/** The original row data */
|
|
80
|
+
row: DataRow;
|
|
81
|
+
/** GraphQL mutation result */
|
|
82
|
+
result: unknown;
|
|
83
|
+
/** Duration of the mutation in milliseconds */
|
|
84
|
+
durationMs: number;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Payload for 'rowFailure' event - emitted for each failed row mutation
|
|
88
|
+
*/
|
|
89
|
+
export interface RowFailureEventPayload {
|
|
90
|
+
/** Entity name */
|
|
91
|
+
entityName: string;
|
|
92
|
+
/** Row index (0-indexed) */
|
|
93
|
+
rowIndex: number;
|
|
94
|
+
/** The original row data */
|
|
95
|
+
row: DataRow;
|
|
96
|
+
/** Error that occurred */
|
|
97
|
+
error: Error;
|
|
98
|
+
/** Number of retry attempts made */
|
|
99
|
+
retryAttempts: number;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Payload for 'cancelled' event - emitted when processing is cancelled
|
|
103
|
+
*/
|
|
104
|
+
export interface CancelledEventPayload {
|
|
105
|
+
/** Reason for cancellation */
|
|
106
|
+
reason: string;
|
|
107
|
+
/** Metrics at time of cancellation */
|
|
108
|
+
metrics: ProcessingMetrics;
|
|
109
|
+
/** Entity being processed when cancelled (if any) */
|
|
110
|
+
currentEntity?: string;
|
|
111
|
+
/** How long processing ran before cancellation */
|
|
112
|
+
elapsedMs: number;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Payload for 'finished' event - emitted when processing completes successfully
|
|
116
|
+
*/
|
|
117
|
+
export interface FinishedEventPayload {
|
|
118
|
+
/** Final processing metrics */
|
|
119
|
+
metrics: ProcessingMetrics;
|
|
120
|
+
/** Total duration in milliseconds */
|
|
121
|
+
durationMs: number;
|
|
122
|
+
/** Whether all entities succeeded */
|
|
123
|
+
allSuccessful: boolean;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Payload for 'errored' event - emitted when a fatal error occurs
|
|
127
|
+
*/
|
|
128
|
+
export interface ErroredEventPayload {
|
|
129
|
+
/** The error that occurred */
|
|
130
|
+
error: Error;
|
|
131
|
+
/** Metrics at time of error */
|
|
132
|
+
metrics: ProcessingMetrics;
|
|
133
|
+
/** Entity being processed when error occurred (if any) */
|
|
134
|
+
currentEntity?: string;
|
|
135
|
+
/** How long processing ran before error */
|
|
136
|
+
elapsedMs: number;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Map of event types to their payload types
|
|
140
|
+
*/
|
|
141
|
+
export interface GQLIngestEventMap {
|
|
142
|
+
started: StartedEventPayload;
|
|
143
|
+
progress: ProgressEventPayload;
|
|
144
|
+
entityStart: EntityStartEventPayload;
|
|
145
|
+
entityComplete: EntityCompleteEventPayload;
|
|
146
|
+
rowSuccess: RowSuccessEventPayload;
|
|
147
|
+
rowFailure: RowFailureEventPayload;
|
|
148
|
+
cancelled: CancelledEventPayload;
|
|
149
|
+
finished: FinishedEventPayload;
|
|
150
|
+
errored: ErroredEventPayload;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Options for configuring event emission behavior
|
|
154
|
+
*/
|
|
155
|
+
export interface EventOptions {
|
|
156
|
+
/** Emit rowSuccess/rowFailure events (can be verbose for large datasets). Default: true */
|
|
157
|
+
emitRowEvents?: boolean;
|
|
158
|
+
/** Interval for progress events in milliseconds. Default: 1000 (1 second) */
|
|
159
|
+
progressInterval?: number;
|
|
160
|
+
/** Emit progress events. Default: true */
|
|
161
|
+
emitProgressEvents?: boolean;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Default event options
|
|
165
|
+
*/
|
|
166
|
+
export declare const DEFAULT_EVENT_OPTIONS: Required<EventOptions>;
|
|
167
|
+
//# sourceMappingURL=events.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/lib/events.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B,SAAS,GACT,UAAU,GACV,aAAa,GACb,gBAAgB,GAChB,YAAY,GACZ,YAAY,GACZ,WAAW,GACX,UAAU,GACV,SAAS,CAAC;AAEd;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,aAAa,EAAE,MAAM,CAAC;IACtB,+CAA+C;IAC/C,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,sCAAsC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,+BAA+B;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,cAAc,EAAE,MAAM,CAAC;IACvB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,eAAe,EAAE,MAAM,CAAC;IACxB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,OAAO,EAAE,aAAa,CAAC;IACvB,uCAAuC;IACvC,OAAO,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,GAAG,EAAE,OAAO,CAAC;IACb,8BAA8B;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,GAAG,EAAE,OAAO,CAAC;IACb,0BAA0B;IAC1B,KAAK,EAAE,KAAK,CAAC;IACb,oCAAoC;IACpC,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,8BAA8B;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,qDAAqD;IACrD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,+BAA+B;IAC/B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,qCAAqC;IACrC,aAAa,EAAE,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,8BAA8B;IAC9B,KAAK,EAAE,KAAK,CAAC;IACb,+BAA+B;IAC/B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,mBAAmB,CAAC;IAC7B,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,WAAW,EAAE,uBAAuB,CAAC;IACrC,cAAc,EAAE,0BAA0B,CAAC;IAC3C,UAAU,EAAE,sBAAsB,CAAC;IACnC,UAAU,EAAE,sBAAsB,CAAC;IACnC,SAAS,EAAE,qBAAqB,CAAC;IACjC,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,OAAO,EAAE,mBAAmB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,2FAA2F;IAC3F,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,6EAA6E;IAC7E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0CAA0C;IAC1C,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,QAAQ,CAAC,YAAY,CAIxD,CAAC"}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { EventEmitter } from "events";
|
|
2
|
+
import { GraphQLClientWrapper } from "./graphql-client";
|
|
3
|
+
import { DataMapper } from "./mapper";
|
|
4
|
+
import { ProcessingMetrics } from "./metrics";
|
|
5
|
+
import { Logger } from "./logger";
|
|
6
|
+
import { EventOptions } from "./events";
|
|
7
|
+
/**
|
|
8
|
+
* Options for initializing GQLIngest client
|
|
9
|
+
*/
|
|
10
|
+
export interface GQLIngestOptions {
|
|
11
|
+
/** GraphQL endpoint URL */
|
|
12
|
+
endpoint: string;
|
|
13
|
+
/** Optional headers to include in GraphQL requests */
|
|
14
|
+
headers?: Record<string, string>;
|
|
15
|
+
/** Logger instance. Defaults to silent no-op logger. */
|
|
16
|
+
logger?: Logger;
|
|
17
|
+
/** Override data format detection (csv, json, yaml, jsonl) */
|
|
18
|
+
formatOverride?: string;
|
|
19
|
+
/** Event emission options */
|
|
20
|
+
eventOptions?: EventOptions;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Options for ingesting data
|
|
24
|
+
*/
|
|
25
|
+
export interface IngestOptions {
|
|
26
|
+
/** Comma-separated list or array of specific entities to process */
|
|
27
|
+
entities?: string[] | string;
|
|
28
|
+
/** Override data format detection for this operation */
|
|
29
|
+
format?: string;
|
|
30
|
+
/** AbortSignal for external cancellation */
|
|
31
|
+
signal?: AbortSignal;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Result of an ingestion operation
|
|
35
|
+
*/
|
|
36
|
+
export interface IngestResult {
|
|
37
|
+
/** Processing metrics */
|
|
38
|
+
metrics: ProcessingMetrics;
|
|
39
|
+
/** Whether the operation was successful */
|
|
40
|
+
success: boolean;
|
|
41
|
+
/** Any errors that occurred */
|
|
42
|
+
errors?: string[];
|
|
43
|
+
/** Whether the operation was cancelled */
|
|
44
|
+
cancelled?: boolean;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Main class for programmatic access to gql-ingest functionality.
|
|
48
|
+
* Extends EventEmitter for progress monitoring and cancellation support.
|
|
49
|
+
*/
|
|
50
|
+
export declare class GQLIngest extends EventEmitter {
|
|
51
|
+
private endpoint;
|
|
52
|
+
private headers;
|
|
53
|
+
private logger;
|
|
54
|
+
private formatOverride?;
|
|
55
|
+
private metrics;
|
|
56
|
+
private client;
|
|
57
|
+
private mapper;
|
|
58
|
+
private eventOptions;
|
|
59
|
+
private abortController;
|
|
60
|
+
private isProcessing;
|
|
61
|
+
private startTime;
|
|
62
|
+
private progressIntervalId;
|
|
63
|
+
private currentWave;
|
|
64
|
+
private totalWaves;
|
|
65
|
+
private entitiesCompleted;
|
|
66
|
+
private totalEntities;
|
|
67
|
+
private currentEntity;
|
|
68
|
+
constructor(options: GQLIngestOptions);
|
|
69
|
+
/**
|
|
70
|
+
* Cancel the current ingestion process
|
|
71
|
+
* @param reason Optional reason for cancellation
|
|
72
|
+
*/
|
|
73
|
+
cancel(reason?: string): void;
|
|
74
|
+
/**
|
|
75
|
+
* Check if processing is currently in progress
|
|
76
|
+
*/
|
|
77
|
+
get processing(): boolean;
|
|
78
|
+
/**
|
|
79
|
+
* Safely emit an event, catching any errors from listeners
|
|
80
|
+
*/
|
|
81
|
+
private safeEmit;
|
|
82
|
+
/**
|
|
83
|
+
* Start the progress interval timer
|
|
84
|
+
*/
|
|
85
|
+
private startProgressInterval;
|
|
86
|
+
/**
|
|
87
|
+
* Stop the progress interval timer
|
|
88
|
+
*/
|
|
89
|
+
private stopProgressInterval;
|
|
90
|
+
/**
|
|
91
|
+
* Emit a progress event with current state
|
|
92
|
+
*/
|
|
93
|
+
private emitProgressEvent;
|
|
94
|
+
/**
|
|
95
|
+
* Handle cancellation and emit event
|
|
96
|
+
*/
|
|
97
|
+
private handleCancellation;
|
|
98
|
+
/**
|
|
99
|
+
* Combine multiple AbortSignals into one
|
|
100
|
+
*/
|
|
101
|
+
private combineSignals;
|
|
102
|
+
/**
|
|
103
|
+
* Ingest data from a configuration directory
|
|
104
|
+
* @param configPath Path to configuration directory (containing data/, graphql/, mappings/ subdirectories)
|
|
105
|
+
* @param options Optional ingestion options
|
|
106
|
+
* @returns Promise with ingestion result
|
|
107
|
+
*/
|
|
108
|
+
ingest(configPath: string, options?: IngestOptions): Promise<IngestResult>;
|
|
109
|
+
/**
|
|
110
|
+
* Ingest specific entities from a configuration directory
|
|
111
|
+
* @param configPath Path to configuration directory
|
|
112
|
+
* @param entities Array of entity names to process
|
|
113
|
+
* @returns Promise with ingestion result
|
|
114
|
+
*/
|
|
115
|
+
ingestEntities(configPath: string, entities: string[]): Promise<IngestResult>;
|
|
116
|
+
/**
|
|
117
|
+
* Get current processing metrics
|
|
118
|
+
* @returns Current metrics
|
|
119
|
+
*/
|
|
120
|
+
getMetrics(): ProcessingMetrics;
|
|
121
|
+
/**
|
|
122
|
+
* Get a summary of the current metrics
|
|
123
|
+
* @returns Formatted summary string
|
|
124
|
+
*/
|
|
125
|
+
getMetricsSummary(): string;
|
|
126
|
+
/**
|
|
127
|
+
* Get the GraphQL client for advanced usage
|
|
128
|
+
* @returns GraphQL client wrapper
|
|
129
|
+
*/
|
|
130
|
+
getClient(): GraphQLClientWrapper;
|
|
131
|
+
/**
|
|
132
|
+
* Get the data mapper for advanced usage
|
|
133
|
+
* @returns Data mapper
|
|
134
|
+
*/
|
|
135
|
+
getMapper(): DataMapper;
|
|
136
|
+
/**
|
|
137
|
+
* Set the logger instance
|
|
138
|
+
* @param logger Logger instance to use
|
|
139
|
+
*/
|
|
140
|
+
setLogger(logger: Logger): void;
|
|
141
|
+
/**
|
|
142
|
+
* Update headers for GraphQL requests
|
|
143
|
+
* @param headers New headers to use
|
|
144
|
+
*/
|
|
145
|
+
setHeaders(headers: Record<string, string>): void;
|
|
146
|
+
/**
|
|
147
|
+
* Process entities in dependency-aware waves with abort support
|
|
148
|
+
*/
|
|
149
|
+
private processEntitiesInWaves;
|
|
150
|
+
/**
|
|
151
|
+
* Utility function to chunk array into smaller arrays
|
|
152
|
+
*/
|
|
153
|
+
private chunkArray;
|
|
154
|
+
}
|
|
155
|
+
//# sourceMappingURL=gql-ingest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gql-ingest.d.ts","sourceRoot":"","sources":["../../src/lib/gql-ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAoB,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAGhE,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAE9C,OAAO,EAEL,YAAY,EAOb,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6BAA6B;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAC7B,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,yBAAyB;IACzB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,2CAA2C;IAC3C,OAAO,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;GAGG;AACH,qBAAa,SAAU,SAAQ,YAAY;IACzC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAyB;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAC,CAAS;IAChC,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,YAAY,CAAyB;IAG7C,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,kBAAkB,CAA+C;IAGzE,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,aAAa,CAAqB;gBAE9B,OAAO,EAAE,gBAAgB;IAoBrC;;;OAGG;IACH,MAAM,CAAC,MAAM,SAAgC,GAAG,IAAI;IAMpD;;OAEG;IACH,IAAI,UAAU,IAAI,OAAO,CAExB;IAED;;OAEG;IACH,OAAO,CAAC,QAAQ;IAYhB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAQ7B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAO5B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAiBzB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAmB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IAgBtB;;;;;OAKG;IACG,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAgMhF;;;;;OAKG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IAInF;;;OAGG;IACH,UAAU,IAAI,iBAAiB;IAI/B;;;OAGG;IACH,iBAAiB,IAAI,MAAM;IAI3B;;;OAGG;IACH,SAAS,IAAI,oBAAoB;IAIjC;;;OAGG;IACH,SAAS,IAAI,UAAU;IAIvB;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAY/B;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAYjD;;OAEG;YACW,sBAAsB;IA2EpC;;OAEG;IACH,OAAO,CAAC,UAAU;CAQnB"}
|
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
import { MetricsCollector } from "./metrics";
|
|
2
2
|
import { RetryConfig } from "./config";
|
|
3
|
+
import { Logger } from "./logger";
|
|
3
4
|
export declare class GraphQLClientWrapper {
|
|
4
5
|
private client;
|
|
5
6
|
private metrics?;
|
|
6
|
-
private
|
|
7
|
-
constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector,
|
|
8
|
-
executeMutation(mutation: string, variables: Record<string, any>, retryConfig?: RetryConfig): Promise<any>;
|
|
7
|
+
private logger;
|
|
8
|
+
constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector, logger?: Logger);
|
|
9
|
+
executeMutation(mutation: string, variables: Record<string, any>, retryConfig?: RetryConfig, signal?: AbortSignal): Promise<any>;
|
|
9
10
|
private isRetryableError;
|
|
10
11
|
private calculateDelay;
|
|
11
|
-
|
|
12
|
+
/**
|
|
13
|
+
* Sleep that can be interrupted by abort signal
|
|
14
|
+
*/
|
|
15
|
+
private sleepWithSignal;
|
|
12
16
|
setHeaders(headers: Record<string, string>): void;
|
|
13
17
|
}
|
|
14
18
|
//# sourceMappingURL=graphql-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graphql-client.d.ts","sourceRoot":"","sources":["../../src/lib/graphql-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAwB,WAAW,EAAE,MAAM,UAAU,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAE9C,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,OAAO,CAAC,CAAmB;IACnC,OAAO,CAAC,MAAM,CAAS;gBAGrB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,MAAM,GAAE,MAAmB;IASvB,eAAe,CACnB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC9B,WAAW,CAAC,EAAE,WAAW,EACzB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,GAAG,CAAC;IA4Ef,OAAO,CAAC,gBAAgB;IAWxB,OAAO,CAAC,cAAc;IAatB;;OAEG;IACH,OAAO,CAAC,eAAe;IAmBvB,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;CAG3C"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/lib/index.ts"],"names":[],"mappings":"AACA,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,uBAAuB,CAAC;AACtC,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC"}
|