@oronts/vendure-data-hub-plugin 0.1.0

package/dist/src/extractors/extractor-handler-registry.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Centralized extractor handler registry.
+ *
+ * Single source of truth for extractor adapter definitions AND handler classes.
+ * Adding a new extractor requires only:
+ * 1. Create the handler class in its directory
+ * 2. Add an entry to EXTRACTOR_HANDLER_REGISTRY below
+ *
+ * The plugin providers array, ExtractorRegistryService, BUILTIN_ADAPTERS,
+ * and EXTRACTOR_CODE constants all derive from this registry automatically.
+ */
+import { Type } from '@vendure/core';
+import { DataExtractor, BatchDataExtractor } from '../types/index';
+import { AdapterDefinition } from '../sdk/types';
+type ExtractorType = DataExtractor | BatchDataExtractor;
+/**
+ * Registry entry carrying both the handler class and its adapter definition.
+ */
+interface ExtractorRegistryEntry {
+    handler: Type<ExtractorType>;
+    definition: AdapterDefinition;
+}
+/**
+ * Maps extractor codes to their NestJS injectable handler classes and adapter definitions.
+ *
+ * To register a new extractor:
+ * 1. Create its handler class and schema
+ * 2. Add its entry here with both handler and definition
+ */
+export declare const EXTRACTOR_HANDLER_REGISTRY: Map<string, ExtractorRegistryEntry>;
+/** All extractor adapter definitions, auto-derived from the registry + format-only extractors */
+export declare const EXTRACTOR_ADAPTERS: AdapterDefinition[];
+/** All extractor handler classes, for use as NestJS providers */
+export declare const EXTRACTOR_PROVIDERS: Array<Type<ExtractorType>>;
+/**
+ * Auto-derived extractor code constants from registry keys + format-only extractors.
+ * Keys are SCREAMING_SNAKE_CASE versions of the camelCase registry codes.
+ * E.g., 'httpApi' -> EXTRACTOR_CODE.HTTP_API = 'httpApi'
+ */
+export declare const EXTRACTOR_CODE: Record<string, string>;
+export {};
+//# sourceMappingURL=extractor-handler-registry.d.ts.map

package/dist/src/extractors/extractor-handler-registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractor-handler-registry.d.ts","sourceRoot":"","sources":["../../../src/extractors/extractor-handler-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAoBjD,KAAK,aAAa,GAAG,aAAa,GAAG,kBAAkB,CAAC;AAExD;;GAEG;AACH,UAAU,sBAAsB;IAC5B,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;IAC7B,UAAU,EAAE,iBAAiB,CAAC;CACjC;AAED;;;;;;GAMG;AACH,eAAO,MAAM,0BAA0B,qCAqGrC,CAAC;AAoFH,iGAAiG;AACjG,eAAO,MAAM,kBAAkB,EAAE,iBAAiB,EAGjD,CAAC;AAEF,iEAAiE;AACjE,eAAO,MAAM,mBAAmB,EAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAEtC,CAAC;AAEtB;;;;GAIG;AACH,eAAO,MAAM,cAAc,EAQtB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC"}

package/dist/src/extractors/extractor-handler-registry.js

@@ -0,0 +1,231 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EXTRACTOR_CODE = exports.EXTRACTOR_PROVIDERS = exports.EXTRACTOR_ADAPTERS = exports.EXTRACTOR_HANDLER_REGISTRY = void 0;
+const http_api_1 = require("./http-api");
+const webhook_1 = require("./webhook");
+const vendure_query_1 = require("./vendure-query");
+const file_1 = require("./file");
+const ftp_1 = require("./ftp");
+const s3_1 = require("./s3");
+const database_1 = require("./database");
+const graphql_1 = require("./graphql");
+const cdc_1 = require("./cdc");
+const schema_1 = require("./http-api/schema");
+const schema_2 = require("./vendure-query/schema");
+const schema_3 = require("./graphql/schema");
+const schema_4 = require("./cdc/schema");
+const schema_5 = require("./database/schema");
+const schema_6 = require("./webhook/schema");
+const schema_7 = require("./file/schema");
+const schema_8 = require("./ftp/schema");
+const schema_9 = require("./s3/schema");
+/**
+ * Maps extractor codes to their NestJS injectable handler classes and adapter definitions.
+ *
+ * To register a new extractor:
+ * 1. Create its handler class and schema
+ * 2. Add its entry here with both handler and definition
+ */
+exports.EXTRACTOR_HANDLER_REGISTRY = new Map([
+    ['httpApi', {
+            handler: http_api_1.HttpApiExtractor,
+            definition: {
+                type: 'EXTRACTOR',
+                code: 'httpApi',
+                description: 'Extract records from HTTP/REST APIs with authentication, pagination (offset, cursor, page, link-header), and rate limiting.',
+                icon: 'globe',
+                color: '#8b5cf6',
+                schema: schema_1.HTTP_API_EXTRACTOR_SCHEMA,
+            },
+        }],
+    ['webhook', {
+            handler: webhook_1.WebhookExtractor,
+            definition: {
+                type: 'EXTRACTOR',
+                code: 'webhook',
+                description: 'Process incoming webhook payloads as records.',
+                icon: 'webhook',
+                color: '#ec4899',
+                schema: schema_6.WEBHOOK_EXTRACTOR_SCHEMA,
+            },
+        }],
+    ['vendureQuery', {
+            handler: vendure_query_1.VendureQueryExtractor,
+            definition: {
+                type: 'EXTRACTOR',
+                code: 'vendureQuery',
+                description: 'Extract data from Vendure entities (Products, Customers, Orders, etc.)',
+                icon: 'box',
+                color: '#6366f1',
+                wizardHidden: true,
+                schema: schema_2.VENDURE_QUERY_EXTRACTOR_SCHEMA,
+            },
+        }],
+    ['file', {
+            handler: file_1.FileExtractor,
+            definition: {
+                type: 'EXTRACTOR',
+                code: 'file',
+                description: 'Extract data from local files (CSV, JSON, XML, Excel) with glob patterns.',
+                icon: 'file',
+                color: '#3b82f6',
+                schema: schema_7.FILE_EXTRACTOR_SCHEMA,
+            },
+        }],
+    ['ftp', {
+            handler: ftp_1.FtpExtractor,
+            definition: {
+                type: 'EXTRACTOR',
+                code: 'ftp',
+                description: 'Extract data from FTP/SFTP servers.',
+                icon: 'upload',
+                color: '#14b8a6',
+                schema: schema_8.FTP_EXTRACTOR_SCHEMA,
+            },
+        }],
+    ['s3', {
+            handler: s3_1.S3Extractor,
+            definition: {
+                type: 'EXTRACTOR',
+                code: 's3',
+                description: 'Extract data from AWS S3 or S3-compatible storage.',
+                icon: 'database',
+                color: '#f59e0b',
+                schema: schema_9.S3_EXTRACTOR_SCHEMA,
+            },
+        }],
+    ['database', {
+            handler: database_1.DatabaseExtractor,
+            definition: {
+                type: 'EXTRACTOR',
+                code: 'database',
+                description: 'Extract data from SQL databases (PostgreSQL, MySQL, SQLite, etc.) with pagination and incremental support.',
+                icon: 'database',
+                color: '#0ea5e9',
+                schema: schema_5.DATABASE_EXTRACTOR_SCHEMA,
+            },
+        }],
+    ['graphql', {
+            handler: graphql_1.GraphQLExtractor,
+            definition: {
+                type: 'EXTRACTOR',
+                code: 'graphql',
+                description: 'Extract records by querying a GraphQL endpoint with optional cursor pagination.',
+                icon: 'code',
+                color: '#e11d48',
+                schema: schema_3.GRAPHQL_EXTRACTOR_SCHEMA,
+            },
+        }],
+    ['cdc', {
+            handler: cdc_1.CdcExtractor,
+            definition: {
+                type: 'EXTRACTOR',
+                code: 'cdc',
+                description: 'Poll a database table for changes using a timestamp or version column (Change Data Capture).',
+                icon: 'refresh-cw',
+                color: '#0ea5e9',
+                schema: schema_4.CDC_EXTRACTOR_SCHEMA,
+            },
+        }],
+]);
+/**
+ * Format-only and utility extractors that have no runtime handler class.
+ * These are parsed inline by the extract executor.
+ */
+const FORMAT_ONLY_EXTRACTORS = [
+    {
+        type: 'EXTRACTOR',
+        code: 'inMemory',
+        description: 'Extract records from in-memory data. Use for webhook payloads, inline data, or test fixtures.',
+        icon: 'database',
+        color: '#64748b',
+        wizardHidden: true,
+        schema: {
+            fields: [
+                { key: 'data', label: 'Data (JSON)', type: 'json', description: 'Array of objects or single object. For webhooks, data is injected from trigger payload.' },
+            ],
+        },
+    },
+    {
+        type: 'EXTRACTOR',
+        code: 'generator',
+        description: 'Generate test records with configurable fields and count. Useful for testing pipelines.',
+        icon: 'sparkles',
+        color: '#f59e0b',
+        wizardHidden: true,
+        schema: {
+            fields: [
+                { key: 'count', label: 'Record count', type: 'number', required: true, description: 'Number of records to generate' },
+                { key: 'template', label: 'Template (JSON)', type: 'json', description: 'Template object with field generators' },
+            ],
+        },
+    },
+    {
+        type: 'EXTRACTOR',
+        code: 'csv',
+        description: 'Extract records from CSV. Use file upload for imports, or csvText/rows for inline data.',
+        icon: 'file-text',
+        color: '#3b82f6',
+        schema: {
+            fields: [
+                { key: 'fileId', label: 'Upload CSV File', type: 'file', description: 'Upload a CSV file to extract data from. Preferred method for file imports.' },
+                { key: 'delimiter', label: 'Delimiter', type: 'string', description: 'Character to separate fields (default: ,)' },
+                { key: 'hasHeader', label: 'Header row', type: 'boolean', description: 'Whether first row is header (default: true)' },
+                { key: 'csvText', label: 'CSV text (alternative)', type: 'json', description: 'Raw CSV string instead of file upload' },
+                { key: 'rows', label: 'Rows (JSON array)', type: 'json', description: 'Alternative to file: array of arrays or objects' },
+                { key: 'csvPath', label: 'CSV file path', type: 'string', description: 'Read from filesystem path (dev/testing only)' },
+            ],
+        },
+    },
+    {
+        type: 'EXTRACTOR',
+        code: 'json',
+        description: 'Extract records from JSON. Use file upload for imports, or jsonText for inline data.',
+        icon: 'file',
+        color: '#eab308',
+        schema: {
+            fields: [
+                { key: 'fileId', label: 'Upload JSON File', type: 'file', description: 'Upload a JSON file to extract data from. Preferred method for file imports.' },
+                { key: 'itemsPath', label: 'Items path', type: 'string', description: 'Dot-path to array of records (e.g., "data.items")' },
+                { key: 'jsonText', label: 'JSON text (alternative)', type: 'json', description: 'Raw JSON string instead of file upload' },
+                { key: 'jsonPath', label: 'JSON file path', type: 'string', description: 'Read from filesystem path (dev/testing only)' },
+            ],
+        },
+    },
+    {
+        type: 'EXTRACTOR',
+        code: 'xml',
+        description: 'Extract records from XML documents with configurable root and item element paths.',
+        icon: 'code',
+        color: '#f97316',
+        schema: {
+            fields: [
+                { key: 'fileId', label: 'Upload XML File', type: 'file', description: 'Upload an XML file to extract data from.' },
+                { key: 'itemsPath', label: 'Items path', type: 'string', description: 'Dot-path to array of records inside parsed XML' },
+                { key: 'rootElement', label: 'Root element', type: 'string', description: 'XML root element name to start parsing from' },
+                { key: 'itemElement', label: 'Item element', type: 'string', description: 'Element name for each record (e.g., "product")' },
+                { key: 'xmlPath', label: 'XML file path', type: 'string', description: 'Read from filesystem path (dev/testing only)' },
+            ],
+        },
+    },
+];
+/** All extractor adapter definitions, auto-derived from the registry + format-only extractors */
+exports.EXTRACTOR_ADAPTERS = [
+    ...Array.from(exports.EXTRACTOR_HANDLER_REGISTRY.values()).map(e => e.definition),
+    ...FORMAT_ONLY_EXTRACTORS,
+];
+/** All extractor handler classes, for use as NestJS providers */
+exports.EXTRACTOR_PROVIDERS = Array.from(exports.EXTRACTOR_HANDLER_REGISTRY.values()).map(e => e.handler);
+/**
+ * Auto-derived extractor code constants from registry keys + format-only extractors.
+ * Keys are SCREAMING_SNAKE_CASE versions of the camelCase registry codes.
+ * E.g., 'httpApi' -> EXTRACTOR_CODE.HTTP_API = 'httpApi'
+ */
+exports.EXTRACTOR_CODE = Object.fromEntries([
+    ...Array.from(exports.EXTRACTOR_HANDLER_REGISTRY.keys()),
+    ...FORMAT_ONLY_EXTRACTORS.map(e => e.code),
+].map(code => [
+    code.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toUpperCase(),
+    code,
+]));
+//# sourceMappingURL=extractor-handler-registry.js.map

package/dist/src/extractors/extractor-handler-registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractor-handler-registry.js","sourceRoot":"","sources":["../../../src/extractors/extractor-handler-registry.ts"],"names":[],"mappings":";;;AAcA,yCAA8C;AAC9C,uCAA6C;AAC7C,mDAAwD;AACxD,iCAAuC;AACvC,+BAAqC;AACrC,6BAAmC;AACnC,yCAA+C;AAC/C,uCAA6C;AAC7C,+BAAqC;AACrC,8CAA8D;AAC9D,mDAAwE;AACxE,6CAA4D;AAC5D,yCAAoD;AACpD,8CAA8D;AAC9D,6CAA4D;AAC5D,0CAAsD;AACtD,yCAAoD;AACpD,wCAAkD;AAYlD;;;;;;GAMG;AACU,QAAA,0BAA0B,GAAG,IAAI,GAAG,CAAiC;IAC9E,CAAC,SAAS,EAAE;YACR,OAAO,EAAE,2BAAgB;YACzB,UAAU,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,6HAA6H;gBAC1I,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,kCAAyB;aACpC;SACJ,CAAC;IACF,CAAC,SAAS,EAAE;YACR,OAAO,EAAE,0BAAgB;YACzB,UAAU,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,+CAA+C;gBAC5D,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,iCAAwB;aACnC;SACJ,CAAC;IACF,CAAC,cAAc,EAAE;YACb,OAAO,EAAE,qCAAqB;YAC9B,UAAU,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,cAAc;gBACpB,WAAW,EAAE,wEAAwE;gBACrF,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,SAAS;gBAChB,YAAY,EAAE,IAAI;gBAClB,MAAM,EAAE,uCAA8B;aACzC;SACJ,CAAC;IACF,CAAC,MAAM,EAAE;YACL,OAAO,EAAE,oBAAa;YACtB,UAAU,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,MAAM;gBACZ,WAAW,EAAE,2EAA2E;gBACxF,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,8BAAqB;aAChC;SACJ,CAAC;IACF,CAAC,KAAK,EAAE;YACJ,OAAO,EAAE,kBAAY;YACrB,UAAU,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,KAAK;gBACX,WAAW,EAAE,qCAAqC;gBAClD,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,6BAAoB;aAC/B;SACJ,CAAC;IACF,CAAC,IAAI,EAAE;YACH,OAAO,EAAE,gBAAW;YACpB,UAAU,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,IAAI;gBACV,WAAW,EAAE,oDAAoD;gBACjE,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,4BAAmB;aAC9B;SACJ,CAAC;IACF,CAAC,UAAU,EAAE;YACT,OAAO,EAAE,4BAAiB;YAC1B,UAAU,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,UAAU;gBAChB,WAAW,EAAE,4GAA4G;gBACzH,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,kCAAyB;aACpC;SACJ,CAAC;IACF,CAAC,SAAS,EAAE;YACR,OAAO,EAAE,0BAAgB;YACzB,UAAU,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,iFAAiF;gBAC9F,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,iCAAwB;aACnC;SACJ,CAAC;IACF,CAAC,KAAK,EAAE;YACJ,OAAO,EAAE,kBAAY;YACrB,UAAU,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,KAAK;gBACX,WAAW,EAAE,8FAA8F;gBAC3G,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,6BAAoB;aAC/B;SACJ,CAAC;CACL,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,sBAAsB,GAAwB;IAChD;QACI,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,UAAU;QAChB,WAAW,EAAE,+FAA+F;QAC5G,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,SAAS;QAChB,YAAY,EAAE,IAAI;QAClB,MAAM,EAAE;YACJ,MAAM,EAAE;gBACJ,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,yFAAyF,EAAE;aAC9J;SACJ;KACJ;IACD;QACI,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,WAAW;QACjB,WAAW,EAAE,yFAAyF;QACtG,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,SAAS;QAChB,YAAY,EAAE,IAAI;QAClB,MAAM,EAAE;YACJ,MAAM,EAAE;gBACJ,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,EAAE;gBACrH,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,uCAAuC,EAAE;aACpH;SACJ;KACJ;IACD;QACI,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,KAAK;QACX,WAAW,EAAE,yFAAyF;QACtG,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,SAAS;QAChB,MAAM,EAAE;YACJ,MAAM,EAAE;gBACJ,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,4EAA4E,EAAE;gBACpJ,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,2CAA2C,EAAE;gBAClH,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,6CAA6C,EAAE;gBACtH,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,wBAAwB,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,uCAAuC,EAAE;gBACvH,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iDAAiD,EAAE;gBACzH,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,8CAA8C,EAAE;aAC1H;SACJ;KACJ;IACD;QACI,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,MAAM;QACZ,WAAW,EAAE,sFAAsF;QACnG,IAAI,EAAE,MAAM;QACZ,KAAK,EAAE,SAAS;QAChB,MAAM,EAAE;YACJ,MAAM,EAAE;gBACJ,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,6EAA6E,EAAE;gBACtJ,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,mDAAmD,EAAE;gBAC3H,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,yBAAyB,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,wCAAwC,EAAE;gBAC1H,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,8CAA8C,EAAE;aAC5H;SACJ;KACJ;IACD;QACI,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,KAAK;QACX,WAAW,EAAE,mFAAmF;QAChG,IAAI,EAAE,MAAM;QACZ,KAAK,EAAE,SAAS;QAChB,MAAM,EAAE;YACJ,MAAM,EAAE;gBACJ,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,0CAA0C,EAAE;gBAClH,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,gDAAgD,EAAE;gBACxH,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6CAA6C,EAAE;gBACzH,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,gDAAgD,EAAE;gBAC5H,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,8CAA8C,EAAE;aAC1H;SACJ;KACJ;CACJ,CAAC;AAEF,iGAAiG;AACpF,QAAA,kBAAkB,GAAwB;IACnD,GAAG,KAAK,CAAC,IAAI,CAAC,kCAA0B,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;IACzE,GAAG,sBAAsB;CAC5B,CAAC;AAEF,iEAAiE;AACpD,QAAA,mBAAmB,GAA+B,KAAK,CAAC,IAAI,CACrE,kCAA0B,CAAC,MAAM,EAAE,CACtC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;AAEtB;;;;GAIG;AACU,QAAA,cAAc,GAAG,MAAM,CAAC,WAAW,CAC5C;IACI,GAAG,KAAK,CAAC,IAAI,CAAC,kCAA0B,CAAC,IAAI,EAAE,CAAC;IAChD,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;CAC7C,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;IACV,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAC,WAAW,EAAE;IACzD,IAAI;CACP,CAAC,CACqB,CAAC"}

package/dist/src/extractors/extractor-registry.service.d.ts

@@ -0,0 +1,58 @@
+import { OnModuleInit } from '@nestjs/common';
+import { ModuleRef } from '@nestjs/core';
+import { DataExtractor, BatchDataExtractor, ExtractorCategory } from '../types/index';
+import { DataHubLoggerFactory } from '../services/logger';
+export type ExtractorRegistrationCallback = (registry: ExtractorRegistryService) => void | Promise<void>;
+export interface ExtractorMetadata {
+    code: string;
+    name: string;
+    description?: string;
+    category: ExtractorCategory;
+    version?: string;
+    icon?: string;
+    supportsPagination?: boolean;
+    supportsIncremental?: boolean;
+    supportsCancellation?: boolean;
+    isStreaming: boolean;
+    isBatch: boolean;
+}
+export interface ExtractorInfo {
+    extractor: DataExtractor | BatchDataExtractor;
+    metadata: ExtractorMetadata;
+    registeredAt: Date;
+    registeredBy?: string;
+}
+export declare class ExtractorRegistryService implements OnModuleInit {
+    private moduleRef;
+    private readonly logger;
+    private extractors;
+    private customCallbacks;
+    constructor(moduleRef: ModuleRef, loggerFactory: DataHubLoggerFactory);
+    onModuleInit(): Promise<void>;
+    private registerBuiltInExtractors;
+    addRegistrationCallback(callback: ExtractorRegistrationCallback): void;
+    register(extractor: DataExtractor | BatchDataExtractor, registeredBy?: string): void;
+    registerAll(extractors: Array<DataExtractor | BatchDataExtractor>, registeredBy?: string): void;
+    getExtractor(code: string): DataExtractor | BatchDataExtractor | undefined;
+    getExtractorInfo(code: string): ExtractorInfo | undefined;
+    getStreamingExtractor(code: string): DataExtractor | undefined;
+    getBatchExtractor(code: string): BatchDataExtractor | undefined;
+    listExtractors(): Array<DataExtractor | BatchDataExtractor>;
+    hasExtractor(code: string): boolean;
+    getExtractorMetadata(): ExtractorMetadata[];
+    getExtractorsByCategory(): Record<string, ExtractorMetadata[]>;
+    findExtractors(criteria: {
+        category?: ExtractorCategory;
+        supportsIncremental?: boolean;
+        supportsPagination?: boolean;
+    }): ExtractorMetadata[];
+    unregister(code: string): boolean;
+    clear(): void;
+    get count(): number;
+    getCategoryLabels(): Record<ExtractorCategory, string>;
+    validateRequiredExtractors(codes: string[]): {
+        valid: boolean;
+        missing: string[];
+    };
+}
+//# sourceMappingURL=extractor-registry.service.d.ts.map

package/dist/src/extractors/extractor-registry.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractor-registry.service.d.ts","sourceRoot":"","sources":["../../../src/extractors/extractor-registry.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,iBAAiB,EAGpB,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAiB,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAIzE,MAAM,MAAM,6BAA6B,GAAG,CAAC,QAAQ,EAAE,wBAAwB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAEzG,MAAM,WAAW,iBAAiB;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,WAAW,EAAE,OAAO,CAAC;IACrB,OAAO,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC1B,SAAS,EAAE,aAAa,GAAG,kBAAkB,CAAC;IAC9C,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,YAAY,EAAE,IAAI,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,qBACa,wBAAyB,YAAW,YAAY;IAMrD,OAAO,CAAC,SAAS;IALrB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,UAAU,CAAoC;IACtD,OAAO,CAAC,eAAe,CAAuC;gBAGlD,SAAS,EAAE,SAAS,EAC5B,aAAa,EAAE,oBAAoB;IAKjC,YAAY;YAiCJ,yBAAyB;IAqBvC,uBAAuB,CAAC,QAAQ,EAAE,6BAA6B,GAAG,IAAI;IAItE,QAAQ,CACJ,SAAS,EAAE,aAAa,GAAG,kBAAkB,EAC7C,YAAY,CAAC,EAAE,MAAM,GACtB,IAAI;IAgCP,WAAW,CACP,UAAU,EAAE,KAAK,CAAC,aAAa,GAAG,kBAAkB,CAAC,EACrD,YAAY,CAAC,EAAE,MAAM,GACtB,IAAI;IAMP,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,kBAAkB,GAAG,SAAS;IAI1E,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAIzD,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAQ9D,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS;IAQ/D,cAAc,IAAI,KAAK,CAAC,aAAa,GAAG,kBAAkB,CAAC;IAI3D,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAInC,oBAAoB,IAAI,iBAAiB,EAAE;IAI3C,uBAAuB,IAAI,MAAM,CAAC,MAAM,EAAE,iBAAiB,EAAE,CAAC;IAmB9D,cAAc,CAAC,QAAQ,EAAE;QACrB,QAAQ,CAAC,EAAE,iBAAiB,CAAC;QAC7B,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,kBAAkB,CAAC,EAAE,OAAO,CAAC;KAChC,GAAG,iBAAiB,EAAE;IAevB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAQjC,KAAK,IAAI,IAAI;IAOb,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,iBAAiB,IAAI,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC;IAatD,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE;CAOrF"}

package/dist/src/extractors/extractor-registry.service.js

@@ -0,0 +1,206 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExtractorRegistryService = void 0;
+const common_1 = require("@nestjs/common");
+const core_1 = require("@nestjs/core");
+const index_1 = require("../types/index");
+const error_utils_1 = require("../utils/error.utils");
+const logger_1 = require("../services/logger");
+const index_2 = require("../constants/index");
+const extractor_handler_registry_1 = require("./extractor-handler-registry");
+let ExtractorRegistryService = class ExtractorRegistryService {
+    constructor(moduleRef, loggerFactory) {
+        this.moduleRef = moduleRef;
+        this.extractors = new Map();
+        this.customCallbacks = [];
+        this.logger = loggerFactory.createLogger(index_2.LOGGER_CONTEXTS.EXTRACTOR_REGISTRY);
+    }
+    async onModuleInit() {
+        try {
+            await this.registerBuiltInExtractors();
+            for (const callback of this.customCallbacks) {
+                try {
+                    await callback(this);
+                }
+                catch (error) {
+                    this.logger.error('Extractor registration callback failed', (0, error_utils_1.toErrorOrUndefined)(error));
+                }
+            }
+            this.logger.info(`Extractor registry initialized`, {
+                recordCount: this.extractors.size,
+            });
+            for (const [code, info] of this.extractors) {
+                this.logger.debug(`Registered: ${info.metadata.name} (${code})`, {
+                    adapterCode: code,
+                    category: info.metadata.category,
+                });
+            }
+        }
+        catch (error) {
+            this.logger.error('Failed to initialize extractor registry', (0, error_utils_1.toErrorOrUndefined)(error));
+        }
+    }
+    async registerBuiltInExtractors() {
+        let registered = 0;
+        for (const [code, entry] of extractor_handler_registry_1.EXTRACTOR_HANDLER_REGISTRY) {
+            try {
+                const extractor = this.moduleRef.get(entry.handler, { strict: false });
+                this.register(extractor, 'built-in');
+                registered++;
+            }
+            catch (error) {
+                this.logger.warn(`Failed to register built-in extractor`, {
+                    adapterCode: code,
+                    error: (0, error_utils_1.getErrorMessage)(error),
+                });
+            }
+        }
+        this.logger.debug(`Registered built-in extractors`, {
+            recordCount: registered,
+        });
+    }
+    addRegistrationCallback(callback) {
+        this.customCallbacks.push(callback);
+    }
+    register(extractor, registeredBy) {
+        const code = extractor.code;
+        if (this.extractors.has(code)) {
+            this.logger.warn(`Overwriting existing extractor`, { adapterCode: code });
+        }
+        const metadata = {
+            code: extractor.code,
+            name: extractor.name,
+            description: extractor.description,
+            category: extractor.category,
+            version: extractor.version,
+            icon: extractor.icon,
+            supportsPagination: extractor.supportsPagination,
+            supportsIncremental: extractor.supportsIncremental,
+            supportsCancellation: extractor.supportsCancellation,
+            isStreaming: (0, index_1.isStreamingExtractor)(extractor),
+            isBatch: (0, index_1.isBatchExtractor)(extractor),
+        };
+        const info = {
+            extractor,
+            metadata,
+            registeredAt: new Date(),
+            registeredBy,
+        };
+        this.extractors.set(code, info);
+        this.logger.debug(`Registered extractor: ${extractor.name}`, { adapterCode: code });
+    }
+    registerAll(extractors, registeredBy) {
+        for (const extractor of extractors) {
+            this.register(extractor, registeredBy);
+        }
+    }
+    getExtractor(code) {
+        return this.extractors.get(code)?.extractor;
+    }
+    getExtractorInfo(code) {
+        return this.extractors.get(code);
+    }
+    getStreamingExtractor(code) {
+        const extractor = this.getExtractor(code);
+        if (extractor && (0, index_1.isStreamingExtractor)(extractor)) {
+            return extractor;
+        }
+        return undefined;
+    }
+    getBatchExtractor(code) {
+        const extractor = this.getExtractor(code);
+        if (extractor && (0, index_1.isBatchExtractor)(extractor)) {
+            return extractor;
+        }
+        return undefined;
+    }
+    listExtractors() {
+        return Array.from(this.extractors.values()).map(info => info.extractor);
+    }
+    hasExtractor(code) {
+        return this.extractors.has(code);
+    }
+    getExtractorMetadata() {
+        return Array.from(this.extractors.values()).map(info => info.metadata);
+    }
+    getExtractorsByCategory() {
+        const result = {};
+        for (const info of this.extractors.values()) {
+            const category = info.metadata.category || 'custom';
+            if (!result[category]) {
+                result[category] = [];
+            }
+            result[category].push(info.metadata);
+        }
+        // Sort by name within each category
+        for (const category of Object.keys(result)) {
+            result[category].sort((a, b) => a.name.localeCompare(b.name));
+        }
+        return result;
+    }
+    findExtractors(criteria) {
+        return this.getExtractorMetadata().filter(meta => {
+            if (criteria.category && meta.category !== criteria.category) {
+                return false;
+            }
+            if (criteria.supportsIncremental !== undefined && meta.supportsIncremental !== criteria.supportsIncremental) {
+                return false;
+            }
+            if (criteria.supportsPagination !== undefined && meta.supportsPagination !== criteria.supportsPagination) {
+                return false;
+            }
+            return true;
+        });
+    }
+    unregister(code) {
+        const existed = this.extractors.delete(code);
+        if (existed) {
+            this.logger.debug(`Unregistered extractor`, { adapterCode: code });
+        }
+        return existed;
+    }
+    clear() {
+        const count = this.extractors.size;
+        this.extractors.clear();
+        this.customCallbacks = [];
+        this.logger.debug('Cleared all extractors', { recordCount: count });
+    }
+    get count() {
+        return this.extractors.size;
+    }
+    getCategoryLabels() {
+        return {
+            'DATA_SOURCE': 'Data Sources',
+            'FILE_SYSTEM': 'File System',
+            'CLOUD_STORAGE': 'Cloud Storage',
+            'DATABASE': 'Databases',
+            'API': 'APIs',
+            'WEBHOOK': 'Webhooks',
+            'VENDURE': 'Vendure',
+            'CUSTOM': 'Custom',
+        };
+    }
+    validateRequiredExtractors(codes) {
+        const missing = codes.filter(code => !this.hasExtractor(code));
+        return {
+            valid: missing.length === 0,
+            missing,
+        };
+    }
+};
+exports.ExtractorRegistryService = ExtractorRegistryService;
+exports.ExtractorRegistryService = ExtractorRegistryService = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [core_1.ModuleRef,
+        logger_1.DataHubLoggerFactory])
+], ExtractorRegistryService);
+//# sourceMappingURL=extractor-registry.service.js.map

package/dist/src/extractors/extractor-registry.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractor-registry.service.js","sourceRoot":"","sources":["../../../src/extractors/extractor-registry.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA0D;AAC1D,uCAAyC;AACzC,0CAMwB;AACxB,sDAA2E;AAC3E,+CAAyE;AACzE,8CAAqD;AACrD,6EAA0E;AA0BnE,IAAM,wBAAwB,GAA9B,MAAM,wBAAwB;IAKjC,YACY,SAAoB,EAC5B,aAAmC;QAD3B,cAAS,GAAT,SAAS,CAAW;QAJxB,eAAU,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC9C,oBAAe,GAAoC,EAAE,CAAC;QAM1D,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,YAAY,CAAC,uBAAe,CAAC,kBAAkB,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,YAAY;QACd,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;YAEvC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC1C,IAAI,CAAC;oBACD,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACzB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACb,IAAI,CAAC,MAAM,CAAC,KAAK,CACb,wCAAwC,EACxC,IAAA,gCAAkB,EAAC,KAAK,CAAC,CAC5B,CAAC;gBACN,CAAC;YACL,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;gBAC/C,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;aACpC,CAAC,CAAC;YAEH,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,IAAI,GAAG,EAAE;oBAC7D,WAAW,EAAE,IAAI;oBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ;iBACnC,CAAC,CAAC;YACP,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,CAAC,KAAK,CACb,yCAAyC,EACzC,IAAA,gCAAkB,EAAC,KAAK,CAAC,CAC5B,CAAC;QACN,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,yBAAyB;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,uDAA0B,EAAE,CAAC;YACrD,IAAI,CAAC;gBACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBACvE,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;gBACrC,UAAU,EAAE,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,EAAE;oBACtD,WAAW,EAAE,IAAI;oBACjB,KAAK,EAAE,IAAA,6BAAe,EAAC,KAAK,CAAC;iBAChC,CAAC,CAAC;YACP,CAAC;QACL,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;YAChD,WAAW,EAAE,UAAU;SAC1B,CAAC,CAAC;IACP,CAAC;IAED,uBAAuB,CAAC,QAAuC;QAC3D,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,QAAQ,CACJ,SAA6C,EAC7C,YAAqB;QAErB,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;QAE5B,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9E,CAAC;QAED,MAAM,QAAQ,GAAsB;YAChC,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,QAAQ,EAAE,SAAS,CAAC,QAAQ;YAC5B,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;YAChD,mBAAmB,EAAE,SAAS,CAAC,mBAAmB;YAClD,oBAAoB,EAAE,SAAS,CAAC,oBAAoB;YACpD,WAAW,EAAE,IAAA,4BAAoB,EAAC,SAAS,CAAC;YAC5C,OAAO,EAAE,IAAA,wBAAgB,EAAC,SAAS,CAAC;SACvC,CAAC;QAEF,MAAM,IAAI,GAAkB;YACxB,SAAS;YACT,QAAQ;YACR,YAAY,EAAE,IAAI,IAAI,EAAE;YACxB,YAAY;SACf,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,SAAS,CAAC,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,WAAW,CACP,UAAqD,EACrD,YAAqB;QAErB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACjC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAC3C,CAAC;IACL,CAAC;IAED,YAAY,CAAC,IAAY;QACrB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,CAAC;IAChD,CAAC;IAED,gBAAgB,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,qBAAqB,CAAC,IAAY;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,SAAS,IAAI,IAAA,4BAAoB,EAAC,SAAS,CAAC,EAAE,CAAC;YAC/C,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,iBAAiB,CAAC,IAAY;QAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,SAAS,IAAI,IAAA,wBAAgB,EAAC,SAAS,CAAC,EAAE,CAAC;YAC3C,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,cAAc;QACV,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5E,CAAC;IAED,YAAY,CAAC,IAAY;QACrB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,oBAAoB;QAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3E,CAAC;IAED,uBAAuB;QACnB,MAAM,MAAM,GAAwC,EAAE,CAAC;QAEvD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC;YACpD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACpB,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;YAC1B,CAAC;YACD,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzC,CAAC;QAED,oCAAoC;QACpC,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACzC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,cAAc,CAAC,QAId;QACG,OAAO,IAAI,CAAC,oBAAoB,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YAC7C,IAAI,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBAC3D,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,IAAI,QAAQ,CAAC,mBAAmB,KAAK,SAAS,IAAI,IAAI,CAAC,mBAAmB,KAAK,QAAQ,CAAC,mBAAmB,EAAE,CAAC;gBAC1G,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,IAAI,QAAQ,CAAC,kBAAkB,KAAK,SAAS,IAAI,IAAI,CAAC,kBAAkB,KAAK,QAAQ,CAAC,kBAAkB,EAAE,CAAC;gBACvG,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC,CAAC;IACP,CAAC;IAED,UAAU,CAAC,IAAY;QACnB,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,KAAK;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QACnC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAChC,CAAC;IAED,iBAAiB;QACb,OAAO;YACH,aAAa,EAAE,cAAc;YAC7B,aAAa,EAAE,aAAa;YAC5B,eAAe,EAAE,eAAe;YAChC,UAAU,EAAE,WAAW;YACvB,KAAK,EAAE,MAAM;YACb,SAAS,EAAE,UAAU;YACrB,SAAS,EAAE,SAAS;YACpB,QAAQ,EAAE,QAAQ;SACrB,CAAC;IACN,CAAC;IAED,0BAA0B,CAAC,KAAe;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,OAAO;YACH,KAAK,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO;SACV,CAAC;IACN,CAAC;CACJ,CAAA;AAnOY,4DAAwB;mCAAxB,wBAAwB;IADpC,IAAA,mBAAU,GAAE;qCAOc,gBAAS;QACb,6BAAoB;GAP9B,wBAAwB,CAmOpC"}

package/dist/src/extractors/file/file.extractor.d.ts

@@ -0,0 +1,20 @@
+import { DataExtractor, ExtractorContext, ExtractorValidationResult, ConnectionTestResult, ExtractorPreviewResult, RecordEnvelope, StepConfigSchema, ExtractorCategory } from '../../types/index';
+import { FileParserService } from '../../parsers/file-parser.service';
+import { FileExtractorConfig } from './types';
+export declare class FileExtractor implements DataExtractor<FileExtractorConfig> {
+    private readonly fileParser;
+    readonly type: "EXTRACTOR";
+    readonly code = "file";
+    readonly name = "File Extractor";
+    readonly category: ExtractorCategory;
+    readonly supportsPagination = false;
+    readonly supportsIncremental = true;
+    readonly supportsCancellation = true;
+    constructor(fileParser: FileParserService);
+    readonly schema: StepConfigSchema;
+    extract(context: ExtractorContext, config: FileExtractorConfig): AsyncGenerator<RecordEnvelope, void, undefined>;
+    validate(_context: ExtractorContext, config: FileExtractorConfig): Promise<ExtractorValidationResult>;
+    testConnection(_context: ExtractorContext, config: FileExtractorConfig): Promise<ConnectionTestResult>;
+    preview(_context: ExtractorContext, config: FileExtractorConfig, limit?: number): Promise<ExtractorPreviewResult>;
+}
+//# sourceMappingURL=file.extractor.d.ts.map

package/dist/src/extractors/file/file.extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file.extractor.d.ts","sourceRoot":"","sources":["../../../../src/extractors/file/file.extractor.ts"],"names":[],"mappings":"AAMA,OAAO,EACH,aAAa,EACb,gBAAgB,EAChB,yBAAyB,EACzB,oBAAoB,EACpB,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,iBAAiB,EAAgB,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAE,mBAAmB,EAA2B,MAAM,SAAS,CAAC;AAOvE,qBACa,aAAc,YAAW,aAAa,CAAC,mBAAmB,CAAC;IASxD,OAAO,CAAC,QAAQ,CAAC,UAAU;IARvC,QAAQ,CAAC,IAAI,EAAG,WAAW,CAAU;IACrC,QAAQ,CAAC,IAAI,UAAU;IACvB,QAAQ,CAAC,IAAI,oBAAoB;IACjC,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,CAAiB;IACrD,QAAQ,CAAC,kBAAkB,SAAS;IACpC,QAAQ,CAAC,mBAAmB,QAAQ;IACpC,QAAQ,CAAC,oBAAoB,QAAQ;gBAER,UAAU,EAAE,iBAAiB;IAE1D,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAyB;IAEnD,OAAO,CACV,OAAO,EAAE,gBAAgB,EACzB,MAAM,EAAE,mBAAmB,GAC5B,cAAc,CAAC,cAAc,EAAE,IAAI,EAAE,SAAS,CAAC;IA0H5C,QAAQ,CACV,QAAQ,EAAE,gBAAgB,EAC1B,MAAM,EAAE,mBAAmB,GAC5B,OAAO,CAAC,yBAAyB,CAAC;IAwC/B,cAAc,CAChB,QAAQ,EAAE,gBAAgB,EAC1B,MAAM,EAAE,mBAAmB,GAC5B,OAAO,CAAC,oBAAoB,CAAC;IAsB1B,OAAO,CACT,QAAQ,EAAE,gBAAgB,EAC1B,MAAM,EAAE,mBAAmB,EAC3B,KAAK,GAAE,MAAsC,GAC9C,OAAO,CAAC,sBAAsB,CAAC;CA2CrC"}