@forzalabs/remora 1.0.21 → 1.1.1

package/engines/ai/LLM.js

@@ -1,255 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const openai_1 = __importDefault(require("openai"));
-const zod_1 = require("openai/helpers/zod");
-const zod_2 = require("zod");
-const ProcessENVManager_1 = __importDefault(require("../ProcessENVManager"));
-const baseProducersSystemPrompt = `
-# TASK
-You are an agent tasked with creating the mapping between an INPUT DATA SPEC and one or more OUTPUT DATA SPEC.
-The mapping between the two is made by creating PRODUCERS and CONSUMERS.
-You are tasked with creating the PRODUCER(S) that will then be used.
-
-# PRODUCERS
-A producer maps directly to a dataset and exposes it's dimensions.
-
-## FIELDS
-- classification: make your best guess if the field falls under any of these regulations
-
-# RULES
-- Add only the required fields to comply with the OUTPUT DATA SPEC
-- Add fields that you think are important
-- The name of the producer must be the same as the name of the dataset.
-- Avoid creating multiple providers with similar data.
-- Try to create the least number of providers
-- Awlays include this exact property as the first -> "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/producer-schema.json"
-- Based on the producer select the source that makes the most sense to connect otherwise leave the string "<source_name>"
-
-# FORMAT
-The INPUT DATA SPEC will be the first 10 rows from the dataset that the producer needs to map to.
-The OUTPUT DATA SPEC is a JSON of the desired output that the consumer needs to return.
-The result must be returned as a JSON object.
-
-# INPUT DATA SPEC
-{{input data spec}}
-
-# OUTPUT DATA SPEC
-{{output data spec}}
-
-# File name
-{{file name}}
-
-# SOURCES
-{{sources}}
-`;
-const baseConsumersSystemPrompt = `
-# TASK
-You are an agent tasked with creating the mapping between a list of PRODUCERS and one or more OUTPUT DATA SPEC.
-You are going to receive a list of PRODUCERS that expose some dimensions, and you will create the CONSUMERS that comply with the provided OUTPUT DATA SPEC.
-
-# CONSUMERS
-A consumer takes the data from one or more producers and changes it's shape to transform it into the required output schema.
-
-## FIELDS
-- fields.from: used to distinct between the producers imported by the consumer. The value is the name of the producer.
-
-# RULES
-- If a field is not needed, do not add it e.g.
-- Only import a producer once
-- Awlays include this exact property as the first -> $schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
-- Use "API" as the only valid output format.
-- The "from" must contain only the name of the producer
-
-# FORMAT
-The INPUT DATA SPEC will be the first 10 rows from the dataset that the producer needs to map to.
-The OUTPUT DATA SPEC is a JSON of the desired output that the consumer needs to return.
-The result must be returned as a JSON object.
-
-# PRODUCERS
-{{producers}}
-
-# OUTPUT DATA SPEC
-{{output data spec}}
-
-# EXAMPLES
-producers: """
-{
-    "name": "claim",
-    "dimensions": [
-        {
-            "name": "id",
-            "type": "string",
-            "pk": true
-        },
-        {
-            "name": "amount",
-            "type": "number"
-        },
-        {
-            "name": "provider",
-            "type": "string"
-        },
-        {
-            "name": "date",
-            "type": "datetime"
-        }
-    ]
-}
-"""
-
-output data spec: """
-{
-    "name": "claim",
-    "fields": [
-        {
-            "name": "id",
-            "type": "string",
-            "pk": true
-        },
-        {
-            "name": "amount",
-            "type": "number"
-        },
-        {
-            "name": "provider",
-            "type": "string"
-        },
-        {
-            "name": "date",
-            "type": "datetime"
-        }
-    ]
-}
-"""
-
-resulting consumer: """
-{
-    "name": "claims",
-    "fields": [
-        {
-            "key": "id"
-        },
-        {
-            "key": "amount"
-        },
-        {
-            "key": "provider"
-        },
-        {
-            "key": "date",
-            "alias": "Creation date"
-        }
-    ],
-    "outputs": [
-        {
-            "format": "API"
-        }
-    ],
-    "producers": [
-        {
-            "name": "claim"
-        }
-    ]
-}
-"""
-`;
-class LLM {
-    constructor() {
-        this.inferProducers = (input, outputs, fileName, sources) => __awaiter(this, void 0, void 0, function* () {
-            let systemPrompt = baseProducersSystemPrompt;
-            systemPrompt = systemPrompt.replace('{{input data spec}}', input.map(x => `- ${JSON.stringify(x)}`).join('\n'));
-            systemPrompt = systemPrompt.replace('{{output data spec}}', outputs.map(x => `- ${JSON.stringify(x)}`).join('\n'));
-            systemPrompt = systemPrompt.replace('{{file name}}', fileName);
-            systemPrompt = systemPrompt.replace('{{sources}}', sources.map(x => `- ${JSON.stringify(x)}`).join('\n'));
-            const res = yield this._client.chat.completions.create({
-                model: 'gpt-4o',
-                messages: [
-                    { role: 'system', content: systemPrompt }
-                ],
-                response_format: (0, zod_1.zodResponseFormat)(zod_2.z.object({
-                    producers: zod_2.z.array(zod_2.z.object({
-                        $schema: zod_2.z.string().describe('The schema of the producer. This should always be the same.'),
-                        name: zod_2.z.string(),
-                        description: zod_2.z.string(),
-                        source: zod_2.z.string().describe('The name of the source linked to this producer.'),
-                        settings: zod_2.z.object({
-                            fileKey: zod_2.z.string().describe('The name of the file'),
-                            fileType: zod_2.z.string().describe('The file extension (CSV | JSONL | JSON)')
-                        }),
-                        dimensions: zod_2.z.array(zod_2.z.object({
-                            name: zod_2.z.string(),
-                            // alias: z.string().optional(),
-                            description: zod_2.z.string().optional(),
-                            type: zod_2.z.enum(['string', 'number', 'datetime']),
-                            pk: zod_2.z.boolean().optional(),
-                            classification: zod_2.z.array(zod_2.z.enum(['PHI', 'PII', 'GDPR'])).optional()
-                        }))
-                    }))
-                }), 'environment')
-            });
-            const msg = res.choices[0].message;
-            return JSON.parse(msg.content);
-        });
-        this.inferConsumers = (producers, outputs) => __awaiter(this, void 0, void 0, function* () {
-            let systemPrompt = baseConsumersSystemPrompt;
-            systemPrompt = systemPrompt.replace('{{producers}}', producers.map(x => `- ${JSON.stringify(x)}`).join('\n'));
-            systemPrompt = systemPrompt.replace('{{output data spec}}', outputs.map(x => `- ${JSON.stringify(x)}`).join('\n'));
-            const item = {
-                model: 'gpt-4o',
-                messages: [
-                    { role: 'system', content: systemPrompt }
-                ],
-                response_format: (0, zod_1.zodResponseFormat)(zod_2.z.object({
-                    consumers: zod_2.z.array(zod_2.z.object({
-                        $schema: zod_2.z.string().describe('The schema of the consumer. This should always be the same.'),
-                        name: zod_2.z.string(),
-                        description: zod_2.z.string(),
-                        producers: zod_2.z.array(zod_2.z.object({
-                            name: zod_2.z.string().describe('References one of the producers. Must be unique, there can\'t be two entry with the same name.'),
-                            joins: zod_2.z.array(zod_2.z.object({
-                                otherName: zod_2.z.string(),
-                                relationship: zod_2.z.enum(['one-to-one', 'one-to-many', 'many-to-one']),
-                                sql: zod_2.z.string()
-                            })).optional().describe('Which other producer to join this one with. Omit if empty.')
-                        })),
-                        fields: zod_2.z.array(zod_2.z.object({
-                            key: zod_2.z.string(),
-                            from: zod_2.z.string().optional(),
-                            alias: zod_2.z.string().optional()
-                            // grouping: z.object({
-                            //     groupingKey: z.string(),
-                            //     subFields: z.array(z.lazy(() => z.object({
-                            //         key: z.string(),
-                            //         from: z.string().optional()
-                            //     })))
-                            // }).optional()
-                        })),
-                        outputs: zod_2.z.array(zod_2.z.object({
-                            format: zod_2.z.enum(['SQL', 'API', 'CSV', 'PARQUET', 'JSON'])
-                        }))
-                    }))
-                }), 'environment')
-            };
-            const res = yield this._client.chat.completions.create(item);
-            const msg = res.choices[0].message;
-            const finalDraft = JSON.parse(msg.content);
-            return finalDraft;
-        });
-        this._client = new openai_1.default({
-            apiKey: ProcessENVManager_1.default.getEnvVariable('OPENAI_API_KEY')
-        });
-    }
-}
-exports.default = LLM;

package/engines/consumer/ConsumerManager.js

@@ -1,218 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const Algo_1 = __importDefault(require("../../core/Algo"));
-const Environment_1 = __importDefault(require("../Environment"));
-const ProducerManager_1 = __importDefault(require("../producer/ProducerManager"));
-class ConsumerManagerClass {
-    constructor() {
-        /**
-         * Returns the full list of fields that are used by a consumer, while keeping the nested structure of fields.
-         * If there are *, then replace them with the actual fields found in the underlying producer/consumer
-         */
-        this.getExpandedFields = (consumer) => {
-            (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            const availableColumns = this.getAvailableColumns(consumer);
-            const convertedFields = consumer.fields.map(x => {
-                var _a;
-                return ({
-                    cField: x,
-                    finalKey: (_a = x.alias) !== null && _a !== void 0 ? _a : x.key
-                });
-            });
-            const expandedFields = convertedFields.flatMap(x => this.expandField(consumer, x, availableColumns));
-            return expandedFields;
-        };
-        /**
-         * Return all the available columns (dimensions and measures) to the consumer given its producers
-         */
-        this.getAvailableColumns = (consumer) => {
-            const availableColumns = consumer.producers.flatMap(cProd => {
-                var _a, _b;
-                const producer = Environment_1.default.getProducer(cProd.name);
-                if (!producer) {
-                    const subConsumer = Environment_1.default.getConsumer(cProd.name);
-                    (0, Affirm_1.default)(subConsumer, `No producer found with name "${cProd.name}"`);
-                    return this.getAvailableColumns(subConsumer);
-                }
-                else {
-                    const dims = producer.dimensions.map(x => ({
-                        consumerAlias: null,
-                        consumerKey: null,
-                        nameInProducer: x.name,
-                        aliasInProducer: x.alias,
-                        dimension: x,
-                        owner: cProd.name
-                    }));
-                    const meas = (_b = (_a = producer.measures) === null || _a === void 0 ? void 0 : _a.map(x => ({
-                        consumerAlias: null,
-                        consumerKey: null,
-                        nameInProducer: x.name,
-                        aliasInProducer: x.name,
-                        measure: x,
-                        owner: cProd.name
-                    }))) !== null && _b !== void 0 ? _b : [];
-                    return [...dims, ...meas];
-                }
-            });
-            return availableColumns;
-        };
-        /**
-         * If the field is '*' then replace them with the actual fields found in the underlying producer/consumer
-         */
-        this.expandField = (consumer, field, availableColumns) => {
-            var _a;
-            (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            (0, Affirm_1.default)(field, 'Invalid consumer field');
-            const expandedFields = [];
-            if (field.cField.key === '*') {
-                // If the producers are "union" then they have the same dimensions, meaning that I can use the "*"
-                if (consumer.producers.length > 1 && consumer.producers.every(x => x.union)) {
-                    const first = consumer.producers[0];
-                    const firstColumns = availableColumns.filter(x => x.owner === first.name);
-                    expandedFields.push(...firstColumns.map(x => ({
-                        cField: {
-                            key: x.nameInProducer,
-                            alias: x.nameInProducer,
-                            from: x.owner
-                        },
-                        finalKey: x.nameInProducer,
-                        dimension: x.dimension,
-                        measure: x.measure
-                    })));
-                }
-                else {
-                    const from = (_a = field.cField.from) !== null && _a !== void 0 ? _a : (consumer.producers.length === 1 ? consumer.producers[0].name : null);
-                    availableColumns.filter(x => x.owner === from).forEach(col => {
-                        expandedFields.push({
-                            cField: {
-                                key: col.nameInProducer,
-                                alias: col.nameInProducer,
-                                from: col.owner
-                            },
-                            finalKey: col.nameInProducer,
-                            dimension: col.dimension,
-                            measure: col.measure
-                        });
-                    });
-                }
-            }
-            else {
-                const col = ConsumerManager.searchFieldInColumns(field.cField, availableColumns, consumer);
-                (0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.cField.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
-                expandedFields.push(Object.assign(Object.assign({}, field), { dimension: col.dimension, measure: col.measure }));
-            }
-            return expandedFields;
-        };
-        this.searchFieldInColumns = (field, columns, consumer) => {
-            var _a, _b;
-            (0, Affirm_1.default)(field, 'Invalid field');
-            (0, Affirm_1.default)(columns, 'Invalid columns');
-            (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            let column = null;
-            if (field.from) {
-                column = columns.find(x => x.owner === field.from && x.nameInProducer === field.key);
-            }
-            else if (consumer.producers.length === 1 && !field.from) {
-                column = columns.find(x => x.nameInProducer === field.key);
-            }
-            else if (!field.fixed) {
-                const matches = columns.filter(x => x.nameInProducer === field.key);
-                (0, Affirm_1.default)(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map(x => x.name).join(', ')})`);
-                if (matches.length === 1) {
-                    // Need to check if the producers have "union" if they do, I don't care about this check
-                    const cProd = consumer.producers.find(x => x.name === matches[0].owner);
-                    if (!cProd.union)
-                        (0, Affirm_1.default)(matches.length === 1, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is ambiguos between the fields with same name from the producers: ${matches.map(x => x.owner).join(', ')}`);
-                }
-                column = matches[0];
-            }
-            if (!column) {
-                // If the consumer doesn't find the field in the producer but has a default value AND set_default onError
-                // then instead of failing, create a placeholder column for the producer
-                if (field.fixed === true && Algo_1.default.hasVal(field.default)) {
-                    column = {
-                        aliasInProducer: field.key,
-                        nameInProducer: (_a = field.alias) !== null && _a !== void 0 ? _a : field.key,
-                        consumerAlias: (_b = field.alias) !== null && _b !== void 0 ? _b : field.key,
-                        consumerKey: field.key,
-                        owner: field.from,
-                        dimension: { name: field.key, type: typeof field.default }
-                    };
-                }
-            }
-            return column;
-        };
-        this.getSource = (consumer) => {
-            const producers = consumer.producers.map(x => Environment_1.default.getProducer(x.name));
-            (0, Affirm_1.default)(producers.length > 0, 'No producers found');
-            (0, Affirm_1.default)(producers.every(x => x), `Invalid producer found in consumer "${consumer.name}"`);
-            const sources = producers.map(x => Environment_1.default.getSource(x.source));
-            (0, Affirm_1.default)(sources.length > 0, 'No sources found');
-            (0, Affirm_1.default)(sources.every(x => x), `Invalid source found in consumer "${consumer.name}"`);
-            // For now we only support connecting producers of the same engine type to a consumer, so we give an error if we detect different ones
-            const uniqEngines = Algo_1.default.uniqBy(sources, 'engine');
-            (0, Affirm_1.default)(uniqEngines.length === 1, `Sources with different engines were used in the consumer "${consumer.name}" (${uniqEngines.join(', ')})`);
-            // For now we also only support consumers that have producers ALL having the same exact source
-            const uniqNames = Algo_1.default.uniqBy(sources, 'name');
-            (0, Affirm_1.default)(uniqNames.length === 1, `Producers with different sources were used in the consumer "${consumer.name}" (${uniqNames.join(', ')})`);
-            return [sources[0], producers[0]];
-        };
-        this.compile = (consumer) => {
-            var _a, _b;
-            (0, Affirm_1.default)(consumer, `Invalid consumer`);
-            const availableColumns = this.getAvailableColumns(consumer);
-            const selectedColumns = [];
-            const flat = consumer.fields;
-            for (let i = 0; i < flat.length; i++) {
-                const field = flat[i];
-                // TODO: replace with the new funcitons in the consumermanager to reduce diplicate code
-                if (field.key === '*') {
-                    const from = (_a = field.from) !== null && _a !== void 0 ? _a : (consumer.producers.length === 1 ? consumer.producers[0].name : null);
-                    availableColumns.filter(x => x.owner === from).forEach(col => {
-                        col.consumerKey = col.nameInProducer;
-                        col.consumerAlias = col.nameInProducer;
-                        selectedColumns.push(col);
-                    });
-                }
-                else {
-                    const col = ConsumerManager.searchFieldInColumns(field, availableColumns, consumer);
-                    (0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
-                    col.consumerKey = field.key;
-                    col.consumerAlias = (_b = field.alias) !== null && _b !== void 0 ? _b : field.key;
-                    selectedColumns.push(col);
-                }
-            }
-            const columnsWithNoAlias = selectedColumns.filter(x => !x.consumerAlias || !x.consumerKey);
-            (0, Affirm_1.default)(columnsWithNoAlias.length === 0, `Consumer "${consumer.name}" compilation error: some selected fields don't have a correct alias or key (${columnsWithNoAlias.map(x => x.nameInProducer).join(', ')})`);
-            return selectedColumns;
-        };
-        this.getOutputShape = (consumer) => {
-            (0, Affirm_1.default)(consumer, `Invalid consumer`);
-            const compiled = this.compile(consumer);
-            const outDimensions = compiled.map(x => {
-                var _a, _b, _c, _d, _e, _f, _g;
-                return ({
-                    name: (_a = x.consumerAlias) !== null && _a !== void 0 ? _a : x.consumerKey,
-                    type: (_b = x.dimension) === null || _b === void 0 ? void 0 : _b.type,
-                    classification: (_c = x.dimension) === null || _c === void 0 ? void 0 : _c.classification,
-                    description: (_e = (_d = x.dimension) === null || _d === void 0 ? void 0 : _d.description) !== null && _e !== void 0 ? _e : (_f = x.measure) === null || _f === void 0 ? void 0 : _f.description,
-                    mask: ProducerManager_1.default.getMask(x.dimension),
-                    pk: (_g = x.dimension) === null || _g === void 0 ? void 0 : _g.pk
-                });
-            });
-            return {
-                _version: consumer._version,
-                name: consumer.name,
-                description: consumer.description,
-                metadata: consumer.metadata,
-                dimensions: outDimensions
-            };
-        };
-    }
-}
-const ConsumerManager = new ConsumerManagerClass();
-exports.default = ConsumerManager;

package/engines/consumer/ConsumerOnFinishManager.js

@@ -1,202 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const Environment_1 = __importDefault(require("../Environment"));
-const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
-class ConsumerOnFinishManagerClass {
-    constructor() {
-        this.onConsumerSuccess = (consumer, executionId) => __awaiter(this, void 0, void 0, function* () {
-            void executionId;
-            for (const output of consumer.outputs) {
-                if (output.onSuccess)
-                    yield this.performOnSuccessActions(consumer, output);
-            }
-        });
-        this.onConsumerError = (consumer, executionId) => __awaiter(this, void 0, void 0, function* () {
-            void executionId;
-            for (const output of consumer.outputs) {
-                if (output.onError)
-                    yield this.performOnSuccessActions(consumer, output);
-            }
-        });
-        this.performOnSuccessActions = (consumer, output) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            (0, Affirm_1.default)(output, 'Invalid output');
-            if (!output.onSuccess || output.onSuccess.length === 0)
-                return;
-            for (const onSuccess of output.onSuccess) {
-                switch (onSuccess.action) {
-                    case 'move-file': {
-                        yield this.moveSourceFiles(consumer, onSuccess.moveToDestination);
-                        break;
-                    }
-                    default:
-                        throw new Error(`On success action "${onSuccess.action}" is not implemented yet.`);
-                }
-            }
-        });
-        this.moveSourceFiles = (consumer, moveDestination) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(moveDestination, 'Move destination is required for move-file action');
-            const destinationSource = Environment_1.default.getSource(moveDestination);
-            (0, Affirm_1.default)(destinationSource, `Destination source "${moveDestination}" not found`);
-            // Get all unique source files from all producers used by this consumer
-            const sourceFilesToMove = [];
-            for (const consumerProducer of consumer.producers) {
-                const producer = Environment_1.default.getProducer(consumerProducer.name);
-                (0, Affirm_1.default)(producer, `Producer "${consumerProducer.name}" not found`);
-                const source = Environment_1.default.getSource(producer.source);
-                (0, Affirm_1.default)(source, `Source "${producer.source}" not found`);
-                // Only handle file-based sources that have fileKey
-                if (producer.settings.fileKey) {
-                    // Handle wildcard patterns (files with %)
-                    if (producer.settings.fileKey.includes('%')) {
-                        // Get all files matching the pattern
-                        const matchingFiles = yield this.getMatchingFiles(source, producer.settings.fileKey);
-                        for (const fileKey of matchingFiles) {
-                            sourceFilesToMove.push({
-                                sourceName: producer.source,
-                                filePath: this.getSourceFilePath(source, fileKey),
-                                fileKey
-                            });
-                        }
-                    }
-                    else {
-                        // Single file
-                        sourceFilesToMove.push({
-                            sourceName: producer.source,
-                            filePath: this.getSourceFilePath(source, producer.settings.fileKey),
-                            fileKey: producer.settings.fileKey
-                        });
-                    }
-                }
-            }
-            // Move all files to destination
-            yield this.moveFiles(sourceFilesToMove, destinationSource);
-        });
-        this.getMatchingFiles = (source, fileKeyPattern) => __awaiter(this, void 0, void 0, function* () {
-            const sourceDriver = yield DriverFactory_1.default.instantiateSource(source);
-            if (source.engine === 'aws-s3') {
-                // S3 driver has a public listFiles method that handles patterns
-                const s3Driver = sourceDriver;
-                return yield s3Driver.listFiles(fileKeyPattern);
-            }
-            else if (source.engine === 'local') {
-                // Local driver now has a public listFiles method consistent with S3
-                const localDriver = sourceDriver;
-                return localDriver.listFiles(fileKeyPattern);
-            }
-            throw new Error(`Unsupported source engine for file listing: ${source.engine}`);
-        });
-        this.getSourceFilePath = (source, fileKey) => {
-            if (source.engine === 'local') {
-                return `${source.authentication['path']}/${fileKey}`;
-            }
-            else if (source.engine === 'aws-s3') {
-                // For S3, we return the key as the path since S3 uses keys instead of file paths
-                return fileKey;
-            }
-            throw new Error(`Unsupported source engine for file move: ${source.engine}`);
-        };
-        this.moveFiles = (files, destinationSource) => __awaiter(this, void 0, void 0, function* () {
-            for (const file of files) {
-                const sourceSource = Environment_1.default.getSource(file.sourceName);
-                if (sourceSource.engine === 'local' && destinationSource.engine === 'local') {
-                    // Local to Local move
-                    yield this.moveLocalToLocal(file.filePath, destinationSource, file.fileKey);
-                }
-                else if (sourceSource.engine === 'local' && destinationSource.engine === 'aws-s3') {
-                    // Local to S3 move
-                    yield this.moveLocalToS3(file.filePath, destinationSource, file.fileKey);
-                }
-                else if (sourceSource.engine === 'aws-s3' && destinationSource.engine === 'local') {
-                    // S3 to Local move
-                    yield this.moveS3ToLocal(sourceSource, file.fileKey, destinationSource);
-                }
-                else if (sourceSource.engine === 'aws-s3' && destinationSource.engine === 'aws-s3') {
-                    // S3 to S3 move
-                    yield this.moveS3ToS3(sourceSource, file.fileKey, destinationSource);
-                }
-                else {
-                    throw new Error(`Unsupported move operation from ${sourceSource.engine} to ${destinationSource.engine}`);
-                }
-            }
-        });
-        this.moveLocalToLocal = (sourceFilePath, destinationSource, fileKey) => __awaiter(this, void 0, void 0, function* () {
-            const sourceDriver = yield DriverFactory_1.default.instantiateSource(this.findSourceForPath(sourceFilePath));
-            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
-            // Read file from source
-            const fileContent = sourceDriver.readFile(fileKey);
-            // Save to destination
-            yield destinationDriver.saveFile(fileKey, fileContent);
-            // Delete from source
-            sourceDriver.deleteFile(fileKey);
-        });
-        this.findSourceForPath = (filePath) => {
-            // Extract directory from file path for source creation
-            const directory = filePath.substring(0, filePath.lastIndexOf('/'));
-            return {
-                name: 'temp-source',
-                engine: 'local',
-                authentication: { path: directory }
-            };
-        };
-        this.moveLocalToS3 = (sourceFilePath, destinationSource, fileKey) => __awaiter(this, void 0, void 0, function* () {
-            const sourceDriver = yield DriverFactory_1.default.instantiateSource(this.findSourceForPath(sourceFilePath));
-            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
-            // Read file from local source
-            const fileContent = sourceDriver.readFile(fileKey);
-            // Upload to S3 destination
-            yield destinationDriver.saveFile(fileKey, fileContent);
-            // Remove source file after successful upload
-            sourceDriver.deleteFile(fileKey);
-        });
-        this.moveS3ToLocal = (sourceSource, fileKey, destinationSource) => __awaiter(this, void 0, void 0, function* () {
-            const sourceDriver = yield DriverFactory_1.default.instantiateSource(sourceSource);
-            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
-            // Download from S3
-            const content = yield sourceDriver.downloadFile(fileKey);
-            // Save to local destination
-            yield destinationDriver.saveFile(fileKey, content);
-            // Delete from S3 source
-            yield sourceDriver.deleteFile(fileKey);
-        });
-        this.moveS3ToS3 = (sourceSource, fileKey, destinationSource) => __awaiter(this, void 0, void 0, function* () {
-            const sourceDriver = yield DriverFactory_1.default.instantiateSource(sourceSource);
-            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
-            // Copy from source S3 to destination S3
-            yield destinationDriver.copyFromS3(sourceSource.authentication['bucket'], fileKey, fileKey);
-            // Delete from source S3
-            yield sourceDriver.deleteFile(fileKey);
-        });
-        this.performOnErrorActions = (consumer, output) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            (0, Affirm_1.default)(output, 'Invalid output');
-            if (!output.onError || output.onError.length === 0)
-                return;
-            for (const onError of output.onError) {
-                switch (onError.action) {
-                    case 'move-file': {
-                        yield this.moveSourceFiles(consumer, onError.moveToDestination);
-                        break;
-                    }
-                    default:
-                        throw new Error(`On success action "${onError.action}" is not implemented yet.`);
-                }
-            }
-        });
-    }
-}
-const ConsumerOnFinishManager = new ConsumerOnFinishManagerClass();
-exports.default = ConsumerOnFinishManager;