@forzalabs/remora 1.0.21 → 1.1.1

package/engines/parsing/ParseCompression.js

@@ -1,101 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const fs = __importStar(require("fs"));
-const decompress_1 = __importDefault(require("decompress"));
-const zlib_1 = __importDefault(require("zlib"));
-const path_1 = __importDefault(require("path"));
-const promises_1 = require("stream/promises");
-class ParseCompressionClass {
-    constructor() {
-        this.decompressToFile = (compressionType, fileKey, standardPath, executionPath) => __awaiter(this, void 0, void 0, function* () {
-            // Ensure the directory for the file exists
-            const fileDir = path_1.default.dirname(executionPath);
-            if (!fs.existsSync(fileDir)) {
-                fs.mkdirSync(fileDir, { recursive: true });
-            }
-            switch (compressionType === null || compressionType === void 0 ? void 0 : compressionType.toUpperCase()) {
-                case 'ZIP':
-                case 'TAR': {
-                    this._warningPossibleConvertionError(compressionType, fileKey);
-                    yield (0, decompress_1.default)(path_1.default.join(standardPath, fileKey), executionPath);
-                    return executionPath;
-                }
-                case 'GZ': {
-                    this._warningPossibleConvertionError(compressionType, fileKey);
-                    yield this._gzipFile(fileKey, standardPath, executionPath);
-                    return executionPath;
-                }
-                case null:
-                case undefined: {
-                    if (fileKey.includes('.zip') || fileKey.includes('.tar') || fileKey.includes('.gz'))
-                        console.log(`The file ${fileKey} seems to use a compressionType while there are no producer settings for compressionType `);
-                    // No decompression needed for request
-                    return path_1.default.join(standardPath, fileKey);
-                }
-                default: {
-                    // throw an error if the request ask for an unsupported compression type
-                    throw new Error(`this compression type it has not been implemented ${compressionType}`);
-                }
-            }
-        });
-        this._warningPossibleConvertionError = (compression, fileKey) => {
-            if (!fileKey.split('.').includes(compression.toLowerCase()) && compression) {
-                console.warn(`The compressionType ${compression.toUpperCase()} seems different to the file compression of ${fileKey}`);
-            }
-        };
-        this._gzipFile = (fileKey, standardPath, localPath) => __awaiter(this, void 0, void 0, function* () {
-            const fileContents = fs.createReadStream(path_1.default.join(standardPath, fileKey));
-            const unzip = zlib_1.default.createGunzip();
-            const writeStream = fs.createWriteStream(localPath);
-            yield (0, promises_1.pipeline)(fileContents, unzip, writeStream);
-        });
-        // TODO add a decompression method fo tarball (.tar.gz)
-    }
-}
-const ParseCompression = new ParseCompressionClass();
-exports.default = ParseCompression;

package/engines/parsing/ParseHelper.js

@@ -1,18 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const ParseHelper = {
-    shouldHaveHeader: (fileType, hasHeaderRow) => {
-        if (fileType === 'CSV' || fileType === 'XLS' || fileType === 'XLSX' || fileType === 'TXT') {
-            if (fileType !== 'TXT')
-                return true;
-            else if (fileType === 'TXT' && (hasHeaderRow === true || hasHeaderRow === undefined))
-                return true;
-            else
-                return false;
-        }
-        else {
-            return false;
-        }
-    }
-};
-exports.default = ParseHelper;

package/engines/parsing/ParseManager.js

@@ -1,54 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const CSVParser_1 = __importDefault(require("./CSVParser"));
-const Constants_1 = __importDefault(require("../../Constants"));
-const ProducerManager_1 = __importDefault(require("../producer/ProducerManager"));
-class ParseManagerClass {
-    constructor() {
-        this._extractHeader = (headerLine, delimiter, producer, discover) => {
-            var _a, _b, _c, _d, _e, _f;
-            (0, Affirm_1.default)(headerLine, `Invalid CSV header line for producer "${producer.name}"`);
-            (0, Affirm_1.default)(delimiter, 'Invalid CSV delimiter');
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            let columns = ProducerManager_1.default.getColumns(producer);
-            const headerColumns = CSVParser_1.default.parseRow(headerLine, delimiter).map(x => x.trim());
-            // If I'm discovering the file, then it means that the dimensions are not set, so I use the ones that I get from the file directly
-            if (discover)
-                columns = headerColumns.map(x => ({ nameInProducer: x }));
-            const csvColumns = [];
-            for (const pColumn of columns) {
-                // Skip sourceFilename dimensions - they don't exist in the source file
-                // They are added dynamically by the driver when reading the file
-                if (((_a = pColumn.dimension) === null || _a === void 0 ? void 0 : _a.sourceFilename) === true) {
-                    // Find the index of $source_filename in the header (it was added by the driver)
-                    const sourceFilenameIndex = headerColumns.findIndex(x => x === Constants_1.default.SOURCE_FILENAME_COLUMN);
-                    if (sourceFilenameIndex > -1) {
-                        csvColumns.push({
-                            index: sourceFilenameIndex,
-                            name: Constants_1.default.SOURCE_FILENAME_COLUMN,
-                            saveAs: pColumn.nameInProducer,
-                            type: (_c = (_b = pColumn.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string'
-                        });
-                    }
-                    continue;
-                }
-                const columnKey = (_d = pColumn.aliasInProducer) !== null && _d !== void 0 ? _d : pColumn.nameInProducer;
-                const csvColumnIndex = headerColumns.findIndex(x => x === columnKey);
-                (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
-                csvColumns.push({
-                    index: csvColumnIndex,
-                    name: columnKey,
-                    saveAs: pColumn.nameInProducer,
-                    type: (_f = (_e = pColumn.dimension) === null || _e === void 0 ? void 0 : _e.type) !== null && _f !== void 0 ? _f : 'string'
-                });
-            }
-            return csvColumns;
-        };
-    }
-}
-const ParseManager = new ParseManagerClass();
-exports.default = ParseManager;

package/engines/parsing/XLSParser.js

@@ -1,87 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const DriverHelper_1 = __importDefault(require("../../drivers/DriverHelper"));
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const xlsx_1 = __importDefault(require("xlsx"));
-class XLSParserClass {
-    constructor() {
-        this.getStreamXls = (filePath, sheetName) => __awaiter(this, void 0, void 0, function* () {
-            const fileXls = xlsx_1.default.readFile(filePath);
-            (0, Affirm_1.default)(fileXls.SheetNames.length > 0, 'The Excel file has no sheets.');
-            const targetSheetName = yield this.getTargetSheetName(fileXls, sheetName);
-            const workSheet = fileXls.Sheets[targetSheetName];
-            const xlsStream = xlsx_1.default.stream.to_csv(workSheet);
-            return xlsStream;
-        });
-        this.getHeaderXls = (filePath, sheetName) => __awaiter(this, void 0, void 0, function* () {
-            const header = yield DriverHelper_1.default.quickReadStream(yield this.getStreamXls(filePath, sheetName), 2);
-            return header[0];
-        });
-        this.getHeaderXlsFromStream = (stream, sheetName) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_1, _b, _c;
-            let header;
-            const workbook = xlsx_1.default.read(stream);
-            const targetSheetName = yield this.getTargetSheetName(workbook, sheetName);
-            const xlsxStream = xlsx_1.default.stream.to_csv(workbook.Sheets[targetSheetName]);
-            try {
-                for (var _d = true, xlsxStream_1 = __asyncValues(xlsxStream), xlsxStream_1_1; xlsxStream_1_1 = yield xlsxStream_1.next(), _a = xlsxStream_1_1.done, !_a; _d = true) {
-                    _c = xlsxStream_1_1.value;
-                    _d = false;
-                    const line = _c;
-                    if (line != '') {
-                        header = line;
-                        break;
-                    }
-                }
-            }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = xlsxStream_1.return)) yield _b.call(xlsxStream_1);
-                }
-                finally { if (e_1) throw e_1.error; }
-            }
-            (0, Affirm_1.default)(header, `the file received from the stream is empty`);
-            return header;
-        });
-        this.getTargetSheetName = (workbook, sheetName) => __awaiter(this, void 0, void 0, function* () {
-            let targetSheetName = sheetName ? sheetName : workbook.SheetNames[0];
-            if (!targetSheetName) {
-                (0, Affirm_1.default)(workbook.SheetNames.length > 0, 'The Excel file has no sheets.');
-                targetSheetName = workbook.SheetNames[0];
-            }
-            else {
-                (0, Affirm_1.default)(workbook.SheetNames.includes(targetSheetName), `The sheet "${targetSheetName}" doesn't exist in the excel (available: ${workbook.SheetNames.join(', ')})`);
-            }
-            return targetSheetName;
-        });
-        this.parseXLSStream = (xlsStream, sheetName) => __awaiter(this, void 0, void 0, function* () {
-            const workbook = xlsx_1.default.read(xlsStream);
-            const targetSheetName = yield this.getTargetSheetName(workbook, sheetName);
-            const workSheet = workbook.Sheets[targetSheetName];
-            const csvStream = xlsx_1.default.stream.to_csv(workSheet);
-            return csvStream;
-        });
-    }
-}
-const XLSParser = new XLSParserClass();
-exports.default = XLSParser;

package/engines/parsing/XMLParser.js

@@ -1,115 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const fast_xml_parser_1 = require("fast-xml-parser");
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const Algo_1 = __importDefault(require("../../core/Algo"));
-const fs = __importStar(require("fs"));
-const DEFAULT_OPTIONS = {
-    attributeNamePrefix: '@_',
-    ignoreAttributes: false,
-    parseAttributeValue: true,
-    parseTagValue: true,
-    trimValues: true,
-    isArray: () => {
-        return false;
-    }
-};
-class XMLParserClass {
-    constructor(options) {
-        this.xmlToJson = (xmlData) => {
-            (0, Affirm_1.default)(xmlData, 'XML data cannot be empty');
-            try {
-                const parsedData = this._parser.parse(xmlData);
-                if (typeof parsedData === 'object' && parsedData !== null) {
-                    const rootKeys = Object.keys(parsedData);
-                    if (rootKeys.length === 1) {
-                        const potentialArray = parsedData[rootKeys[0]];
-                        if (Array.isArray(potentialArray)) {
-                            return potentialArray;
-                        }
-                        if (typeof potentialArray === 'object' && potentialArray !== null) {
-                            const innerKeys = Object.keys(potentialArray);
-                            if (innerKeys.length === 1 && Array.isArray(potentialArray[innerKeys[0]])) {
-                                return potentialArray[innerKeys[0]];
-                            }
-                            if (Array.isArray(potentialArray)) {
-                                return potentialArray;
-                            }
-                            const values = Object.values(potentialArray).filter(Array.isArray);
-                            if (values.length === 1 && Array.isArray(values[0])) {
-                                return values[0];
-                            }
-                            return [potentialArray];
-                        }
-                        return [potentialArray];
-                    }
-                }
-                return parsedData;
-            }
-            catch (error) {
-                console.error('Error parsing XML:', error);
-                throw new Error('Failed to parse XML data.');
-            }
-        };
-        // TO DO: improve performances for larger file using streams instead of an array of string
-        this.readXmlLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
-            const fileContent = fs.readFileSync(fileUri, 'utf-8');
-            const jsonData = XMLParser.xmlToJson(fileContent);
-            // Convert JSON data to string lines. This might need adjustment based on XML structure.
-            // Assuming jsonData is an array of objects, where each object is a record.
-            let lines = Array.isArray(jsonData) ? jsonData.map(item => JSON.stringify(item)) : [JSON.stringify(jsonData)];
-            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
-                lines = lines.slice(lineFrom, lineTo + 1);
-            }
-            return lines;
-        });
-        this._parser = new fast_xml_parser_1.XMLParser(Object.assign(Object.assign({}, DEFAULT_OPTIONS), options)); // Use 'as any' if type issues persist with library
-    }
-}
-const XMLParser = new XMLParserClass();
-exports.default = XMLParser;

package/engines/producer/ProducerEngine.js

@@ -1,127 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
-const Environment_1 = __importDefault(require("../Environment"));
-const DatasetManager_1 = __importDefault(require("../dataset/DatasetManager"));
-const Logger_1 = __importDefault(require("../../helper/Logger"));
-class ProducerEngineClass {
-    constructor() {
-        this.readFile = (producer, options) => __awaiter(this, void 0, void 0, function* () {
-            var _a, _b, _c;
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            (0, Affirm_1.default)(options, 'Invalid options');
-            if (options.readmode === 'lines')
-                (0, Affirm_1.default)(options.lines, 'Invalid lines');
-            const source = Environment_1.default.getSource(producer.source);
-            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
-            const driver = yield DriverFactory_1.default.instantiateSource(source);
-            (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
-            const { settings: { fileKey, fileType, sheetName, hasHeaderRow } } = producer;
-            let dataset = DatasetManager_1.default.create(producer);
-            let lines = [];
-            switch (options.readmode) {
-                case 'lines':
-                    lines = yield driver.readLinesInRange({
-                        fileKey,
-                        fileType,
-                        options: { lineFrom: options.lines.from, lineTo: options.lines.to, sheetName, hasHeaderRow },
-                        httpApi: (_a = producer.settings) === null || _a === void 0 ? void 0 : _a.httpApi
-                    });
-                    break;
-                case 'all':
-                    lines = yield driver.readAll({
-                        fileKey, fileType,
-                        options: { sheetName, hasHeaderRow },
-                        httpApi: (_b = producer.settings) === null || _b === void 0 ? void 0 : _b.httpApi
-                    });
-                    break;
-                case 'download':
-                    dataset = yield driver.download(dataset);
-                    break;
-            }
-            switch ((_c = producer.settings.fileType) === null || _c === void 0 ? void 0 : _c.toUpperCase()) {
-                case 'CSV':
-                case 'TXT':
-                    return { data: lines, dataset, dataType: 'lines-of-text' };
-                case 'XLS':
-                case 'XLSX':
-                    return { data: lines, dataset, dataType: 'lines-of-text' };
-                case 'PARQUET':
-                case 'JSONL':
-                case 'JSON': {
-                    if (lines.length === 1) {
-                        // Attempt to handle cases where a single line might contain multiple JSON objects separated by newlines
-                        // Or if the entire file content is a single JSON array stringified.
-                        try {
-                            const parsedAsArray = JSON.parse(lines[0]);
-                            if (Array.isArray(parsedAsArray)) {
-                                return { data: parsedAsArray, dataset, dataType: 'array-of-json' };
-                            }
-                        }
-                        catch (error) {
-                            // If parsing as array fails, proceed to split by newline
-                            console.warn('Failed to parse single line as JSON array, splitting by newline:', error);
-                        }
-                        lines = lines[0].split('\\n');
-                    }
-                    const json = lines.filter(line => line.trim() !== '').map(x => JSON.parse(x));
-                    return { data: json, dataset, dataType: 'array-of-json' };
-                }
-                case 'XML': {
-                    // The driver's _readXmlLines method now returns an array of JSON strings.
-                    // Each string needs to be parsed into a JSON object.
-                    const json = lines.filter(line => line.trim() !== '').map(x => JSON.parse(x));
-                    return { data: json, dataset, dataType: 'array-of-json' };
-                }
-                default:
-                    throw new Error(`Invalid file type "${producer.settings.fileType}" for engine type "${source.engine}" for producer "${producer.name}": not supported`);
-            }
-        });
-        this.readSampleData = (producer_1, ...args_1) => __awaiter(this, [producer_1, ...args_1], void 0, function* (producer, sampleSize = 10, discover = false) {
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            (0, Affirm_1.default)(sampleSize > 0, 'Sample size must be greater than 0');
-            const source = Environment_1.default.getSource(producer.source);
-            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
-            let dataset = DatasetManager_1.default.create(producer);
-            switch (source.engine) {
-                case 'aws-redshift': {
-                    const sql = `SELECT * FROM "${source.authentication['schema']}"."${producer.settings.sqlTable}" LIMIT ${sampleSize}`;
-                    (0, Affirm_1.default)(sql, `Invalid SQL from deployment compilation for producer "${producer.name}"`);
-                    const driver = yield DriverFactory_1.default.instantiateSource(source);
-                    (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
-                    const res = yield driver.query(sql);
-                    dataset = yield dataset.loadFromMemory(res.rows, producer, discover);
-                    break;
-                }
-                case 'local':
-                case 'aws-s3':
-                case 'delta-share': {
-                    const fileData = yield this.readFile(producer, { readmode: 'lines', lines: { from: 0, to: sampleSize } });
-                    dataset = yield dataset.loadFromMemory(fileData.data, producer, discover);
-                    break;
-                }
-                default:
-                    throw new Error(`Invalid engine type "${source.engine}" for producer "${producer.name}": not supported`);
-            }
-            const sampleData = [...yield dataset.readLines(sampleSize)];
-            dataset.destroy();
-            Logger_1.default.log(`Finished reading sample dataset:\n${dataset.printStats()}`);
-            return sampleData;
-        });
-    }
-}
-const ProducerEngine = new ProducerEngineClass();
-exports.default = ProducerEngine;

package/engines/producer/ProducerManager.js

@@ -1,43 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const SecretManager_1 = __importDefault(require("../SecretManager"));
-class ProducerManagerClass {
-    constructor() {
-        this.getColumns = (producer) => {
-            var _a;
-            (0, Affirm_1.default)(producer, `Invalid producer`);
-            (0, Affirm_1.default)((_a = producer.settings.fileKey) !== null && _a !== void 0 ? _a : producer.settings.sqlTable, `Missing required file key in producer settings`);
-            (0, Affirm_1.default)(producer.settings.fileType, `Missing required file type in producer settings`);
-            const columns = producer.dimensions.map(x => ({
-                aliasInProducer: x.alias,
-                nameInProducer: x.name,
-                consumerAlias: null,
-                consumerKey: null,
-                owner: producer.name,
-                dimension: x
-            }));
-            return columns;
-        };
-        this.getMask = (dimension) => {
-            if (!dimension || !dimension.mask)
-                return null;
-            const mask = SecretManager_1.default.replaceSecret(dimension.mask);
-            const allowedMaskValues = [
-                'hash',
-                'mask',
-                'crypt',
-                'random',
-                'seeded-random',
-                'none'
-            ];
-            (0, Affirm_1.default)(allowedMaskValues.includes(mask), `Mask value "${mask}" in dimension "${dimension.name}" is not a valid value.`);
-            return mask;
-        };
-    }
-}
-const ProducerManager = new ProducerManagerClass();
-exports.default = ProducerManager;