@forzalabs/remora 0.1.8-nasco.3 → 0.2.0

package/drivers/DriverHelper.js

@@ -28,12 +28,17 @@ const Affirm_1 = __importDefault(require("../core/Affirm"));
 const DriverHelper = {
     appendToUnifiedFile: (options) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(options, 'Invalid options');
-        const { append, destinationPath, fileKey, headerLine, stream, fileType, hasHeaderRow, delimiter } = options;
+        const { append, destinationPath, fileKey, headerLine, stream, fileType, hasHeaderRow, delimiter, sourceFilename } = options;
         (0, Affirm_1.default)(headerLine, `Invalid header line`);
         const keys = (fileType === 'JSON' || fileType === 'JSONL')
             ? Object.keys(JSON.parse(headerLine))
             : [];
         const shouldValidateHeader = fileType === 'CSV' || (fileType === 'TXT' && hasHeaderRow === true);
+        // When sourceFilename is set, the headerLine includes $source_filename at the end.
+        // For validation, we need to compare against the original header without this suffix.
+        const originalHeaderLine = sourceFilename
+            ? headerLine.slice(0, headerLine.lastIndexOf(delimiter))
+            : headerLine;
         let isFirstLine = true;
         let hasValidatedHeader = shouldValidateHeader ? false : true;
         let leftoverData = '';
@@ -50,8 +55,8 @@ const DriverHelper = {
                     const line = lines[i];
                     // Header validation for first line
                     if (!hasValidatedHeader && isFirstLine && i === 0) {
-                        if (shouldValidateHeader && headerLine && headerLine.trim() !== '' && line.trim() !== headerLine.trim()) {
-                            const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${line}\n\t-main: ${headerLine}`;
+                        if (shouldValidateHeader && originalHeaderLine && originalHeaderLine.trim() !== '' && line.trim() !== originalHeaderLine.trim()) {
+                            const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${line}\n\t-main: ${originalHeaderLine}`;
                             Logger_1.default.log(msg);
                             return callback(new Error(msg));
                         }
@@ -77,7 +82,7 @@ const DriverHelper = {
                 // Process any remaining data
                 if (leftoverData.trim()) {
                     if (shouldIncludeLine(leftoverData, -1)) {
-                        callback(null, Buffer.from(processLine(leftoverData)));
+                        callback(null, Buffer.from(processLine(leftoverData) + '\n'));
                     }
                     else {
                         callback(null, null);
@@ -101,21 +106,28 @@ const DriverHelper = {
         };
         const processLine = (line) => {
             lineCount++;
+            let processedLine;
             switch (fileType) {
                 case 'JSON':
                 case 'JSONL': {
                     try {
                         const parsed = JSON.parse(line);
-                        return keys.map(k => parsed[k]).join(delimiter);
+                        processedLine = keys.map(k => parsed[k]).join(delimiter);
                     }
                     catch (error) {
                         Logger_1.default.log(`Failed parsing in JSON line -> file: ${fileKey}; index: ${globalIndex}; line: ${line}; err: ${error === null || error === void 0 ? void 0 : error.name}`);
                         throw error;
                     }
+                    break;
                 }
                 default:
-                    return line;
+                    processedLine = line;
+            }
+            // If sourceFilename is provided, append it to each line
+            if (sourceFilename) {
+                processedLine = processedLine + delimiter + sourceFilename;
             }
+            return processedLine;
         };
         const writeOptions = append ? { flags: 'a' } : {};
         const writeStream = (0, fs_1.createWriteStream)(destinationPath, writeOptions);

package/drivers/HttpApiDriver.js

@@ -0,0 +1,204 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpApiSourceDriver = void 0;
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
+const Algo_1 = __importDefault(require("../core/Algo"));
+const Logger_1 = __importDefault(require("../helper/Logger"));
+const DriverHelper_1 = __importDefault(require("./DriverHelper"));
+class HttpApiSourceDriver {
+    constructor() {
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(source, 'Invalid source');
+            (0, Affirm_1.default)(source.authentication, 'Invalid authentication');
+            (0, Affirm_1.default)(source.authentication.url, 'HTTP API source requires a URL in authentication.url');
+            this._source = source;
+            this._baseUrl = SecretManager_1.default.replaceSecret(source.authentication.url);
+            this._httpMethod = source.authentication.httpMethod || 'GET';
+            this._timeout = source.authentication.timeout || 30000; // 30 seconds default
+            this._headers = source.authentication.headers ? Object.assign({}, source.authentication.headers) : {};
+            this._queryParams = source.authentication.queryParams ? Object.assign({}, source.authentication.queryParams) : {};
+            // Handle different authentication methods
+            switch (source.authentication.method) {
+                case 'bearer-token': {
+                    (0, Affirm_1.default)(source.authentication.bearerToken, 'Bearer token authentication requires bearerToken');
+                    this._headers['Authorization'] = `Bearer ${SecretManager_1.default.replaceSecret(source.authentication.bearerToken)}`;
+                    break;
+                }
+                case 'api-key': {
+                    (0, Affirm_1.default)(source.authentication.apiKey, 'API key authentication requires apiKey');
+                    const apiKeyHeader = source.authentication.apiKeyHeader || 'X-API-Key';
+                    this._headers[apiKeyHeader] = SecretManager_1.default.replaceSecret(source.authentication.apiKey);
+                    break;
+                }
+                case 'username-password': {
+                    (0, Affirm_1.default)(source.authentication.user && source.authentication.password, 'Username-password authentication requires user and password');
+                    const credentials = Buffer.from(`${SecretManager_1.default.replaceSecret(source.authentication.user)}:${SecretManager_1.default.replaceSecret(source.authentication.password)}`).toString('base64');
+                    this._headers['Authorization'] = `Basic ${credentials}`;
+                    break;
+                }
+                case 'none':
+                    // No authentication required
+                    break;
+                default:
+                    throw new Error(`Authentication method "${source.authentication.method}" is not supported for HTTP API sources`);
+            }
+            // Test connection
+            try {
+                yield this._makeRequest(this._baseUrl);
+                Logger_1.default.log(`HTTP API connection to ${this._baseUrl} successful`);
+            }
+            catch (error) {
+                throw new Error(`Failed to connect to HTTP API at ${this._baseUrl}: ${error.message}`);
+            }
+            return this;
+        });
+        this._makeRequest = (url, options) => __awaiter(this, void 0, void 0, function* () {
+            const method = (options === null || options === void 0 ? void 0 : options.method) || this._httpMethod;
+            const headers = Object.assign(Object.assign({}, this._headers), options === null || options === void 0 ? void 0 : options.additionalHeaders);
+            const queryParams = Object.assign(Object.assign({}, this._queryParams), options === null || options === void 0 ? void 0 : options.additionalQueryParams);
+            // Build URL with query parameters
+            const urlWithParams = new URL(url);
+            Object.entries(queryParams).forEach(([key, value]) => {
+                urlWithParams.searchParams.append(key, value);
+            });
+            const fetchOptions = {
+                method,
+                headers,
+                signal: AbortSignal.timeout(this._timeout)
+            };
+            if ((options === null || options === void 0 ? void 0 : options.body) && (method === 'POST' || method === 'PUT' || method === 'PATCH')) {
+                fetchOptions.body = typeof options.body === 'string'
+                    ? options.body
+                    : JSON.stringify(options.body);
+                if (!headers['Content-Type']) {
+                    headers['Content-Type'] = 'application/json';
+                }
+            }
+            const response = yield fetch(urlWithParams.toString(), fetchOptions);
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
+            const contentType = response.headers.get('content-type');
+            if (contentType === null || contentType === void 0 ? void 0 : contentType.includes('application/json')) {
+                return yield response.json();
+            }
+            else {
+                return yield response.text();
+            }
+        });
+        this.execute = (_sql) => __awaiter(this, void 0, void 0, function* () {
+            void _sql;
+            throw new Error('SQL execution is not supported for HTTP API sources. Use query() or readAll() instead.');
+        });
+        this.query = (_sql, _values) => __awaiter(this, void 0, void 0, function* () {
+            void _sql;
+            void _values;
+            throw new Error('SQL queries are not supported for HTTP API sources. Use readAll() to fetch data from the API.');
+        });
+        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
+            try {
+                const endpoint = producer.settings.fileKey || '';
+                const url = endpoint.startsWith('http') ? endpoint : `${this._baseUrl}${endpoint}`;
+                yield this._makeRequest(url, { method: 'HEAD' });
+                return true;
+            }
+            catch (error) {
+                if (error.message.includes('404')) {
+                    return false;
+                }
+                throw error;
+            }
+        });
+        this.readAll = (request, values) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(request, 'Invalid read request');
+            (0, Affirm_1.default)(request.fileKey, 'Invalid file key (endpoint path)');
+            const endpoint = request.fileKey;
+            const url = endpoint.startsWith('http') ? endpoint : `${this._baseUrl}${endpoint}`;
+            // Convert IQueryParameter[] to query params if provided
+            const additionalQueryParams = {};
+            if (values && values.length > 0) {
+                values.forEach(param => {
+                    additionalQueryParams[param.name] = param.value;
+                });
+            }
+            const data = yield this._makeRequest(url, { additionalQueryParams });
+            // Convert response to string array (lines)
+            return this._extractObjectsFromResponse(data, request.httpApi).map(x => JSON.stringify(x));
+        });
+        this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(request, 'Invalid read request');
+            (0, Affirm_1.default)(request.options, 'Invalid read request options');
+            const allLines = yield this.readAll(request);
+            const { lineFrom, lineTo } = request.options;
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                return allLines.slice(lineFrom, lineTo);
+            }
+            return allLines;
+        });
+        this.download = (dataset) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(dataset, 'Invalid dataset');
+            const file = dataset.getFile();
+            (0, Affirm_1.default)(file, 'Invalid dataset file');
+            (0, Affirm_1.default)(file.fileKey, 'Invalid file key (endpoint path)');
+            const endpoint = file.fileKey;
+            const url = endpoint.startsWith('http') ? endpoint : `${this._baseUrl}${endpoint}`;
+            const data = yield this._makeRequest(url);
+            const apiObjects = this._extractObjectsFromResponse(data, file.httpApi);
+            dataset.setFirstLine(JSON.stringify(apiObjects[0]));
+            const totalLineCount = yield DriverHelper_1.default.appendObjectsToUnifiedFile({
+                append: true,
+                delimiter: dataset.getDelimiter(),
+                destinationPath: dataset.getPath(),
+                objects: apiObjects
+            });
+            dataset.setCount(totalLineCount);
+            return dataset;
+        });
+        this._extractObjectsFromResponse = (data, httpApi) => {
+            let itemsData = [];
+            if (httpApi && httpApi.dataProperty && httpApi.dataProperty.length > 0) {
+                itemsData = data[httpApi.dataProperty];
+            }
+            else {
+                if (typeof data === 'string') {
+                    itemsData = data.split('\n').filter(line => line.trim().length > 0);
+                }
+                else if (Array.isArray(data)) {
+                    itemsData = data;
+                }
+                else if (typeof data === 'object' && data !== null) {
+                    const dataObj = data;
+                    if (dataObj.data && Array.isArray(dataObj.data)) {
+                        itemsData = dataObj.data;
+                    }
+                    else if (dataObj.results && Array.isArray(dataObj.results)) {
+                        itemsData = dataObj.results;
+                    }
+                    else if (dataObj.items && Array.isArray(dataObj.items)) {
+                        itemsData = dataObj.items;
+                    }
+                    else {
+                        // Single object, return as single line
+                        itemsData = [data];
+                    }
+                }
+            }
+            return itemsData;
+        };
+    }
+}
+exports.HttpApiSourceDriver = HttpApiSourceDriver;
+exports.default = HttpApiSourceDriver;

package/drivers/LocalDriver.js

@@ -65,6 +65,7 @@ const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"))
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
 const Logger_1 = __importDefault(require("../helper/Logger"));
 const DriverHelper_1 = __importDefault(require("./DriverHelper"));
+const Constants_1 = __importDefault(require("../Constants"));
 class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -123,7 +124,8 @@ class LocalSourceDriver {
             (0, Affirm_1.default)(file, 'Invalid dataset file');
             (0, Affirm_1.default)(file.fileKey, 'Invalid file key');
             (0, Affirm_1.default)(file.fileType, `Invalid file type`);
-            const copyLocally = (fileKey_1, headerLine_1, ...args_1) => __awaiter(this, [fileKey_1, headerLine_1, ...args_1], void 0, function* (fileKey, headerLine, appendMode = false) {
+            const includeSourceFilename = file.includeSourceFilename === true;
+            const copyLocally = (fileKey_1, headerLine_1, ...args_1) => __awaiter(this, [fileKey_1, headerLine_1, ...args_1], void 0, function* (fileKey, headerLine, appendMode = false, sourceFilename) {
                 const sourceFilePath = path_1.default.join(this._path, fileKey);
                 (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
                 // Copy and validate header in a single stream pass
@@ -136,7 +138,8 @@ class LocalSourceDriver {
                     headerLine,
                     fileType: file.fileType,
                     hasHeaderRow: file.hasHeaderRow,
-                    delimiter: dataset.getDelimiter()
+                    delimiter: dataset.getDelimiter(),
+                    sourceFilename
                 });
             });
             const { fileKey } = file;
@@ -145,21 +148,32 @@ class LocalSourceDriver {
                 Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
                 Affirm_1.default.hasItems(allFileKeys, `The file key "${fileKey}" doesn't have any matches in path "${this._path}".`);
                 // Get header line from the first file
-                const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, allFileKeys[0]), 1))[0];
+                let headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, allFileKeys[0]), 1))[0];
+                // If including source filename, append a placeholder column name to the header
+                if (includeSourceFilename) {
+                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
                 dataset.setFirstLine(headerLine);
                 let totalLineCount = 0;
                 // Copy files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
-                    totalLineCount += yield copyLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
+                    const currentFileKey = allFileKeys[i];
+                    // Pass the filename (just the basename) if includeSourceFilename is enabled
+                    const sourceFilename = includeSourceFilename ? path_1.default.basename(currentFileKey) : undefined;
+                    totalLineCount += yield copyLocally(currentFileKey, headerLine, i > 0, sourceFilename); // Append mode for subsequent files
                 }
                 dataset.setCount(totalLineCount);
                 return dataset;
             }
             else {
-                // For single file, no header validation needed
-                const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, fileKey), 1))[0];
+                // For single file, include the filename if configured
+                let headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, fileKey), 1))[0];
+                if (includeSourceFilename) {
+                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
                 dataset.setFirstLine(headerLine);
-                const totalLineCount = yield copyLocally(fileKey, headerLine, false);
+                const sourceFilename = includeSourceFilename ? path_1.default.basename(fileKey) : undefined;
+                const totalLineCount = yield copyLocally(fileKey, headerLine, false, sourceFilename);
                 dataset.setCount(totalLineCount);
                 return dataset;
             }

package/drivers/S3Driver.js

@@ -24,6 +24,7 @@ const client_s3_1 = require("@aws-sdk/client-s3");
 const Affirm_1 = __importDefault(require("../core/Affirm"));
 const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
 const readline_1 = __importDefault(require("readline"));
+const path_1 = __importDefault(require("path"));
 const Algo_1 = __importDefault(require("../core/Algo"));
 const xlsx_1 = __importDefault(require("xlsx"));
 const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); // Added XMLParser import
@@ -32,6 +33,7 @@ const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"))
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
 const DriverHelper_1 = __importDefault(require("./DriverHelper"));
 const Logger_1 = __importDefault(require("../helper/Logger"));
+const Constants_1 = __importDefault(require("../Constants"));
 class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -213,7 +215,8 @@ class S3SourceDriver {
             (0, Affirm_1.default)(file, 'Invalid dataset file');
             (0, Affirm_1.default)(file.fileKey, 'Invalid file key');
             (0, Affirm_1.default)(file.fileType, `Invalid file type`);
-            const downloadLocally = (fileUrl_1, headerLine_1, ...args_1) => __awaiter(this, [fileUrl_1, headerLine_1, ...args_1], void 0, function* (fileUrl, headerLine, appendMode = false) {
+            const includeSourceFilename = file.includeSourceFilename === true;
+            const downloadLocally = (fileUrl_1, headerLine_1, ...args_1) => __awaiter(this, [fileUrl_1, headerLine_1, ...args_1], void 0, function* (fileUrl, headerLine, appendMode = false, sourceFilename) {
                 // Download and validate header in a single stream pass
                 const command = new client_s3_1.GetObjectCommand({
                     Bucket: this._bucketName,
@@ -230,7 +233,8 @@ class S3SourceDriver {
                     headerLine,
                     fileType: file.fileType,
                     hasHeaderRow: file.hasHeaderRow,
-                    delimiter: dataset.getDelimiter()
+                    delimiter: dataset.getDelimiter(),
+                    sourceFilename
                 });
             });
             const { fileKey } = file;
@@ -246,18 +250,25 @@ class S3SourceDriver {
                 const firstFileResponse = yield this._client.send(firstFileCommand);
                 (0, Affirm_1.default)(firstFileResponse.Body, 'Failed to fetch first file from S3');
                 const firstFileStream = firstFileResponse.Body;
-                const headerLine = yield this.getFirstLineFromStream(firstFileStream);
+                let headerLine = yield this.getFirstLineFromStream(firstFileStream);
+                // If including source filename, append a placeholder column name to the header
+                if (includeSourceFilename) {
+                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
                 dataset.setFirstLine(headerLine);
                 let totalLineCount = 0;
                 // Download files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
-                    totalLineCount += yield downloadLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
+                    const currentFileKey = allFileKeys[i];
+                    // Pass the filename (just the basename) if includeSourceFilename is enabled
+                    const sourceFilename = includeSourceFilename ? path_1.default.basename(currentFileKey) : undefined;
+                    totalLineCount += yield downloadLocally(currentFileKey, headerLine, i > 0, sourceFilename); // Append mode for subsequent files
                 }
                 dataset.setCount(totalLineCount);
                 return dataset;
             }
             else {
-                // Get header line from the first file
+                // Get header line from the single file
                 const firstFileCommand = new client_s3_1.GetObjectCommand({
                     Bucket: this._bucketName,
                     Key: fileKey
@@ -265,10 +276,15 @@ class S3SourceDriver {
                 const firstFileResponse = yield this._client.send(firstFileCommand);
                 (0, Affirm_1.default)(firstFileResponse.Body, 'Failed to fetch first file from S3');
                 const firstFileStream = firstFileResponse.Body;
-                const headerLine = yield this.getFirstLineFromStream(firstFileStream);
+                let headerLine = yield this.getFirstLineFromStream(firstFileStream);
+                // If including source filename, append a placeholder column name to the header
+                if (includeSourceFilename) {
+                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
                 dataset.setFirstLine(headerLine);
-                // For single file, no header validation needed
-                const totalLineCount = yield downloadLocally(fileKey, headerLine);
+                // Pass the filename if includeSourceFilename is enabled
+                const sourceFilename = includeSourceFilename ? path_1.default.basename(fileKey) : undefined;
+                const totalLineCount = yield downloadLocally(fileKey, headerLine, false, sourceFilename);
                 dataset.setCount(totalLineCount);
                 return dataset;
             }

package/engines/UserManager.js

@@ -24,6 +24,18 @@ class UserManagerClass {
                 return MOCK_USER;
             // TODO: figure out how to handle users
         };
+        this.getRemoraWorkerUser = () => {
+            const remora = {
+                _id: '__remora_worker__',
+                auth: { oid: '', provider: 'internal' },
+                email: '',
+                name: 'Remora Worker',
+                roles: ['root'],
+                _signature: '',
+                lastLogin: new Date().toJSON()
+            };
+            return remora;
+        };
         this.findOIDC = (oid) => __awaiter(this, void 0, void 0, function* () {
             return yield DatabaseEngine_1.default.findOne(Settings_1.default.db.collections.users, { 'auth.oid': oid });
         });

package/engines/ai/LLM.js

@@ -164,26 +164,6 @@ resulting consumer: """
 }
 """
 `;
-const baseQASystemPrompt = `
-# TASK
-You are an agent tasked with ensuring that the CONSUMER(S) created follow the guidelines given.
-You are going to receive a list of CONSUMERS and you need to return in the correct JSON format the same CONSUMERS with the needed updates to ensure that they follow all the rules.
-
-# CONSUMER DEFINITION
-A consumer takes the data from one or more producers and changes it's shape to transform it into the required output schema.
-## FIELDS
-- fields.from: used to distinct between the producers imported by the consumer. The value is the name of the producer.
-
-# RULES
-- If a field is not needed, do not add it e.g.
-- Only import a producer once
-- Awlays include this exact property as the first -> "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
-- Use "API" as the only valid output format.
-- The "from" must contain only the name of the producer
-
-# CONSUMERS
-{{consumers}}
-`;
 class LLM {
     constructor() {
         this.inferProducers = (input, outputs, fileName, sources) => __awaiter(this, void 0, void 0, function* () {
@@ -192,7 +172,7 @@ class LLM {
             systemPrompt = systemPrompt.replace('{{output data spec}}', outputs.map(x => `- ${JSON.stringify(x)}`).join('\n'));
             systemPrompt = systemPrompt.replace('{{file name}}', fileName);
             systemPrompt = systemPrompt.replace('{{sources}}', sources.map(x => `- ${JSON.stringify(x)}`).join('\n'));
-            const res = yield this._client.beta.chat.completions.parse({
+            const res = yield this._client.chat.completions.create({
                 model: 'gpt-4o',
                 messages: [
                     { role: 'system', content: systemPrompt }
@@ -219,7 +199,7 @@ class LLM {
                 }), 'environment')
             });
             const msg = res.choices[0].message;
-            return msg.parsed;
+            return JSON.parse(msg.content);
         });
         this.inferConsumers = (producers, outputs) => __awaiter(this, void 0, void 0, function* () {
             let systemPrompt = baseConsumersSystemPrompt;
@@ -261,9 +241,9 @@ class LLM {
                     }))
                 }), 'environment')
             };
-            const res = yield this._client.beta.chat.completions.parse(item);
+            const res = yield this._client.chat.completions.create(item);
             const msg = res.choices[0].message;
-            const finalDraft = msg.parsed;
+            const finalDraft = JSON.parse(msg.content);
             // Do some manual adjustments cause some things still don't work...
             if (finalDraft && finalDraft.consumers) {
                 for (const cons of finalDraft.consumers) {

package/engines/consumer/ConsumerEngine.js

@@ -127,10 +127,10 @@ class ConsumerEngineClass {
                 }
             }
         });
-        this.execute = (consumer, options, user) => __awaiter(this, void 0, void 0, function* () {
+        this.execute = (consumer, options, user, details) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(consumer, `Invalid consumer`);
             (0, Affirm_1.default)(options, `Invalid execute consume options`);
-            const { usageId } = UsageManager_1.default.startUsage(consumer, user);
+            const { usageId } = UsageManager_1.default.startUsage(consumer, user, details);
             try {
                 const execution = new ExecutionEnvironment_1.default(consumer, usageId);
                 const result = yield execution.run(options);

package/engines/dataset/Dataset.js

@@ -730,7 +730,7 @@ class Dataset {
             console.log(`DS ${this.name} (${this._count} | ${this._iterations})`);
             console.log(this._dimensions.map(x => x.name).join(this._delimiter));
             const records = yield this.readLines(count);
-            records.forEach(x => console.log(full ? x : x.stringify()));
+            records.forEach((x, i) => console.log(`[${i}]`, full ? x : x.stringify()));
             console.log('----------');
         });
         this.printStats = () => {