@forzalabs/remora 0.0.19 → 0.0.21

package/drivers/LocalDriver.js

@@ -52,6 +52,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalDriverDestination = void 0;
 const fs = __importStar(require("fs"));
 const path_1 = __importDefault(require("path"));
 const readline_1 = __importDefault(require("readline"));
@@ -60,9 +61,9 @@ class LocalDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
             const fileURL = source.authentication['path'];
-            (0, Affirm_1.default)(fileURL, 'file path is not entered in configuration');
+            (0, Affirm_1.default)(fileURL, `Missing file path in the authentication of source "${source.name}"`);
             const exist = fs.existsSync(fileURL);
-            (0, Affirm_1.default)(exist, 'The path entered is incorrect: ' + fileURL);
+            (0, Affirm_1.default)(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
             this._path = source.authentication['path'];
             return this;
         });
@@ -119,4 +120,71 @@ class LocalDriver {
         });
     }
 }
+class LocalDriverDestination {
+    constructor() {
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(source, `Invalid source`);
+            const fileURL = source.authentication['path'];
+            (0, Affirm_1.default)(fileURL, `Missing file path in the authentication of source "${source.name}"`);
+            const exist = fs.existsSync(fileURL);
+            (0, Affirm_1.default)(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
+            this._path = source.authentication['path'];
+            return this;
+        });
+        this.uploadFile = (options) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(options, 'Invalid upload options');
+            (0, Affirm_1.default)(options.name, 'File name is required');
+            (0, Affirm_1.default)(options.content != null, 'File content is required');
+            const folder = this._path;
+            try {
+                if (!fs.existsSync(folder))
+                    fs.mkdirSync(folder, { recursive: true });
+                const filePath = path_1.default.join(folder, options.name);
+                fs.writeFileSync(filePath, options.content);
+                return { bucket: folder, key: filePath, res: true };
+            }
+            catch (error) {
+                throw new Error(`Failed to upload local file "${options.name}": ${error.message}`);
+            }
+        });
+        this.multipartUpload = (options) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(options, 'Invalid upload options');
+            (0, Affirm_1.default)(options.name, 'File name is required');
+            (0, Affirm_1.default)(options.contents && Array.isArray(options.contents), 'Contents must be an array');
+            (0, Affirm_1.default)(options.contents.length > 0, 'Contents array cannot be empty');
+            const folder = this._path;
+            try {
+                if (!fs.existsSync(folder)) {
+                    fs.mkdirSync(folder, { recursive: true });
+                }
+                const filePath = path_1.default.join(folder, options.name);
+                // Create or truncate the file first
+                fs.writeFileSync(filePath, '');
+                // Append each chunk
+                for (const chunk of options.contents) {
+                    (0, Affirm_1.default)(typeof chunk === 'string', 'Each chunk must be a string');
+                    fs.appendFileSync(filePath, chunk);
+                }
+                return { bucket: folder, key: filePath, res: true };
+            }
+            catch (error) {
+                // Clean up the partial file if it exists
+                const filePath = path_1.default.join(folder, options.name);
+                if (fs.existsSync(filePath)) {
+                    try {
+                        fs.unlinkSync(filePath);
+                    }
+                    catch (cleanupError) {
+                        console.error(`Failed to clean up partial file after error: ${cleanupError.message}`);
+                        throw cleanupError;
+                    }
+                }
+                throw new Error(`Failed to complete local multipart upload for "${options.name}": ${error.message}`);
+            }
+        });
+    }
+}
+exports.LocalDriverDestination = LocalDriverDestination;
 exports.default = LocalDriver;

package/drivers/S3Driver.js

@@ -45,6 +45,58 @@ class S3Driver {
             (0, Affirm_1.default)(res.$metadata.httpStatusCode === 200, `Failed to upload the file "${name}" to the bucket "${this._bucketName}": status code ${res.$metadata.httpStatusCode}`);
             return { res: true, key: name, bucket: this._bucketName };
         });
+        this.multipartUpload = (options) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(options, `Invalid upload options`);
+            (0, Affirm_1.default)(options.contents && options.contents.length > 0, 'No contents provided for multipart upload');
+            (0, Affirm_1.default)(options.name, 'No filename provided for multipart upload');
+            try {
+                // Create the multipart upload
+                const createMultipartUploadRes = yield this._client.send(new client_s3_1.CreateMultipartUploadCommand({
+                    Bucket: this._bucketName,
+                    Key: options.name
+                }));
+                const uploadId = createMultipartUploadRes.UploadId;
+                (0, Affirm_1.default)(uploadId, 'Failed to initiate multipart upload');
+                // Upload each part
+                const uploadPromises = options.contents.map((content, index) => __awaiter(this, void 0, void 0, function* () {
+                    const partNumber = index + 1;
+                    const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
+                        Bucket: this._bucketName,
+                        Key: options.name,
+                        UploadId: uploadId,
+                        PartNumber: partNumber,
+                        Body: Buffer.from(content)
+                    }));
+                    return {
+                        PartNumber: partNumber,
+                        ETag: uploadPartRes.ETag
+                    };
+                }));
+                const uploadedParts = yield Promise.all(uploadPromises);
+                // Complete the multipart upload
+                const completeRes = yield this._client.send(new client_s3_1.CompleteMultipartUploadCommand({
+                    Bucket: this._bucketName,
+                    Key: options.name,
+                    UploadId: uploadId,
+                    MultipartUpload: {
+                        Parts: uploadedParts
+                    }
+                }));
+                (0, Affirm_1.default)(completeRes.$metadata.httpStatusCode === 200, `Failed to complete multipart upload for "${options.name}": status code ${completeRes.$metadata.httpStatusCode}`);
+                return { res: true, key: options.name, bucket: this._bucketName };
+            }
+            catch (error) {
+                // If anything fails, make sure to abort the multipart upload
+                if (error.UploadId) {
+                    yield this._client.send(new client_s3_1.AbortMultipartUploadCommand({
+                        Bucket: this._bucketName,
+                        Key: options.name,
+                        UploadId: error.UploadId
+                    }));
+                }
+                throw error;
+            }
+        });
     }
 }
 exports.default = S3Driver;

package/engines/DataframeManager.js

@@ -0,0 +1,55 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Algo_1 = __importDefault(require("../core/Algo"));
+const Helper_1 = __importDefault(require("../helper/Helper"));
+class DataframeManagerClass {
+    fill(points, from, to, onlyLastValue, maintainLastValue) {
+        const min = from !== null && from !== void 0 ? from : this.getMinDate(points);
+        const max = to !== null && to !== void 0 ? to : this.getMaxDate(points);
+        const orderPoints = points.length > 0 ? Algo_1.default.orderBy(points, 'x') : [];
+        const filledPoints = [];
+        const currentDate = new Date(min);
+        while (currentDate <= max) {
+            const monthKey = Helper_1.default.formatDateToYYYYMM(currentDate);
+            filledPoints.push({ x: monthKey, y: 0 });
+            currentDate.setMonth(currentDate.getMonth() + 1);
+        }
+        for (let i = 0; i < orderPoints.length; i++) {
+            const point = orderPoints[i];
+            const date = new Date(point.x);
+            const filledPoint = filledPoints.find(x => x.x === Helper_1.default.formatDateToYYYYMM(date));
+            if (filledPoint) {
+                if (!onlyLastValue)
+                    filledPoint.y += point.y;
+                else
+                    filledPoint.y = point.y;
+                if (maintainLastValue) {
+                    const index = filledPoints.findIndex(x => x.x === Helper_1.default.formatDateToYYYYMM(date));
+                    for (let k = index; k < filledPoints.length; k++) {
+                        const nextFilledPoint = filledPoints[k];
+                        nextFilledPoint.y = filledPoint.y;
+                    }
+                }
+            }
+        }
+        return filledPoints;
+    }
+    getMinDate(points) {
+        if (!points || points.length === 0) {
+            const currentDate = new Date();
+            return new Date(currentDate.getFullYear() - 1, currentDate.getMonth(), currentDate.getDate());
+        }
+        return points.reduce((min, point) => (new Date(point.x) < min ? new Date(point === null || point === void 0 ? void 0 : point.x) : min), new Date(points[0].x));
+    }
+    getMaxDate(points) {
+        if (!points || points.length === 0) {
+            return new Date();
+        }
+        return points.reduce((max, point) => (new Date(point.x) > max ? new Date(point.x) : max), new Date(points[0].x));
+    }
+}
+const DataframeManager = new DataframeManagerClass();
+exports.default = DataframeManager;

package/engines/Environment.js

@@ -7,7 +7,8 @@ const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const Affirm_1 = __importDefault(require("../core/Affirm"));
 const SchemaValidator_1 = __importDefault(require("./schema/SchemaValidator"));
-const Validator_1 = __importDefault(require("./Validator"));
+const Validator_1 = __importDefault(require("./validation/Validator"));
+const Constants_1 = __importDefault(require("../Constants"));
 class EnvironmentClass {
     constructor() {
         this._env = null;
@@ -71,9 +72,16 @@ class EnvironmentClass {
                 if (!SchemaValidator_1.default.validate('consumer-schema', consumer))
                     throw new Error(`Invalid consumer configuration: ${consumer.name}`);
             });
+            // Load the project settings
+            const envSettings = new Map(Object.entries(Object.assign({}, projectConfig.settings))
+                .map(([key, value]) => [key, String(value)]));
+            if (!envSettings.has('SQL_MAX_QUERY_ROWS'))
+                envSettings.set('SQL_MAX_QUERY_ROWS', Constants_1.default.defaults.SQL_MAX_QUERY_ROWS.toString());
+            if (!envSettings.has('STRING_MAX_CHARACTERS_LENGTH'))
+                envSettings.set('STRING_MAX_CHARACTERS_LENGTH', Constants_1.default.defaults.STRING_MAX_CHARACTERS_LENGTH.toString());
             // Initialize environment
             this.init({
-                settings: new Map(Object.entries(Object.assign({}, projectConfig.settings)).map(([key, value]) => [key, String(value)])),
+                settings: envSettings,
                 sources,
                 producers,
                 consumers,
@@ -93,7 +101,7 @@ class EnvironmentClass {
          */
         this.getFirstProducer = (producerName) => {
             (0, Affirm_1.default)(producerName, `Invalid producer name`);
-            const prod = this._env.producers.find(x => x.name === producerName);
+            const prod = this._env.producers.find(x => x.name.toLocaleLowerCase() === producerName.toLowerCase());
             if (!prod) {
                 const consumer = this.getConsumer(producerName);
                 (0, Affirm_1.default)(consumer, `Invalid producer name`);
@@ -103,11 +111,11 @@ class EnvironmentClass {
         };
         this.getProducer = (producerName) => {
             (0, Affirm_1.default)(producerName, `Invalid producer name`);
-            return this._env.producers.find(x => x.name === producerName);
+            return this._env.producers.find(x => x.name.toLowerCase() === producerName.toLowerCase());
         };
         this.getConsumer = (consumerName) => {
             (0, Affirm_1.default)(consumerName, `Invalid consumer name`);
-            return this._env.consumers.find(x => x.name === consumerName);
+            return this._env.consumers.find(x => x.name.toLowerCase() === consumerName.toLowerCase());
         };
         this.getSchema = (schemaName) => {
             (0, Affirm_1.default)(schemaName, 'Invalid schema name');

package/engines/ProducerEngine.js

@@ -82,6 +82,7 @@ class ProducerEngineClass {
             }
         });
         this.readFile = (producer, options) => __awaiter(this, void 0, void 0, function* () {
+            var _a;
             (0, Affirm_1.default)(producer, 'Invalid producer');
             (0, Affirm_1.default)(options, 'Invalid options');
             if (options.readmode === 'lines')
@@ -95,7 +96,7 @@ class ProducerEngineClass {
                 lines = yield driver.readLinesInRange({ fileKey: producer.settings.fileKey, lineFrom: options.lines.from, lineTo: options.lines.to });
             else
                 lines = [(yield driver.download({ fileKey: producer.settings.fileKey }))];
-            switch (producer.settings.fileType.toUpperCase()) {
+            switch ((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) {
                 case 'CSV': {
                     return { data: lines, dataType: 'lines-of-text' };
                 }

package/engines/UsageDataManager.js

@@ -0,0 +1,110 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const DSTE_1 = __importDefault(require("../core/dste/DSTE"));
+const DatabaseEngine_1 = __importDefault(require("../database/DatabaseEngine"));
+const DataframeManager_1 = __importDefault(require("./DataframeManager"));
+class UsageDataManager {
+    getUsageDetails() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const now = DSTE_1.default.now();
+            const from = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
+            const prevMonthFrom = new Date(now.getTime() - 60 * 24 * 60 * 60 * 1000);
+            const yearAgo = new Date(now.getFullYear(), now.getMonth() - 11, 1);
+            const collection = 'usage';
+            // Aggregate status counts for current and previous month
+            const getStatusCounts = (start, end) => __awaiter(this, void 0, void 0, function* () {
+                const results = yield DatabaseEngine_1.default.aggregate(collection, [
+                    { $match: { startedAt: { $gte: start, $lte: end } } },
+                    { $group: { _id: '$status', count: { $sum: 1 } } }
+                ]);
+                let success = 0, failed = 0, total = 0;
+                results.forEach((r) => {
+                    total += r.count;
+                    if (r._id === 'success')
+                        success = r.count;
+                    if (r._id === 'failed')
+                        failed = r.count;
+                });
+                return { total, success, failed };
+            });
+            const statusesRequests = yield getStatusCounts(from, now);
+            const prevStatusesRequests = yield getStatusCounts(prevMonthFrom, from);
+            // Monthly success and fails for last 12 months
+            const monthlySuccessPipeline = [
+                { $match: { status: 'success', startedAt: { $gte: yearAgo, $lte: now } } },
+                { $addFields: { year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
+                { $group: { _id: { year: '$year', month: '$month' }, count: { $sum: 1 } } },
+                { $project: { _id: 0, x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$count' } },
+                { $sort: { x: 1 } }
+            ];
+            const monthlyFailsPipeline = [
+                { $match: { status: 'failed', startedAt: { $gte: yearAgo, $lte: now } } },
+                { $addFields: { year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
+                { $group: { _id: { year: '$year', month: '$month' }, count: { $sum: 1 } } },
+                { $project: { _id: 0, x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$count' } },
+                { $sort: { x: 1 } }
+            ];
+            const rawMonthlySuccess = yield DatabaseEngine_1.default.aggregate(collection, monthlySuccessPipeline);
+            const rawMonthlyFails = yield DatabaseEngine_1.default.aggregate(collection, monthlyFailsPipeline);
+            // Top lines per month for last 12 months
+            const topLinesPipeline = [
+                { $match: { startedAt: { $gte: yearAgo, $lte: now } } },
+                { $addFields: { year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
+                { $group: { _id: { year: '$year', month: '$month' }, itemsCount: { $max: '$itemsCount' } } },
+                { $project: { _id: 0, x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$itemsCount' } },
+                { $sort: { x: 1 } }
+            ];
+            const topLines = yield DatabaseEngine_1.default.aggregate(collection, topLinesPipeline);
+            // Top times per month for last 12 months
+            const topTimePipeline = [
+                { $match: { startedAt: { $gte: yearAgo, $lte: now } } },
+                { $addFields: { durationMs: { $subtract: ['$finishedAt', '$startedAt'] }, year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
+                { $group: { _id: { year: '$year', month: '$month' }, maxDuration: { $max: '$durationMs' } } },
+                { $project: { _id: 0, x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$maxDuration' } },
+                { $sort: { x: 1 } }
+            ];
+            const topTime = yield DatabaseEngine_1.default.aggregate(collection, topTimePipeline);
+            // Monthly consumers: for each consumer, per month count
+            const consumerPipeline = [
+                { $match: { startedAt: { $gte: yearAgo, $lte: now } } },
+                { $addFields: { year: { $year: '$startedAt' }, month: { $month: '$startedAt' } } },
+                { $group: { _id: { consumer: '$consumer', year: '$year', month: '$month' }, count: { $sum: 1 } } },
+                { $project: { _id: 0, consumer: '$_id.consumer', x: { $concat: [{ $toString: '$_id.year' }, '-', { $toString: '$_id.month' }] }, y: '$count' } },
+                { $sort: { consumer: 1, x: 1 } }
+            ];
+            const consumersData = yield DatabaseEngine_1.default.aggregate(collection, consumerPipeline);
+            // transform to consumer array
+            const consumerMap = {};
+            consumersData.forEach((r) => {
+                consumerMap[r.consumer] = consumerMap[r.consumer] || [];
+                consumerMap[r.consumer].push({ x: r.x, y: r.y });
+            });
+            const consumers = Object.entries(consumerMap).map(([name, data]) => ({ name, data: DataframeManager_1.default.fill(data !== null && data !== void 0 ? data : [], yearAgo, now) }));
+            // Recent executions
+            const recentExecution = yield DatabaseEngine_1.default.query(collection, { startedAt: { $gte: from, $lte: now } }, { sort: { startedAt: -1 }, limit: 10 });
+            return {
+                statusesRequests,
+                prevStatusesRequests,
+                monthlySuccess: DataframeManager_1.default.fill(rawMonthlySuccess !== null && rawMonthlySuccess !== void 0 ? rawMonthlySuccess : [], yearAgo, now),
+                monthlyFails: DataframeManager_1.default.fill(rawMonthlyFails !== null && rawMonthlyFails !== void 0 ? rawMonthlyFails : [], yearAgo, now),
+                consumers: consumers,
+                topLine: DataframeManager_1.default.fill(topLines !== null && topLines !== void 0 ? topLines : [], yearAgo, now),
+                topTime: DataframeManager_1.default.fill(topTime !== null && topTime !== void 0 ? topTime : [], yearAgo, now),
+                recentExecution
+            };
+        });
+    }
+}
+exports.default = new UsageDataManager();

package/engines/UserManager.js

@@ -46,8 +46,8 @@ const DEV_USER = {
 const MOCK_USER = {
     _id: '__mock__',
     auth: { oid: '', provider: 'azure' },
-    email: '',
-    name: 'mock',
+    email: 'mock.user@email.com',
+    name: 'Mock User',
     roles: ['user'],
     _signature: '',
     lastLogin: new Date().toJSON()

package/engines/Validator.js

@@ -120,7 +120,7 @@ class ValidatorClass {
                     const groupingFields = fields.filter(x => x.grouping);
                     if (groupingFields.length > 1)
                         errors.push(`There can't be 2 fields with grouping defined at the same level (${groupingFields.map(x => x.key).join(', ')}). Level: ${level}`);
-                    groupingFields.forEach(field => {
+                    groupingFields.forEach((field) => {
                         if (field.grouping)
                             errors = [...errors, ...validateGroupingLevels(field.grouping.subFields, level + 1)];
                     });

package/engines/ai/AutoMapperEngine.js

@@ -20,11 +20,11 @@ class AutoMapperEngineClass {
          * input: the first ten lines of the uploaded file
          * outputs: the selected schemas
          */
-        this.map = (input, outputs) => __awaiter(this, void 0, void 0, function* () {
+        this.map = (input, outputs, fileName, sources) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(input, 'Invalid input');
             (0, Affirm_1.default)(outputs, 'Invalid outputs');
             const llm = new LLM_1.default();
-            const producersRes = yield llm.inferProducers(input, outputs);
+            const producersRes = yield llm.inferProducers(input, outputs, fileName, sources);
             const consumersRes = yield llm.inferConsumers(producersRes.producers, outputs);
             return {
                 consumers: consumersRes.consumers,

package/engines/ai/LLM.js

@@ -25,12 +25,16 @@ You are tasked with creating the PRODUCER(S) that will then be used.
 A producer maps directly to a dataset and exposes it's dimensions.
 
 ## FIELDS
-- alias: the reference to the column or property name if different from the desired name property
 - classification: make your best guess if the field falls under any of these regulations
 
 # RULES
 - Add only the required fields to comply with the OUTPUT DATA SPEC
 - Add fields that you think are important
+- The name of the producer must be the same as the name of the dataset.
+- Avoid creating multiple providers with similar data.
+- Try to create the least number of providers
+- Awlays include this exact property as the first -> "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/producer-schema.json"
+- Based on the producer select the source that makes the most sense to connect otherwise leave the string "<source_name>"
 
 # FORMAT
 The INPUT DATA SPEC will be the first 10 rows from the dataset that the producer needs to map to.
@@ -42,6 +46,12 @@ The result must be returned as a JSON object.
 
 # OUTPUT DATA SPEC
 {{output data spec}}
+
+# File name
+{{file name}}
+
+# SOURCES
+{{sources}}
 `;
 const baseConsumersSystemPrompt = `
 # TASK
@@ -52,12 +62,14 @@ You are going to receive a list of PRODUCERS that expose some dimensions, and yo
 A consumer takes the data from one or more producers and changes it's shape to transform it into the required output schema.
 
 ## FIELDS
-- fields.grouping: used when you need to create a nested data structure with many items under a single key. If no groupingKey is used, then don't add this field.
 - fields.from: used to distinct between the producers imported by the consumer. The value is the name of the producer.
 
 # RULES
-- If a field is not needed, do not add it e.g. if there is no need for the "grouping" property, then don't add it.
+- If a field is not needed, do not add it e.g.
 - Only import a producer once
+- Awlays include this exact property as the first -> $schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
+- Use "API" as the only valid output format.
+- The "from" must contain only the name of the producer
 
 # FORMAT
 The INPUT DATA SPEC will be the first 10 rows from the dataset that the producer needs to map to.
@@ -141,7 +153,7 @@ resulting consumer: """
     ],
     "outputs": [
         {
-            "format": "JSON"
+            "format": "API"
         }
     ],
     "producers": [
@@ -159,36 +171,45 @@ You are going to receive a list of CONSUMERS and you need to return in the corre
 
 # CONSUMER DEFINITION
 A consumer takes the data from one or more producers and changes it's shape to transform it into the required output schema.
-
 ## FIELDS
-- fields.grouping: used when you need to create a nested data structure with many items under a single key. If no groupingKey is used, then don't add this field.
 - fields.from: used to distinct between the producers imported by the consumer. The value is the name of the producer.
 
 # RULES
-- If a field is not needed, do not add it e.g. if there is no need for the "grouping" property, then don't add it.
+- If a field is not needed, do not add it e.g.
 - Only import a producer once
+- Awlays include this exact property as the first -> "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
+- Use "API" as the only valid output format.
+- The "from" must contain only the name of the producer
 
 # CONSUMERS
 {{consumers}}
 `;
 class LLM {
     constructor() {
-        this.inferProducers = (input, outputs) => __awaiter(this, void 0, void 0, function* () {
+        this.inferProducers = (input, outputs, fileName, sources) => __awaiter(this, void 0, void 0, function* () {
             let systemPrompt = baseProducersSystemPrompt;
             systemPrompt = systemPrompt.replace('{{input data spec}}', input.map(x => `- ${JSON.stringify(x)}`).join('\n'));
             systemPrompt = systemPrompt.replace('{{output data spec}}', outputs.map(x => `- ${JSON.stringify(x)}`).join('\n'));
+            systemPrompt = systemPrompt.replace('{{file name}}', fileName);
+            systemPrompt = systemPrompt.replace('{{sources}}', sources.map(x => `- ${JSON.stringify(x)}`).join('\n'));
             const res = yield this._client.beta.chat.completions.parse({
-                model: 'o3-mini',
+                model: 'gpt-4o',
                 messages: [
                     { role: 'system', content: systemPrompt }
                 ],
                 response_format: (0, zod_1.zodResponseFormat)(zod_2.z.object({
                     producers: zod_2.z.array(zod_2.z.object({
+                        $schema: zod_2.z.string().describe('The schema of the producer. This should always be the same.'),
                         name: zod_2.z.string(),
                         description: zod_2.z.string(),
                         dimensions: zod_2.z.array(zod_2.z.object({
                             name: zod_2.z.string(),
-                            alias: zod_2.z.string().optional(),
+                            // alias: z.string().optional(),
+                            source: zod_2.z.string().describe('The name of the source linked to this producer.'),
+                            settings: zod_2.z.object({
+                                fileKey: zod_2.z.string().describe('The name of the file'),
+                                fileType: zod_2.z.string().describe('The file extension (CSV | JSONL | JSON)')
+                            }),
                             description: zod_2.z.string().optional(),
                             type: zod_2.z.enum(['string', 'number', 'datetime']),
                             pk: zod_2.z.boolean().optional(),
@@ -205,12 +226,13 @@ class LLM {
             systemPrompt = systemPrompt.replace('{{producers}}', producers.map(x => `- ${JSON.stringify(x)}`).join('\n'));
             systemPrompt = systemPrompt.replace('{{output data spec}}', outputs.map(x => `- ${JSON.stringify(x)}`).join('\n'));
             const item = {
-                model: 'o3-mini',
+                model: 'gpt-4o',
                 messages: [
                     { role: 'system', content: systemPrompt }
                 ],
                 response_format: (0, zod_1.zodResponseFormat)(zod_2.z.object({
                     consumers: zod_2.z.array(zod_2.z.object({
+                        $schema: zod_2.z.string().describe('The schema of the consumer. This should always be the same.'),
                         name: zod_2.z.string(),
                         description: zod_2.z.string(),
                         producers: zod_2.z.array(zod_2.z.object({
@@ -224,13 +246,14 @@ class LLM {
                         fields: zod_2.z.array(zod_2.z.object({
                             key: zod_2.z.string(),
                             from: zod_2.z.string().optional(),
-                            grouping: zod_2.z.object({
-                                groupingKey: zod_2.z.string(),
-                                subFields: zod_2.z.array(zod_2.z.lazy(() => zod_2.z.object({
-                                    key: zod_2.z.string(),
-                                    from: zod_2.z.string().optional()
-                                })))
-                            }).optional()
+                            alias: zod_2.z.string().optional()
+                            // grouping: z.object({
+                            //     groupingKey: z.string(),
+                            //     subFields: z.array(z.lazy(() => z.object({
+                            //         key: z.string(),
+                            //         from: z.string().optional()
+                            //     })))
+                            // }).optional()
                         })),
                         outputs: zod_2.z.array(zod_2.z.object({
                             format: zod_2.z.enum(['SQL', 'API', 'CSV', 'PARQUET', 'JSON'])
@@ -243,12 +266,13 @@ class LLM {
             const firstDraft = msg.parsed;
             const qaSystemPrompt = baseQASystemPrompt.replace('{{consumers}}', firstDraft.consumers.map(x => `- ${JSON.stringify(x)}`).join('\n'));
             const res2 = yield this._client.beta.chat.completions.parse({
-                model: 'o3-mini',
+                model: 'gpt-4o',
                 messages: [
                     { role: 'system', content: qaSystemPrompt }
                 ],
                 response_format: (0, zod_1.zodResponseFormat)(zod_2.z.object({
                     consumers: zod_2.z.array(zod_2.z.object({
+                        $schema: zod_2.z.string().describe('The schema of the consumer. This should always be the same.'),
                         name: zod_2.z.string(),
                         description: zod_2.z.string(),
                         producers: zod_2.z.array(zod_2.z.object({
@@ -262,13 +286,14 @@ class LLM {
                         fields: zod_2.z.array(zod_2.z.object({
                             key: zod_2.z.string(),
                             from: zod_2.z.string().optional(),
-                            grouping: zod_2.z.object({
-                                groupingKey: zod_2.z.string().optional(),
-                                subFields: zod_2.z.array(zod_2.z.lazy(() => zod_2.z.object({
-                                    key: zod_2.z.string(),
-                                    from: zod_2.z.string().optional()
-                                }))).optional()
-                            }).optional()
+                            alias: zod_2.z.string().optional()
+                            // grouping: z.object({
+                            //     groupingKey: z.string().optional(),
+                            //     subFields: z.array(z.lazy(() => z.object({
+                            //         key: z.string(),
+                            //         from: z.string().optional()
+                            //     }))).optional()
+                            // }).optional()
                         })),
                         outputs: zod_2.z.array(zod_2.z.object({
                             format: zod_2.z.enum(['SQL', 'API', 'CSV', 'PARQUET', 'JSON'])

package/engines/consumer/ConsumerManager.js

@@ -113,7 +113,8 @@ class ConsumerManagerClass {
             else {
                 const col = ConsumerManager.searchFieldInColumns(field.cField, availableColumns, consumer);
                 (0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.cField.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
-                expandedFields.push(field);
+                // TODO: CHECK THIS FIX IS GOOD
+                expandedFields.push(Object.assign(Object.assign({}, field), { dimension: col.dimension, measure: col.measure }));
             }
             return expandedFields;
         };