@forzalabs/remora 0.0.12 → 0.0.13

package/documentation/default_resources/consumer.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
   "name": "<consumer name>",
   "description": "<consumer description>",
   "producers": [

package/documentation/default_resources/producer.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/producer-schema.json",
   "name": "<producer name>",
   "description": "<producer description>",
   "source": "<source name>",

package/documentation/default_resources/project.json

@@ -1,4 +1,5 @@
 {
+    "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/project-schema.json",
     "name": "PROJECT_NAME_EXAMPLE",
     "version": "1.0.0",
     "description": "DESCRIPTION_OF_THIS_PROJECT",

package/documentation/default_resources/schema.json

@@ -1,7 +1,7 @@
 {
-  "$schema": "<http://json-schema.org/draft-07/schema#>",
-  "title": "<schema title>",
-  "type": "<object>",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "<example schema>",
+  "type": "object",
   "version": 1,
   "description": "<schema description>",
   "properties": {

package/documentation/default_resources/source.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/source-schema.json",
   "name": "<source name>",
   "description": "<source description>",
   "_version": 1,

package/drivers/RedshiftDriver.js

@@ -16,6 +16,7 @@ const client_redshift_data_1 = require("@aws-sdk/client-redshift-data");
 const Affirm_1 = __importDefault(require("../core/Affirm"));
 const Algo_1 = __importDefault(require("../core/Algo"));
 const Environment_1 = __importDefault(require("../engines/Environment"));
+const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
 class RedshiftDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -23,12 +24,13 @@ class RedshiftDriver {
             this._SQL_MAX_QUERY_ROWS = parseInt(Environment_1.default.get('SQL_MAX_QUERY_ROWS'));
             this._dbName = source.authentication['database'];
             this._workgroup = source.authentication['workgroup'];
+            const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
             const config = {
                 region: source.authentication['region'],
                 credentials: {
-                    accessKeyId: source.authentication['accessKey'],
-                    secretAccessKey: source.authentication['secretKey']
-                    // sessionToken: 'IQoJb3JpZ2luX2VjEDQaCXVzLWVhc3QtMSJIMEYCIQDXHjVYaGyCWCMgYWmnYt652z02H+w0jia8l/ckgu+gSAIhAOaKqkWbitaywpIy4l3lXQfQqW/fYhd187jRsL0kkm+wKokDCCwQABoMNzMwMzM1MTgwOTE5Igz36Yz4UuTQ6KuP5lcq5gKi1zczRVbAI6fnKQGNqg2k18/wJFpQgINNW+Ae9BoFofarAlOkXO5qeloP/5dGMufPVu/hy4tqaDi0Jpz8Zpa1JJjj6Q4l7XZi0+SpCkWwoVRsM/8ojMzowTWnUzan17I0jnG4TQKVEABQjQEACbJMbPwHNm1cN+dEY14cTyA36yJ0ma+8GDLECYnBM4klHKxzYJbD2tdwSgW4KAZ2ttk9+AnrKJUVH4hDI4cPBRv8uIUKGGfzD8cRpMGCByYeLVPc0o5IFCZGqkHncRlAU7dH0xZyGyCeDo3Z3819Ll8vJ41RN4RQOwcWrmL1Ne73KRgH32Ya9xH5j052WxERJPouCk+Lj307XmL0YwpkiQtV7cK5bQQRHtihviGkiK4maZLXXy7aUPQQG57xc0Q3HJXdI6LiXhvKvPKQm2iwAQS+FUZeekf1msT6MkVWd1ibYyhMzFeHBBPvVDlc4NFtLH6rGKAGgkuaMLGunrwGOqUB+nVvTAgcT7amelZRma0TRJsfFezBCAXXWNurD/6qAAKpKgmP+ouE+gH+SmG67IbQqIzK/5dc53sMV7dXDdG4/XGvmWfTph1iHZN/c1bd/rYPfVzy3K3ybTe2Djj0IaTbNndxVPfWIU6cC6cJ6ahEMvzokNnKNUqdDnu/Z1S2xiOfV4P/9vTg7O1XuLUylsCzveeYWRoZJPY4XCUnOgNlqQ/ZwKTC'
+                    accessKeyId: SecretManager_1.default.replaceSecret(source.authentication['accessKey']),
+                    secretAccessKey: SecretManager_1.default.replaceSecret(source.authentication['secretKey']),
+                    sessionToken: sessionToken ? sessionToken : undefined
                 }
             };
             try {

package/drivers/S3Driver.js

@@ -14,15 +14,18 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const client_s3_1 = require("@aws-sdk/client-s3");
 const Affirm_1 = __importDefault(require("../core/Affirm"));
+const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
 class S3Driver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
             this._bucketName = source.authentication['bucketName'];
+            const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
             const config = {
                 region: source.authentication['region'],
                 credentials: {
-                    accessKeyId: source.authentication['accessKey'],
-                    secretAccessKey: source.authentication['secretKey']
+                    accessKeyId: SecretManager_1.default.replaceSecret(source.authentication['accessKey']),
+                    secretAccessKey: SecretManager_1.default.replaceSecret(source.authentication['secretKey']),
+                    sessionToken: sessionToken ? sessionToken : undefined
                 }
             };
             this._client = new client_s3_1.S3Client(config);

package/drivers/S3SourceDriver.js

@@ -22,15 +22,18 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const client_s3_1 = require("@aws-sdk/client-s3");
 const readline_1 = __importDefault(require("readline"));
 const Affirm_1 = __importDefault(require("../core/Affirm"));
+const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
 class S3SourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
             this._bucketName = source.authentication['bucketName'];
+            const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
             const config = {
                 region: source.authentication['region'],
                 credentials: {
-                    accessKeyId: source.authentication['accessKey'],
-                    secretAccessKey: source.authentication['secretKey']
+                    accessKeyId: SecretManager_1.default.replaceSecret(source.authentication['accessKey']),
+                    secretAccessKey: SecretManager_1.default.replaceSecret(source.authentication['secretKey']),
+                    sessionToken: sessionToken ? sessionToken : undefined
                 }
             };
             this._client = new client_s3_1.S3Client(config);

package/engines/Environment.js

@@ -73,9 +73,7 @@ class EnvironmentClass {
             });
             // Initialize environment
             this.init({
-                settings: new Map(Object.entries({
-                    SQL_MAX_QUERY_ROWS: projectConfig.settings.SQL_MAX_QUERY_ROWS.toString()
-                })),
+                settings: new Map(Object.entries(Object.assign({}, projectConfig.settings)).map(([key, value]) => [key, String(value)])),
                 sources,
                 producers,
                 consumers,

package/engines/ProducerEngine.js

@@ -51,8 +51,8 @@ class ProducerEngineClass {
             for (const planStep of plan) {
                 switch (planStep.type) {
                     case 'create-view': {
-                        const internalSchema = Environment_1.default.get('schema');
-                        (0, Affirm_1.default)(internalSchema, `Invalid schema set on the authentication for source "${source.name}"`);
+                        const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
+                        (0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${producer.name}" wanting to create a view)`);
                         const sql = SQLCompiler_1.default.compileProducer(producer, source);
                         const vSQL = `CREATE OR REPLACE VIEW "${internalSchema}"."${SQLUtils_1.default.viewName(producer.name)}" AS ${sql}`;
                         yield driver.execute(vSQL);

package/engines/SecretManager.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+class SecretManagerClass {
+    constructor() {
+        this.replaceSecret = (value) => {
+            if (!value || value.length <= 2 || !value.startsWith('{') || !value.endsWith('}'))
+                return value;
+            const parsedValue = value.slice(1, value.length - 1);
+            return process.env[parsedValue];
+        };
+    }
+}
+const SecretManager = new SecretManagerClass();
+exports.default = SecretManager;

package/engines/Validator.js

@@ -121,7 +121,7 @@ class ValidatorClass {
                     const groupingFields = fields.filter(x => x.grouping);
                     if (groupingFields.length > 1)
                         errors.push(`There can't be 2 fields with grouping defined at the same level (${groupingFields.map(x => x.key).join(', ')}). Level: ${level}`);
-                    groupingFields.forEach(field => {
+                    groupingFields.forEach((field) => {
                         if (field.grouping)
                             errors = [...errors, ...validateGroupingLevels(field.grouping.subFields, level + 1)];
                     });
@@ -136,10 +136,11 @@ class ValidatorClass {
                 }
                 for (let i = 0; i < consumer.outputs.length; i++) {
                     const output = consumer.outputs[i];
-                    if (output.format === 'SQL' && output.accellerated && output.direct)
-                        errors.push(`An output SQL cannot be both direct and accelerated (output: ${output.format})`);
-                    if ((output.format === 'CSV' || output.format === 'JSON' || output.format === 'PARQUET') && !output.exportDestination)
-                        errors.push(`A static file output must have an export destination set (${output.format})`);
+                    const format = output.format.toUpperCase();
+                    if (format === 'SQL' && output.accellerated && output.direct)
+                        errors.push(`An output SQL cannot be both direct and accelerated (output: ${format})`);
+                    if ((format === 'CSV' || format === 'JSON' || format === 'PARQUET') && !output.exportDestination)
+                        errors.push(`A static file output must have an export destination set (${format})`);
                 }
             }
             catch (e) {

package/engines/consumer/ConsumerEngine.js

@@ -28,7 +28,7 @@ class ConsumerEngineClass {
         this.compile = (consumer) => {
             var _a, _b;
             (0, Affirm_1.default)(consumer, `Invalid consumer`);
-            const availableColumns = consumer.producers.flatMap((cProd) => {
+            const availableColumns = consumer.producers.flatMap(cProd => {
                 var _a, _b;
                 const producer = Environment_1.default.getProducer(cProd.name);
                 if (!producer) {
@@ -63,7 +63,7 @@ class ConsumerEngineClass {
                 // TODO: replace with the new funcitons in the consumermanager to reduce diplicate code
                 if (field.key === '*') {
                     const from = (_a = field.from) !== null && _a !== void 0 ? _a : (consumer.producers.length === 1 ? consumer.producers[0].name : null);
-                    availableColumns.filter(x => x.owner === from).forEach((col) => {
+                    availableColumns.filter(x => x.owner === from).forEach(col => {
                         col.consumerKey = col.nameInProducer;
                         col.consumerAlias = col.nameInProducer;
                         selectedColumns.push(col);
@@ -104,8 +104,8 @@ class ConsumerEngineClass {
                     case 'create-materialized-view': {
                         const sql = SQLCompiler_1.default.compileConsumer(consumer);
                         (0, Affirm_1.default)(sql, `Invalid SQL from deployment compilation for consumer "${consumer.name}"`);
-                        const internalSchema = Environment_1.default.get('schema');
-                        (0, Affirm_1.default)(internalSchema, `Invalid schema set on the authentication for source "${source.name}"`);
+                        const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
+                        (0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${consumer.name}" wanting to create a view)`);
                         // TODO When I want to update a materialize view there is no way except killing it and recreating it. The problem is that: 1) it is not said that it can be deleted since that materialize view could have some dependencies 2) we should find a way to update it without it going completely offline.
                         const mvSQL = `
                     DROP MATERIALIZED VIEW IF EXISTS "${internalSchema}"."${SQLUtils_1.default.acceleratedViewName(consumer.name)}";
@@ -116,8 +116,8 @@ class ConsumerEngineClass {
                     case 'create-view': {
                         const sql = SQLCompiler_1.default.compileConsumer(consumer);
                         (0, Affirm_1.default)(sql, `Invalid SQL from deployment compilation for consumer "${consumer.name}"`);
-                        const internalSchema = Environment_1.default.get('schema');
-                        (0, Affirm_1.default)(internalSchema, `Invalid schema set on the authentication for source "${source.name}"`);
+                        const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
+                        (0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${consumer.name}" wanting to create a view)`);
                         const vSQL = `CREATE OR REPLACE VIEW "${internalSchema}"."${SQLUtils_1.default.sanitizeName(consumer.name)}" AS ${sql}`;
                         yield driver.execute(vSQL);
                         break;
@@ -145,14 +145,14 @@ class ConsumerEngineClass {
             (0, Affirm_1.default)(consumer, `Invalid consumer`);
             const compiled = this.compile(consumer);
             const outDimensions = compiled.map(x => {
-                var _a;
+                var _a, _b, _c, _d, _e, _f, _g, _h;
                 return ({
                     name: (_a = x.consumerAlias) !== null && _a !== void 0 ? _a : x.consumerKey,
-                    type: x.dimension.type,
-                    classification: x.dimension.classification,
-                    description: x.dimension.description,
-                    mask: x.dimension.mask,
-                    pk: x.dimension.pk
+                    type: (_b = x.dimension) === null || _b === void 0 ? void 0 : _b.type,
+                    classification: (_c = x.dimension) === null || _c === void 0 ? void 0 : _c.classification,
+                    description: (_e = (_d = x.dimension) === null || _d === void 0 ? void 0 : _d.description) !== null && _e !== void 0 ? _e : (_f = x.measure) === null || _f === void 0 ? void 0 : _f.description,
+                    mask: (_g = x.dimension) === null || _g === void 0 ? void 0 : _g.mask,
+                    pk: (_h = x.dimension) === null || _h === void 0 ? void 0 : _h.pk
                 });
             });
             return {

package/engines/execution/ExecutionPlanner.js

@@ -56,7 +56,7 @@ class ExecutionPlannerClas {
             // TODO: how to handle pagination of SQL results?
             const engineClass = this.getEngineClass(producerEngine);
             for (const output of consumer.outputs) {
-                switch (output.format) {
+                switch (output.format.toUpperCase()) {
                     case 'JSON': {
                         if (engineClass === 'file' && Algo_1.default.hasVal(options))
                             plan.push({ type: 'apply-execution-request-to-result' });

package/engines/file/FileExporter.js

@@ -60,7 +60,7 @@ class FileExporterClass {
             let exportData = null;
             let extension = null;
             // build the actual file in the requested format
-            switch (output.format) {
+            switch (output.format.toUpperCase()) {
                 case 'CSV': {
                     const lines = [];
                     const keys = Object.keys(data[0]);

package/engines/sql/SQLCompiler.js

@@ -53,8 +53,8 @@ class SQLCompilerClass {
                 return sql;
             }
             else {
-                const internalSchema = Environment_1.default.get('schema');
-                (0, Affirm_1.default)(internalSchema, `Invalid schema set on the authentication for source "${source.name}"`);
+                const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
+                (0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${producer.name}" wanting to create a view)`);
                 return `CREATE OR REPLACE VIEW "${internalSchema}"."${producer.name}" AS ${sql}`;
             }
         };
@@ -70,8 +70,8 @@ class SQLCompilerClass {
                 return this.compileProducer(producer, source);
             }
             else {
-                const internalSchema = Environment_1.default.get('schema');
-                (0, Affirm_1.default)(internalSchema, `Invalid schema set on the authentication for source "${source.name}"`);
+                const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
+                (0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${producer.name}" wanting to create a view)`);
                 return `SELECT * FROM "${internalSchema}"."${producer.name}"`;
             }
         };

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.12",
-    "description": "Your Next.js and Node.js application with TypeScript CLI",
+    "version": "0.0.13",
+    "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
     "bin": {