@forzalabs/remora 0.0.14 → 0.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/Constants.js +1 -1
  2. package/actions/init.js +1 -2
  3. package/documentation/README.md +9 -0
  4. package/engines/ProducerEngine.js +5 -1
  5. package/engines/ai/DeveloperEngine.js +1 -0
  6. package/engines/consumer/PostProcessor.js +1 -1
  7. package/engines/file/FileExporter.js +4 -3
  8. package/package.json +1 -1
package/Constants.js CHANGED
@@ -1,7 +1,7 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  const CONSTANTS = {
4
- cliVersion: '0.0.14',
4
+ cliVersion: '0.0.16',
5
5
  lambdaVersion: 1,
6
6
  port: 5069
7
7
  };
package/actions/init.js CHANGED
@@ -32,7 +32,7 @@ const init = () => __awaiter(void 0, void 0, void 0, function* () {
32
32
  const defaultSource = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/source.json`), 'utf-8');
33
33
  const defaultConsumer = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/consumer.json`), 'utf-8');
34
34
  const defaultProducer = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/producer.json`), 'utf-8');
35
- const defaultSchema = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/schema.json`), 'utf-8');
35
+ // const defaultSchema = fs.readFileSync(path.join(__dirname, `../documentation/default_resources/schema.json`), 'utf-8')
36
36
  const defaultRemoraProject = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/project.json`), 'utf-8');
37
37
  const readme = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/README.md`), 'utf-8');
38
38
  const files = [
@@ -43,7 +43,6 @@ const init = () => __awaiter(void 0, void 0, void 0, function* () {
43
43
  { path: 'remora/producers/.gitkeep', content: '' },
44
44
  { path: 'remora/producers/default-producer.json', content: defaultProducer },
45
45
  { path: 'remora/schemas/.gitkeep', content: '' },
46
- { path: 'remora/schemas/default-schemas.json', content: defaultSchema },
47
46
  { path: 'remora/project.json', content: defaultRemoraProject },
48
47
  { path: 'remora/README.md', content: readme }
49
48
  ];
package/documentation/README.md CHANGED
@@ -3,6 +3,15 @@
3
3
  This README explains the available configurations for each resource type in the project.
4
4
  Each section details the expected values for different keys in the JSON files.
5
5
 
6
+ ## JSON Schemas
7
+
8
+ All configuration files follow specific JSON schemas to ensure consistency and validation. You can find the schema definitions at the following URLs:
9
+
10
+ - **Source**: [source-schema.json](https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/source-schema.json)
11
+ - **Producer**: [producer-schema.json](https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/producer-schema.json)
12
+ - **Consumer**: [consumer-schema.json](https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json)
13
+ - **Project**: [project-schema.json](https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/project-schema.json)
14
+
6
15
  ## Remora Configuration
7
16
 
8
17
  The main project configuration file.
package/engines/ProducerEngine.js CHANGED
@@ -132,7 +132,11 @@ class ProducerEngineClass {
132
132
  sampleData = ParseManager_1.default.csvLinesToJson(fileData.data, producer);
133
133
  }
134
134
  else if (((_b = producer.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase()) === 'JSON' || ((_c = producer.settings.fileType) === null || _c === void 0 ? void 0 : _c.toUpperCase()) === 'JSONL') {
135
- sampleData = fileData.data.map(line => JSON.parse(line));
135
+ // With JSON or JSONL the readFile function already parses the strings
136
+ if (typeof fileData.data[0] === 'object')
137
+ sampleData = fileData.data;
138
+ else
139
+ sampleData = fileData.data.map(line => JSON.parse(line));
136
140
  }
137
141
  else {
138
142
  sampleData = fileData.data;
package/engines/ai/DeveloperEngine.js CHANGED
@@ -37,6 +37,7 @@ class DeveloperEngineClass {
37
37
  measures: [],
38
38
  _version: (_a = producer._version) !== null && _a !== void 0 ? _a : 1
39
39
  };
40
+ mappedProducer['$schema'] = producer['$schema'];
40
41
  // Save the mapped producer to file
41
42
  const producerPath = path_1.default.join(process.cwd(), 'remora', 'producers', `${producer.name}.json`);
42
43
  yield promises_1.default.writeFile(producerPath, JSON.stringify(mappedProducer, null, 4), 'utf-8');
package/engines/consumer/PostProcessor.js CHANGED
@@ -58,7 +58,7 @@ class PostProcessorClass {
58
58
  const { key, alias } = field.cField;
59
59
  const fieldKey = alias !== null && alias !== void 0 ? alias : key;
60
60
  const maskType = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.mask;
61
- projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, x[fieldKey]);
61
+ projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, x[key]);
62
62
  }
63
63
  return projected;
64
64
  });
package/engines/file/FileExporter.js CHANGED
@@ -68,7 +68,7 @@ class FileExporterClass {
68
68
  for (let i = 0; i < data.length; i++) {
69
69
  const row = data[i];
70
70
  const columns = keys.map(key => row[key]);
71
- const rowValues = columns.map(c => Algo_1.default.replaceAll(c.toString(), '"', '""')).map(c => `"${c}"`).join(',');
71
+ const rowValues = columns.map(c => { var _a; return Algo_1.default.replaceAll((_a = c === null || c === void 0 ? void 0 : c.toString()) !== null && _a !== void 0 ? _a : '', '"', '""'); }).map(c => `"${c}"`).join(',');
72
72
  lines.push(rowValues);
73
73
  }
74
74
  const csv = lines.join('\n');
@@ -99,14 +99,15 @@ class FileExporterClass {
99
99
  const folder = source.authentication.path;
100
100
  if (!fs.existsSync(folder))
101
101
  fs.mkdirSync(folder);
102
- const path = `${folder}/${consumer.name}_${Algo_1.default.replaceAll(DSTE_1.default.now().toJSON(), ':', '-')}.${extension}`;
102
+ const date = DSTE_1.default.now().toISOString().split('.')[0];
103
+ const path = `${folder}/${consumer.name}_${Algo_1.default.replaceAll(date, ':', '-')}.${extension}`;
103
104
  fs.writeFileSync(path, exportData);
104
105
  return path;
105
106
  }
106
107
  case 'aws-s3': {
107
108
  const driver = yield DriverFactory_1.default.instantiateDestination(source);
108
109
  const date = DSTE_1.default.now().toISOString().split('.')[0];
109
- const res = yield driver.uploadFile({ content: exportData, name: `${consumer.name}_${date}.${extension}` });
110
+ const res = yield driver.uploadFile({ content: exportData, name: `${consumer.name}_${Algo_1.default.replaceAll(date, ':', '-')}.${extension}` });
110
111
  return res.key;
111
112
  }
112
113
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@forzalabs/remora",
3
- "version": "0.0.14",
3
+ "version": "0.0.16",
4
4
  "description": "A powerful CLI tool for seamless data translation.",
5
5
  "main": "index.js",
6
6
  "private": false,