@forzalabs/remora 1.0.22 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/actions/automap.js +26 -42
  2. package/actions/compile.js +27 -43
  3. package/actions/create_consumer.js +24 -40
  4. package/actions/create_producer.js +16 -32
  5. package/actions/debug.js +18 -34
  6. package/actions/deploy.js +30 -46
  7. package/actions/discover.js +13 -29
  8. package/actions/init.js +29 -45
  9. package/actions/mock.js +16 -32
  10. package/actions/run.js +34 -52
  11. package/actions/sample.js +42 -58
  12. package/index.js +109 -119
  13. package/package.json +4 -4
  14. package/workers/ExecutorWorker.js +18 -34
  15. package/Constants.js +0 -34
  16. package/core/Affirm.js +0 -42
  17. package/core/Algo.js +0 -160
  18. package/core/dste/DSTE.js +0 -113
  19. package/core/logger/DebugLogService.js +0 -48
  20. package/core/logger/DevelopmentLogService.js +0 -70
  21. package/core/logger/LocalLogService.js +0 -70
  22. package/core/logger/Logger.js +0 -54
  23. package/database/DatabaseEngine.js +0 -149
  24. package/database/DatabaseStructure.js +0 -27
  25. package/definitions/DatasetDefinitions.js +0 -2
  26. package/definitions/ExecutorDefinitions.js +0 -2
  27. package/definitions/ProcessENV.js +0 -169
  28. package/definitions/agents/DestinationDriver.js +0 -2
  29. package/definitions/agents/SourceDriver.js +0 -2
  30. package/definitions/cli.js +0 -2
  31. package/definitions/database/ApiKeys.js +0 -2
  32. package/definitions/database/Stored.js +0 -7
  33. package/definitions/database/UsageStat.js +0 -2
  34. package/definitions/database/User.js +0 -2
  35. package/definitions/json_schemas/consumer-schema.json +0 -1226
  36. package/definitions/json_schemas/producer-schema.json +0 -308
  37. package/definitions/json_schemas/project-schema.json +0 -100
  38. package/definitions/json_schemas/source-schema.json +0 -249
  39. package/definitions/requests/ConsumerRequest.js +0 -2
  40. package/definitions/requests/Developer.js +0 -2
  41. package/definitions/requests/Mapping.js +0 -2
  42. package/definitions/requests/ProducerRequest.js +0 -2
  43. package/definitions/requests/Request.js +0 -2
  44. package/definitions/resources/Compiled.js +0 -2
  45. package/definitions/resources/Consumer.js +0 -2
  46. package/definitions/resources/Environment.js +0 -2
  47. package/definitions/resources/Library.js +0 -2
  48. package/definitions/resources/Producer.js +0 -2
  49. package/definitions/resources/Project.js +0 -2
  50. package/definitions/resources/Schema.js +0 -2
  51. package/definitions/resources/Source.js +0 -2
  52. package/definitions/temp.js +0 -2
  53. package/definitions/transform/Transformations.js +0 -2
  54. package/drivers/DeltaShareDriver.js +0 -186
  55. package/drivers/DriverFactory.js +0 -72
  56. package/drivers/DriverHelper.js +0 -248
  57. package/drivers/HttpApiDriver.js +0 -208
  58. package/drivers/RedshiftDriver.js +0 -184
  59. package/drivers/files/LocalDestinationDriver.js +0 -146
  60. package/drivers/files/LocalSourceDriver.js +0 -405
  61. package/drivers/s3/S3DestinationDriver.js +0 -197
  62. package/drivers/s3/S3SourceDriver.js +0 -495
  63. package/engines/CryptoEngine.js +0 -75
  64. package/engines/Environment.js +0 -170
  65. package/engines/ProcessENVManager.js +0 -112
  66. package/engines/RandomEngine.js +0 -47
  67. package/engines/SecretManager.js +0 -23
  68. package/engines/UserManager.js +0 -66
  69. package/engines/ai/AutoMapperEngine.js +0 -37
  70. package/engines/ai/DeveloperEngine.js +0 -497
  71. package/engines/ai/LLM.js +0 -255
  72. package/engines/consumer/ConsumerManager.js +0 -218
  73. package/engines/consumer/ConsumerOnFinishManager.js +0 -202
  74. package/engines/dataset/Dataset.js +0 -824
  75. package/engines/dataset/DatasetManager.js +0 -211
  76. package/engines/dataset/DatasetRecord.js +0 -120
  77. package/engines/dataset/DatasetRecordPool.js +0 -77
  78. package/engines/execution/RequestExecutor.js +0 -67
  79. package/engines/parsing/CSVParser.js +0 -60
  80. package/engines/parsing/LineParser.js +0 -71
  81. package/engines/parsing/ParseCompression.js +0 -101
  82. package/engines/parsing/ParseHelper.js +0 -18
  83. package/engines/parsing/ParseManager.js +0 -54
  84. package/engines/parsing/XLSParser.js +0 -87
  85. package/engines/parsing/XMLParser.js +0 -115
  86. package/engines/producer/ProducerEngine.js +0 -127
  87. package/engines/producer/ProducerManager.js +0 -43
  88. package/engines/scheduler/CronScheduler.js +0 -222
  89. package/engines/scheduler/QueueManager.js +0 -314
  90. package/engines/schema/SchemaValidator.js +0 -67
  91. package/engines/transform/JoinEngine.js +0 -232
  92. package/engines/transform/TransformationEngine.js +0 -277
  93. package/engines/transform/TypeCaster.js +0 -59
  94. package/engines/usage/DataframeManager.js +0 -55
  95. package/engines/usage/UsageDataManager.js +0 -151
  96. package/engines/usage/UsageManager.js +0 -65
  97. package/engines/validation/Validator.js +0 -216
  98. package/executors/ConsumerExecutor.js +0 -280
  99. package/executors/Executor.js +0 -177
  100. package/executors/ExecutorOrchestrator.js +0 -331
  101. package/executors/ExecutorPerformance.js +0 -17
  102. package/executors/ExecutorProgress.js +0 -54
  103. package/executors/ExecutorScope.js +0 -52
  104. package/executors/OutputExecutor.js +0 -118
  105. package/executors/ProducerExecutor.js +0 -108
  106. package/helper/Helper.js +0 -149
  107. package/helper/Logger.js +0 -84
  108. package/helper/Runtime.js +0 -20
  109. package/helper/Settings.js +0 -13
  110. package/licencing/LicenceManager.js +0 -64
  111. package/settings.js +0 -12
package/definitions/ProcessENV.js DELETED
@@ -1,169 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.REMORA_ENV_VARIABLES = void 0;
4
- /**
5
- * All the environment variables required for operativity of remora
6
- */
7
- exports.REMORA_ENV_VARIABLES = [
8
- {
9
- name: 'REMORA_LICENCE_KEY',
10
- apps: [
11
- { name: 'cli', required: true },
12
- { name: 'worker', required: true },
13
- { name: 'backend', required: true }
14
- ],
15
- description: 'The licence key to activate this remora licence. If you don\'t have it contact your tech support or reach out to Forza Labs.'
16
- },
17
- {
18
- name: 'REMORA_APP',
19
- apps: [
20
- { name: 'cli', required: true },
21
- { name: 'worker', required: true },
22
- { name: 'backend', required: true }
23
- ],
24
- description: 'The Remora app name (cli, worker, backend, frontend).'
25
- },
26
- {
27
- name: 'REMORA_WORKER_HOST',
28
- apps: [
29
- { name: 'backend', required: true },
30
- { name: 'cli', required: false }
31
- ],
32
- description: 'The host address for the deployed Remora worker instance.'
33
- },
34
- {
35
- name: 'ADMIN_JWT_SECRET_FILE',
36
- apps: [
37
- { name: 'worker', required: true },
38
- { name: 'backend', required: true }
39
- ],
40
- description: 'The secret used to verify the JWT from the admin user of the Remora WebApp. Used for all admin operations.'
41
- },
42
- {
43
- name: 'JWT_SECRET_FILE',
44
- apps: [
45
- { name: 'worker', required: true },
46
- { name: 'backend', required: true }
47
- ],
48
- description: ''
49
- },
50
- {
51
- name: 'ROOT_TEMP_PASSWORD_FILE',
52
- apps: [
53
- { name: 'worker', required: true },
54
- { name: 'backend', required: true }
55
- ],
56
- description: ''
57
- },
58
- {
59
- name: 'REMORA_WORKERS_PATH',
60
- apps: [
61
- { name: 'worker', required: false },
62
- { name: 'cli', required: false }
63
- ],
64
- description: 'A valid path that overrides the default path for the multi-thread workers for the Remora engine.'
65
- },
66
- {
67
- name: 'REMORA_KEY_SECRET_FILE',
68
- apps: [
69
- { name: 'worker', required: true },
70
- { name: 'cli', required: false }
71
- ],
72
- description: 'The CLI '
73
- },
74
- {
75
- name: 'ADMIN_JWT_SECRET_FILE',
76
- apps: [
77
- { name: 'worker', required: true }
78
- ],
79
- description: 'The secret used to verify the JWT from the admin user of the Remora WebApp. Used for all admin operations.'
80
- },
81
- {
82
- name: 'ADMIN_JWT_SECRET_FILE',
83
- apps: [
84
- { name: 'worker', required: true }
85
- ],
86
- description: 'The secret used to verify the JWT from the admin user of the Remora WebApp. Used for all admin operations.'
87
- },
88
- {
89
- name: 'MONGO_URI',
90
- apps: [
91
- { name: 'worker', required: true },
92
- { name: 'backend', required: true }
93
- ],
94
- description: 'The databse uri (Mongo compatible) where to store Remora telemetry.'
95
- },
96
- {
97
- name: 'REMORA_SALT',
98
- apps: [
99
- { name: 'worker', required: false },
100
- { name: 'cli', required: false }
101
- ],
102
- description: 'A secret salt value used when masking data.'
103
- },
104
- {
105
- name: 'OPENAI_API_KEY',
106
- apps: [
107
- { name: 'worker', required: false },
108
- { name: 'cli', required: false }
109
- ],
110
- description: ''
111
- },
112
- {
113
- name: 'AWS_DEFAULT_REGION',
114
- apps: [
115
- { name: 'worker', required: false },
116
- { name: 'cli', required: false }
117
- ],
118
- description: ''
119
- },
120
- {
121
- name: 'AWS_ACCOUNT_ID',
122
- apps: [
123
- { name: 'worker', required: false },
124
- { name: 'cli', required: false }
125
- ],
126
- description: ''
127
- },
128
- {
129
- name: 'AWS_ACCESS_KEY_ID',
130
- apps: [
131
- { name: 'worker', required: false },
132
- { name: 'cli', required: false }
133
- ],
134
- description: ''
135
- },
136
- {
137
- name: 'AWS_SECRET_ACCESS_KEY',
138
- apps: [
139
- { name: 'worker', required: false },
140
- { name: 'cli', required: false }
141
- ],
142
- description: ''
143
- },
144
- {
145
- name: 'AWS_SESSION_TOKEN',
146
- apps: [
147
- { name: 'worker', required: false },
148
- { name: 'cli', required: false }
149
- ],
150
- description: ''
151
- },
152
- {
153
- name: 'DELTA_TOKEN',
154
- apps: [
155
- { name: 'worker', required: false },
156
- { name: 'cli', required: false }
157
- ],
158
- description: ''
159
- },
160
- {
161
- name: 'NODE_ENV',
162
- apps: [
163
- { name: 'worker', required: false },
164
- { name: 'cli', required: false },
165
- { name: 'backend', required: false }
166
- ],
167
- description: ''
168
- }
169
- ];
package/definitions/agents/DestinationDriver.js DELETED
@@ -1,2 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
package/definitions/agents/SourceDriver.js DELETED
@@ -1,2 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
package/definitions/cli.js DELETED
@@ -1,2 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
package/definitions/database/ApiKeys.js DELETED
@@ -1,2 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
package/definitions/database/Stored.js DELETED
@@ -1,7 +0,0 @@
1
- "use strict";
2
- /**
3
- * TODO: these are the documents that are stored in the database.
4
- * TODO: I need to register all the events and usage around these resources (producers and consumers).
5
- * Like all the executions, who executed, for how long and others...
6
- */
7
- Object.defineProperty(exports, "__esModule", { value: true });
package/definitions/database/UsageStat.js DELETED
@@ -1,2 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
package/definitions/database/User.js DELETED
@@ -1,2 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });