@forzalabs/remora 1.0.21 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/actions/automap.js +26 -42
  2. package/actions/compile.js +27 -43
  3. package/actions/create_consumer.js +24 -40
  4. package/actions/create_producer.js +16 -32
  5. package/actions/debug.js +18 -34
  6. package/actions/deploy.js +30 -46
  7. package/actions/discover.js +13 -29
  8. package/actions/init.js +29 -45
  9. package/actions/mock.js +16 -32
  10. package/actions/run.js +34 -52
  11. package/actions/sample.js +42 -58
  12. package/index.js +38 -43
  13. package/package.json +4 -4
  14. package/workers/ExecutorWorker.js +18 -32
  15. package/Constants.js +0 -34
  16. package/core/Affirm.js +0 -42
  17. package/core/Algo.js +0 -160
  18. package/core/dste/DSTE.js +0 -113
  19. package/core/logger/DebugLogService.js +0 -48
  20. package/core/logger/DevelopmentLogService.js +0 -70
  21. package/core/logger/LocalLogService.js +0 -70
  22. package/core/logger/Logger.js +0 -54
  23. package/database/DatabaseEngine.js +0 -149
  24. package/database/DatabaseStructure.js +0 -27
  25. package/definitions/DatasetDefinitions.js +0 -2
  26. package/definitions/ExecutorDefinitions.js +0 -2
  27. package/definitions/ProcessENV.js +0 -2
  28. package/definitions/agents/DestinationDriver.js +0 -2
  29. package/definitions/agents/SourceDriver.js +0 -2
  30. package/definitions/cli.js +0 -2
  31. package/definitions/database/ApiKeys.js +0 -2
  32. package/definitions/database/Stored.js +0 -7
  33. package/definitions/database/UsageStat.js +0 -2
  34. package/definitions/database/User.js +0 -2
  35. package/definitions/json_schemas/consumer-schema.json +0 -1226
  36. package/definitions/json_schemas/producer-schema.json +0 -308
  37. package/definitions/json_schemas/project-schema.json +0 -100
  38. package/definitions/json_schemas/source-schema.json +0 -249
  39. package/definitions/requests/ConsumerRequest.js +0 -2
  40. package/definitions/requests/Developer.js +0 -2
  41. package/definitions/requests/Mapping.js +0 -2
  42. package/definitions/requests/ProducerRequest.js +0 -2
  43. package/definitions/requests/Request.js +0 -2
  44. package/definitions/resources/Compiled.js +0 -2
  45. package/definitions/resources/Consumer.js +0 -2
  46. package/definitions/resources/Environment.js +0 -2
  47. package/definitions/resources/Library.js +0 -2
  48. package/definitions/resources/Producer.js +0 -2
  49. package/definitions/resources/Project.js +0 -2
  50. package/definitions/resources/Schema.js +0 -2
  51. package/definitions/resources/Source.js +0 -2
  52. package/definitions/temp.js +0 -2
  53. package/definitions/transform/Transformations.js +0 -2
  54. package/drivers/DeltaShareDriver.js +0 -186
  55. package/drivers/DriverFactory.js +0 -72
  56. package/drivers/DriverHelper.js +0 -248
  57. package/drivers/HttpApiDriver.js +0 -208
  58. package/drivers/RedshiftDriver.js +0 -184
  59. package/drivers/files/LocalDestinationDriver.js +0 -146
  60. package/drivers/files/LocalSourceDriver.js +0 -405
  61. package/drivers/s3/S3DestinationDriver.js +0 -197
  62. package/drivers/s3/S3SourceDriver.js +0 -495
  63. package/engines/CryptoEngine.js +0 -75
  64. package/engines/Environment.js +0 -170
  65. package/engines/ProcessENVManager.js +0 -83
  66. package/engines/RandomEngine.js +0 -47
  67. package/engines/SecretManager.js +0 -23
  68. package/engines/UserManager.js +0 -66
  69. package/engines/ai/AutoMapperEngine.js +0 -37
  70. package/engines/ai/DeveloperEngine.js +0 -497
  71. package/engines/ai/LLM.js +0 -255
  72. package/engines/consumer/ConsumerManager.js +0 -218
  73. package/engines/consumer/ConsumerOnFinishManager.js +0 -202
  74. package/engines/dataset/Dataset.js +0 -824
  75. package/engines/dataset/DatasetManager.js +0 -211
  76. package/engines/dataset/DatasetRecord.js +0 -120
  77. package/engines/dataset/DatasetRecordPool.js +0 -77
  78. package/engines/execution/RequestExecutor.js +0 -67
  79. package/engines/parsing/CSVParser.js +0 -60
  80. package/engines/parsing/LineParser.js +0 -71
  81. package/engines/parsing/ParseCompression.js +0 -101
  82. package/engines/parsing/ParseHelper.js +0 -18
  83. package/engines/parsing/ParseManager.js +0 -54
  84. package/engines/parsing/XLSParser.js +0 -87
  85. package/engines/parsing/XMLParser.js +0 -115
  86. package/engines/producer/ProducerEngine.js +0 -127
  87. package/engines/producer/ProducerManager.js +0 -43
  88. package/engines/scheduler/CronScheduler.js +0 -222
  89. package/engines/scheduler/QueueManager.js +0 -314
  90. package/engines/schema/SchemaValidator.js +0 -67
  91. package/engines/transform/JoinEngine.js +0 -232
  92. package/engines/transform/TransformationEngine.js +0 -277
  93. package/engines/transform/TypeCaster.js +0 -59
  94. package/engines/usage/DataframeManager.js +0 -55
  95. package/engines/usage/UsageDataManager.js +0 -151
  96. package/engines/usage/UsageManager.js +0 -65
  97. package/engines/validation/Validator.js +0 -216
  98. package/executors/ConsumerExecutor.js +0 -280
  99. package/executors/Executor.js +0 -177
  100. package/executors/ExecutorOrchestrator.js +0 -331
  101. package/executors/ExecutorPerformance.js +0 -17
  102. package/executors/ExecutorProgress.js +0 -54
  103. package/executors/ExecutorScope.js +0 -52
  104. package/executors/OutputExecutor.js +0 -118
  105. package/executors/ProducerExecutor.js +0 -108
  106. package/helper/Helper.js +0 -149
  107. package/helper/Logger.js +0 -84
  108. package/helper/Runtime.js +0 -20
  109. package/helper/Settings.js +0 -13
  110. package/licencing/LicenceManager.js +0 -64
  111. package/settings.js +0 -12
@@ -1,222 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
36
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
37
- return new (P || (P = Promise))(function (resolve, reject) {
38
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
39
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
40
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
41
- step((generator = generator.apply(thisArg, _arguments || [])).next());
42
- });
43
- };
44
- var __importDefault = (this && this.__importDefault) || function (mod) {
45
- return (mod && mod.__esModule) ? mod : { "default": mod };
46
- };
47
- Object.defineProperty(exports, "__esModule", { value: true });
48
- const cron = __importStar(require("node-cron"));
49
- const Environment_1 = __importDefault(require("../Environment"));
50
- const ExecutorOrchestrator_1 = __importDefault(require("../../executors/ExecutorOrchestrator"));
51
- const settings_1 = require("../../settings");
52
- class CronScheduler {
53
- constructor() {
54
- this.scheduledJobs = new Map();
55
- this.isInitialized = false;
56
- }
57
- /**
58
- * Initialize the CRON scheduler by scanning all consumers and scheduling those with CRON triggers
59
- */
60
- initialize() {
61
- if (this.isInitialized) {
62
- console.log('CRON scheduler already initialized');
63
- return;
64
- }
65
- console.log('Initializing CRON scheduler...');
66
- try {
67
- const consumers = Environment_1.default.getAllConsumers();
68
- let cronJobCount = 0;
69
- for (const consumer of consumers) {
70
- if (this.hasCronTrigger(consumer)) {
71
- this.scheduleConsumer(consumer);
72
- cronJobCount++;
73
- }
74
- }
75
- this.isInitialized = true;
76
- console.log(`CRON scheduler initialized with ${cronJobCount} scheduled jobs`);
77
- }
78
- catch (error) {
79
- console.error('Failed to initialize CRON scheduler:', error);
80
- throw error;
81
- }
82
- }
83
- /**
84
- * Check if a consumer has any CRON triggers configured
85
- */
86
- hasCronTrigger(consumer) {
87
- return consumer.outputs.some(output => { var _a; return ((_a = output.trigger) === null || _a === void 0 ? void 0 : _a.type) === 'CRON' && output.trigger.value; });
88
- }
89
- /**
90
- * Schedule a consumer with CRON triggers
91
- */
92
- scheduleConsumer(consumer) {
93
- consumer.outputs.forEach((output, index) => {
94
- var _a;
95
- if (((_a = output.trigger) === null || _a === void 0 ? void 0 : _a.type) === 'CRON' && output.trigger.value) {
96
- const jobKey = `${consumer.name}_output_${index}`;
97
- const cronExpression = output.trigger.value;
98
- try {
99
- // Validate CRON expression
100
- if (!cron.validate(cronExpression)) {
101
- console.error(`Invalid CRON expression for consumer ${consumer.name}: ${cronExpression}`);
102
- return;
103
- }
104
- // Schedule the job
105
- const task = cron.schedule(cronExpression, () => __awaiter(this, void 0, void 0, function* () {
106
- yield this.executeConsumerOutput(consumer, output, index);
107
- }));
108
- // Don't start the task immediately, we'll start it manually
109
- task.stop();
110
- this.scheduledJobs.set(jobKey, task);
111
- task.start();
112
- console.log(`Scheduled CRON job for consumer "${consumer.name}" output ${index} with expression: ${cronExpression}`);
113
- }
114
- catch (error) {
115
- console.error(`Failed to schedule CRON job for consumer ${consumer.name}:`, error);
116
- }
117
- }
118
- });
119
- }
120
- /**
121
- * Execute a consumer output when triggered by CRON
122
- */
123
- executeConsumerOutput(consumer, output, outputIndex) {
124
- return __awaiter(this, void 0, void 0, function* () {
125
- try {
126
- console.log(`Executing CRON job for consumer "${consumer.name}" output ${outputIndex}`);
127
- const user = settings_1.REMORA_WORKER_USER;
128
- const runner = { _id: user._id, name: user.name, type: 'actor' };
129
- const result = yield ExecutorOrchestrator_1.default.launch({
130
- consumer,
131
- details: {
132
- invokedBy: 'CRON',
133
- user: runner
134
- },
135
- logProgress: false
136
- });
137
- console.log(`CRON job completed successfully for consumer "${consumer.name}" output ${outputIndex}`);
138
- // Log execution statistics
139
- if (result) {
140
- console.log(`CRON job stats: ${result.elapsedMS}ms, size: ${result.outputCount}, cycles: ${result.cycles}`);
141
- }
142
- }
143
- catch (error) {
144
- console.error(`CRON job failed for consumer "${consumer.name}" output ${outputIndex}:`, error);
145
- // Optionally, you could implement error handling strategies here:
146
- // - Send notifications
147
- // - Log to a monitoring system
148
- // - Retry logic
149
- // - Disable the job after repeated failures
150
- }
151
- });
152
- }
153
- /**
154
- * Add or update a CRON job for a specific consumer
155
- */
156
- updateConsumerSchedule(consumer) {
157
- // First, remove any existing schedules for this consumer
158
- this.removeConsumerSchedule(consumer.name);
159
- // Then, add new schedules if they have CRON triggers
160
- if (this.hasCronTrigger(consumer)) {
161
- this.scheduleConsumer(consumer);
162
- }
163
- }
164
- /**
165
- * Remove all scheduled jobs for a consumer
166
- */
167
- removeConsumerSchedule(consumerName) {
168
- const jobsToRemove = Array.from(this.scheduledJobs.keys()).filter(key => key.startsWith(`${consumerName}_output_`));
169
- jobsToRemove.forEach(jobKey => {
170
- const task = this.scheduledJobs.get(jobKey);
171
- if (task) {
172
- task.stop();
173
- task.destroy();
174
- this.scheduledJobs.delete(jobKey);
175
- console.log(`Removed CRON job: ${jobKey}`);
176
- }
177
- });
178
- }
179
- /**
180
- * Get information about all scheduled jobs
181
- */
182
- getScheduledJobs() {
183
- return Array.from(this.scheduledJobs.entries()).map(([jobKey, task]) => ({
184
- jobKey,
185
- isRunning: task.getStatus() === 'scheduled'
186
- }));
187
- }
188
- /**
189
- * Stop all scheduled jobs
190
- */
191
- stopAllJobs() {
192
- console.log('Stopping all CRON jobs...');
193
- this.scheduledJobs.forEach((task, jobKey) => {
194
- task.stop();
195
- task.destroy();
196
- console.log(`Stopped CRON job: ${jobKey}`);
197
- });
198
- this.scheduledJobs.clear();
199
- this.isInitialized = false;
200
- console.log('All CRON jobs stopped');
201
- }
202
- /**
203
- * Restart the scheduler (useful for configuration reloads)
204
- */
205
- restart() {
206
- console.log('Restarting CRON scheduler...');
207
- this.stopAllJobs();
208
- this.initialize();
209
- }
210
- /**
211
- * Get the scheduler status
212
- */
213
- getStatus() {
214
- return {
215
- initialized: this.isInitialized,
216
- jobCount: this.scheduledJobs.size,
217
- jobs: this.getScheduledJobs()
218
- };
219
- }
220
- }
221
- // Export a singleton instance
222
- exports.default = new CronScheduler();
@@ -1,314 +0,0 @@
1
- "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __importDefault = (this && this.__importDefault) || function (mod) {
12
- return (mod && mod.__esModule) ? mod : { "default": mod };
13
- };
14
- Object.defineProperty(exports, "__esModule", { value: true });
15
- const client_sqs_1 = require("@aws-sdk/client-sqs");
16
- const Environment_1 = __importDefault(require("../Environment"));
17
- const ProcessENVManager_1 = __importDefault(require("../ProcessENVManager"));
18
- const SecretManager_1 = __importDefault(require("../SecretManager"));
19
- const ExecutorOrchestrator_1 = __importDefault(require("../../executors/ExecutorOrchestrator"));
20
- const settings_1 = require("../../settings");
21
- class QueueManager {
22
- constructor() {
23
- this.queueMappings = new Map();
24
- this.pollingIntervals = new Map();
25
- this.isInitialized = false;
26
- this.POLLING_INTERVAL_MS = 5000; // Poll every 5 seconds
27
- this.MAX_MESSAGES = 10; // Maximum messages to receive in one poll
28
- // Initialize SQS client with default configuration
29
- // Will be reconfigured when we know the specific queue details
30
- this.sqsClient = new client_sqs_1.SQSClient({});
31
- }
32
- /**
33
- * Initialize the Queue Manager by scanning all consumers and setting up queue listeners for those with QUEUE triggers
34
- */
35
- initialize() {
36
- if (this.isInitialized) {
37
- console.log('Queue Manager already initialized');
38
- return;
39
- }
40
- console.log('Initializing Queue Manager...');
41
- try {
42
- const consumers = Environment_1.default.getAllConsumers();
43
- let queueTriggerCount = 0;
44
- for (const consumer of consumers) {
45
- if (this.hasQueueTrigger(consumer)) {
46
- this.setupQueueListeners(consumer);
47
- queueTriggerCount++;
48
- }
49
- }
50
- this.isInitialized = true;
51
- console.log(`Queue Manager initialized with ${queueTriggerCount} queue triggers`);
52
- }
53
- catch (error) {
54
- console.error('Failed to initialize Queue Manager:', error);
55
- throw error;
56
- }
57
- }
58
- /**
59
- * Check if a consumer has any QUEUE triggers configured
60
- */
61
- hasQueueTrigger(consumer) {
62
- return consumer.outputs.some(output => { var _a; return ((_a = output.trigger) === null || _a === void 0 ? void 0 : _a.type) === 'QUEUE' && output.trigger.value; });
63
- }
64
- /**
65
- * Setup queue listeners for a consumer with QUEUE triggers
66
- */
67
- setupQueueListeners(consumer) {
68
- consumer.outputs.forEach((output, index) => {
69
- var _a;
70
- if (((_a = output.trigger) === null || _a === void 0 ? void 0 : _a.type) === 'QUEUE' && output.trigger.value) {
71
- try {
72
- const queueConfig = this.parseQueueConfig(output.trigger.value, output.trigger.metadata);
73
- const mapping = {
74
- consumer,
75
- outputIndex: index,
76
- queueUrl: queueConfig.queueUrl,
77
- messageType: queueConfig.messageType
78
- };
79
- // Add to mappings
80
- if (!this.queueMappings.has(queueConfig.queueUrl)) {
81
- this.queueMappings.set(queueConfig.queueUrl, []);
82
- }
83
- this.queueMappings.get(queueConfig.queueUrl).push(mapping);
84
- // Start polling for this queue if not already started
85
- if (!this.pollingIntervals.has(queueConfig.queueUrl)) {
86
- this.startQueuePolling(queueConfig.queueUrl, queueConfig.region, queueConfig.credentials);
87
- }
88
- console.log(`Setup queue listener for consumer "${consumer.name}" output ${index} on queue: ${queueConfig.queueUrl}`);
89
- }
90
- catch (error) {
91
- console.error(`Failed to setup queue listener for consumer ${consumer.name}:`, error);
92
- }
93
- }
94
- });
95
- }
96
- /**
97
- * Parse queue configuration from trigger value and metadata
98
- */
99
- parseQueueConfig(triggerValue, metadata) {
100
- // triggerValue should be the queue URL or queue name
101
- let queueUrl = triggerValue;
102
- // If it's not a full URL, construct it
103
- if (!queueUrl.startsWith('https://')) {
104
- const region = (metadata === null || metadata === void 0 ? void 0 : metadata.region) || ProcessENVManager_1.default.getEnvVariable('AWS_DEFAULT_REGION') || 'us-east-1';
105
- const accountId = (metadata === null || metadata === void 0 ? void 0 : metadata.accountId) || ProcessENVManager_1.default.getEnvVariable('AWS_ACCOUNT_ID');
106
- if (!accountId) {
107
- throw new Error('AWS Account ID is required for queue trigger. Set it in metadata.accountId or AWS_ACCOUNT_ID environment variable');
108
- }
109
- queueUrl = `https://sqs.${region}.amazonaws.com/${accountId}/${triggerValue}`;
110
- }
111
- // Extract region from URL if not provided in metadata
112
- const urlParts = queueUrl.match(/https:\/\/sqs\.([^.]+)\.amazonaws\.com\//);
113
- const region = (metadata === null || metadata === void 0 ? void 0 : metadata.region) || (urlParts === null || urlParts === void 0 ? void 0 : urlParts[1]) || 'us-east-1';
114
- // Get credentials from metadata or environment
115
- let credentials;
116
- const accessKeyId = (metadata === null || metadata === void 0 ? void 0 : metadata.accessKeyId) || ProcessENVManager_1.default.getEnvVariable('AWS_ACCESS_KEY_ID');
117
- const secretAccessKey = (metadata === null || metadata === void 0 ? void 0 : metadata.secretAccessKey) || ProcessENVManager_1.default.getEnvVariable('AWS_SECRET_ACCESS_KEY');
118
- const sessionToken = (metadata === null || metadata === void 0 ? void 0 : metadata.sessionToken) || ProcessENVManager_1.default.getEnvVariable('AWS_SESSION_TOKEN');
119
- if (accessKeyId && secretAccessKey) {
120
- credentials = {
121
- accessKeyId: SecretManager_1.default.replaceSecret(accessKeyId),
122
- secretAccessKey: SecretManager_1.default.replaceSecret(secretAccessKey),
123
- sessionToken: sessionToken ? SecretManager_1.default.replaceSecret(sessionToken) : undefined
124
- };
125
- }
126
- return {
127
- queueUrl,
128
- messageType: metadata === null || metadata === void 0 ? void 0 : metadata.messageType,
129
- region,
130
- credentials
131
- };
132
- }
133
- /**
134
- * Start polling a specific queue for messages
135
- */
136
- startQueuePolling(queueUrl, region, credentials) {
137
- // Create SQS client for this specific queue
138
- const sqsClient = new client_sqs_1.SQSClient({
139
- region,
140
- credentials
141
- });
142
- const pollQueue = () => __awaiter(this, void 0, void 0, function* () {
143
- try {
144
- const command = new client_sqs_1.ReceiveMessageCommand({
145
- QueueUrl: queueUrl,
146
- MaxNumberOfMessages: this.MAX_MESSAGES,
147
- WaitTimeSeconds: 20, // Long polling
148
- VisibilityTimeout: 300 // 5 minutes to process the message
149
- });
150
- const response = yield sqsClient.send(command);
151
- if (response.Messages && response.Messages.length > 0) {
152
- console.log(`Received ${response.Messages.length} messages from queue: ${queueUrl}`);
153
- for (const message of response.Messages) {
154
- yield this.processMessage(queueUrl, message, sqsClient);
155
- }
156
- }
157
- }
158
- catch (error) {
159
- console.error(`Error polling queue ${queueUrl}:`, error);
160
- }
161
- });
162
- // Start continuous polling
163
- const interval = setInterval(pollQueue, this.POLLING_INTERVAL_MS);
164
- this.pollingIntervals.set(queueUrl, interval);
165
- // Start immediately
166
- pollQueue();
167
- console.log(`Started polling queue: ${queueUrl}`);
168
- }
169
- /**
170
- * Process a message from the queue
171
- */
172
- processMessage(queueUrl, message, sqsClient) {
173
- return __awaiter(this, void 0, void 0, function* () {
174
- try {
175
- const mappings = this.queueMappings.get(queueUrl);
176
- if (!mappings || mappings.length === 0) {
177
- console.log(`No consumer mappings found for queue: ${queueUrl}`);
178
- return;
179
- }
180
- // Parse message body
181
- let messageData;
182
- try {
183
- messageData = JSON.parse(message.Body || '{}');
184
- }
185
- catch (_a) {
186
- console.warn(`Failed to parse message body as JSON for queue ${queueUrl}. Using raw body.`);
187
- messageData = { body: message.Body };
188
- }
189
- let messageProcessedByAnyConsumer = false;
190
- // Process message for each mapped consumer that matches the message criteria
191
- for (const mapping of mappings) {
192
- try {
193
- // Check if message type matches (if specified)
194
- if (mapping.messageType) {
195
- const messageType = messageData.type || messageData.messageType || messageData.eventType;
196
- if (messageType !== mapping.messageType) {
197
- console.log(`Message type ${messageType} does not match expected ${mapping.messageType} for consumer ${mapping.consumer.name} - skipping`);
198
- continue;
199
- }
200
- }
201
- console.log(`Processing queue message for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}`);
202
- const user = settings_1.REMORA_WORKER_USER;
203
- const result = yield ExecutorOrchestrator_1.default.launch({
204
- consumer: mapping.consumer,
205
- details: {
206
- invokedBy: 'QUEUE',
207
- user: { _id: user._id, name: user.name, type: 'actor' }
208
- },
209
- logProgress: false
210
- });
211
- console.log(`Queue trigger completed successfully for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}`);
212
- // Log execution statistics
213
- if (result) {
214
- console.log(`Queue trigger stats: ${result.elapsedMS}ms, size: ${result.outputCount}, cycles: ${result.cycles}`);
215
- }
216
- messageProcessedByAnyConsumer = true;
217
- }
218
- catch (error) {
219
- console.error(`Queue trigger failed for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}:`, error);
220
- // Continue processing for other consumers even if one fails
221
- }
222
- }
223
- // Only delete message from queue if it was processed by at least one consumer
224
- // This ensures messages intended for other consumers or systems remain in the queue
225
- if (messageProcessedByAnyConsumer && message.ReceiptHandle) {
226
- yield sqsClient.send(new client_sqs_1.DeleteMessageCommand({
227
- QueueUrl: queueUrl,
228
- ReceiptHandle: message.ReceiptHandle
229
- }));
230
- console.log(`Deleted processed message ${message.MessageId} from queue`);
231
- }
232
- else if (!messageProcessedByAnyConsumer) {
233
- console.log(`Message ${message.MessageId} was not processed by any consumer - leaving in queue for other consumers or systems`);
234
- }
235
- }
236
- catch (error) {
237
- console.error(`Error processing message from queue ${queueUrl}:`, error);
238
- // Message will remain in queue and be retried or go to DLQ based on queue configuration
239
- }
240
- });
241
- }
242
- /**
243
- * Add or update queue listeners for a specific consumer
244
- */
245
- updateConsumerSchedule(consumer) {
246
- // First, remove any existing listeners for this consumer
247
- this.removeConsumerSchedule(consumer.name);
248
- // Then, add new listeners if they have QUEUE triggers
249
- if (this.hasQueueTrigger(consumer)) {
250
- this.setupQueueListeners(consumer);
251
- }
252
- }
253
- /**
254
- * Remove all queue listeners for a consumer
255
- */
256
- removeConsumerSchedule(consumerName) {
257
- // Remove mappings for this consumer
258
- for (const [queueUrl, mappings] of this.queueMappings.entries()) {
259
- const updatedMappings = mappings.filter(mapping => mapping.consumer.name !== consumerName);
260
- if (updatedMappings.length === 0) {
261
- // No more consumers listening to this queue, stop polling
262
- const interval = this.pollingIntervals.get(queueUrl);
263
- if (interval) {
264
- clearInterval(interval);
265
- this.pollingIntervals.delete(queueUrl);
266
- console.log(`Stopped polling queue: ${queueUrl}`);
267
- }
268
- this.queueMappings.delete(queueUrl);
269
- }
270
- else {
271
- this.queueMappings.set(queueUrl, updatedMappings);
272
- }
273
- }
274
- }
275
- /**
276
- * Stop all queue polling
277
- */
278
- stopAllQueues() {
279
- console.log('Stopping all queue polling...');
280
- this.pollingIntervals.forEach((interval, queueUrl) => {
281
- clearInterval(interval);
282
- console.log(`Stopped polling queue: ${queueUrl}`);
283
- });
284
- this.pollingIntervals.clear();
285
- this.queueMappings.clear();
286
- this.isInitialized = false;
287
- console.log('All queue polling stopped');
288
- }
289
- /**
290
- * Restart the queue manager (useful for configuration reloads)
291
- */
292
- restart() {
293
- console.log('Restarting Queue Manager...');
294
- this.stopAllQueues();
295
- this.initialize();
296
- }
297
- /**
298
- * Get the queue manager status
299
- */
300
- getStatus() {
301
- const queues = Array.from(this.queueMappings.entries()).map(([queueUrl, mappings]) => ({
302
- queueUrl,
303
- consumerCount: mappings.length
304
- }));
305
- return {
306
- initialized: this.isInitialized,
307
- queueCount: this.queueMappings.size,
308
- consumerCount: Array.from(this.queueMappings.values()).flat().length,
309
- queues
310
- };
311
- }
312
- }
313
- // Export a singleton instance
314
- exports.default = new QueueManager();
@@ -1,67 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- const ajv_1 = __importDefault(require("ajv"));
7
- const ajv_formats_1 = __importDefault(require("ajv-formats"));
8
- const fs_1 = __importDefault(require("fs"));
9
- const path_1 = __importDefault(require("path"));
10
- const Affirm_1 = __importDefault(require("../../core/Affirm"));
11
- class SchemaValidatorClass {
12
- constructor() {
13
- this.addSchema = (schema, schemaKey) => {
14
- const isValid = this.ajv.validateSchema(schema);
15
- (0, Affirm_1.default)(isValid, `Invalid JSON Schema: ${JSON.stringify(this.ajv.errors)}`);
16
- this.ajv.addSchema(schema, schemaKey);
17
- };
18
- this.getSchema = (schemaKey) => {
19
- return this.ajv.getSchema(schemaKey);
20
- };
21
- this.validate = (schemaKey, data) => {
22
- var _a;
23
- const validateFn = this.ajv.getSchema(schemaKey);
24
- (0, Affirm_1.default)(validateFn, `Failed to get validator for schema '${schemaKey}'`);
25
- const isValid = validateFn(data);
26
- return { isValid, errors: (_a = validateFn.errors) !== null && _a !== void 0 ? _a : [], data };
27
- };
28
- this.validateAndReturn = (schemaKey, data) => {
29
- const res = this.validate(schemaKey, data);
30
- if (res.isValid)
31
- return res.data;
32
- else
33
- return null;
34
- };
35
- this.unloadSchemas = () => {
36
- const result = this.ajv.removeSchema();
37
- (0, Affirm_1.default)(result, `Failed to remove schemas`);
38
- this.loadInternalSchema();
39
- return true;
40
- };
41
- this.loadInternalSchema = () => {
42
- const internalSchemaDir = path_1.default.join(__dirname, '../../definitions/json_schemas');
43
- const internalSchemas = [
44
- 'project-schema',
45
- 'source-schema',
46
- 'producer-schema',
47
- 'consumer-schema'
48
- ];
49
- internalSchemas.forEach(sn => {
50
- const sp = path_1.default.join(internalSchemaDir, `${sn}.json`);
51
- const parsed = JSON.parse(fs_1.default.readFileSync(sp, 'utf-8'));
52
- this.addSchema(parsed, sn);
53
- });
54
- return true;
55
- };
56
- this.ajv = new ajv_1.default({
57
- allErrors: true,
58
- strict: true,
59
- strictSchema: false,
60
- validateFormats: true
61
- });
62
- (0, ajv_formats_1.default)(this.ajv);
63
- this.loadInternalSchema();
64
- }
65
- }
66
- const SchemaValidator = new SchemaValidatorClass();
67
- exports.default = SchemaValidator;