@friggframework/core 2.0.0--canary.580.9716b6d.0 → 2.0.0--canary.585.8f515e1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/Worker.js +36 -0
- package/core/create-handler.js +64 -0
- package/database/encryption/encryption-schema-registry.js +133 -0
- package/database/encryption/field-encryption-service.js +30 -2
- package/database/encryption/prisma-encryption-extension.js +10 -2
- package/handlers/backend-utils.js +44 -6
- package/integrations/integration-router.js +27 -6
- package/modules/module.js +3 -1
- package/modules/use-cases/get-module-instance-from-type.js +4 -1
- package/modules/use-cases/process-authorization-callback.js +71 -5
- package/package.json +5 -5
- package/queues/queuer-util.js +80 -3
package/core/Worker.js
CHANGED
|
@@ -20,15 +20,45 @@ class Worker {
|
|
|
20
20
|
const records = get(params, 'Records');
|
|
21
21
|
const batchItemFailures = [];
|
|
22
22
|
|
|
23
|
+
console.log(
|
|
24
|
+
`[Worker] run: processing ${records.length} record(s)`
|
|
25
|
+
);
|
|
26
|
+
|
|
23
27
|
for (const record of records) {
|
|
28
|
+
// Log record entry with SQS-provided attributes useful for tracing
|
|
29
|
+
// delivery history (ApproximateReceiveCount for retries, etc.).
|
|
30
|
+
let parsedEvent;
|
|
31
|
+
try {
|
|
32
|
+
parsedEvent = JSON.parse(record.body)?.event;
|
|
33
|
+
} catch {
|
|
34
|
+
parsedEvent = undefined;
|
|
35
|
+
}
|
|
36
|
+
console.log(`[Worker] record begin`, {
|
|
37
|
+
messageId: record.messageId,
|
|
38
|
+
event: parsedEvent,
|
|
39
|
+
receiveCount: record.attributes?.ApproximateReceiveCount,
|
|
40
|
+
});
|
|
41
|
+
|
|
24
42
|
try {
|
|
25
43
|
const runParams = JSON.parse(record.body);
|
|
26
44
|
this._validateParams(runParams);
|
|
27
45
|
await this._run(runParams, context);
|
|
46
|
+
console.log(`[Worker] record success`, {
|
|
47
|
+
messageId: record.messageId,
|
|
48
|
+
event: runParams?.event,
|
|
49
|
+
});
|
|
28
50
|
} catch (error) {
|
|
29
51
|
if (error.isHaltError) {
|
|
30
52
|
// HaltError means "discard this message, don't retry".
|
|
31
53
|
// Treat as success so SQS deletes it from the queue.
|
|
54
|
+
// Logged explicitly — silent discards made prod debugging
|
|
55
|
+
// extremely hard; keep this visible.
|
|
56
|
+
console.warn(`[Worker] record halted (discarded, no retry)`, {
|
|
57
|
+
messageId: record.messageId,
|
|
58
|
+
event: parsedEvent,
|
|
59
|
+
reason: error.message,
|
|
60
|
+
statusCode: error.statusCode,
|
|
61
|
+
});
|
|
32
62
|
continue;
|
|
33
63
|
}
|
|
34
64
|
console.error(`[Worker] Failed to process record ${record.messageId}:`, error);
|
|
@@ -36,6 +66,12 @@ class Worker {
|
|
|
36
66
|
}
|
|
37
67
|
}
|
|
38
68
|
|
|
69
|
+
if (batchItemFailures.length > 0) {
|
|
70
|
+
console.warn(
|
|
71
|
+
`[Worker] run: returning ${batchItemFailures.length} batchItemFailure(s) of ${records.length}`
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
|
|
39
75
|
return { batchItemFailures };
|
|
40
76
|
}
|
|
41
77
|
|
package/core/create-handler.js
CHANGED
|
@@ -5,6 +5,45 @@
|
|
|
5
5
|
const { initDebugLog, flushDebugLog } = require('../logs');
|
|
6
6
|
const { secretsToEnv } = require('./secrets-to-env');
|
|
7
7
|
|
|
8
|
+
// Best-effort extraction of correlation identifiers from a Lambda event.
|
|
9
|
+
// For SQS: pulls messageIds + parsed event/processId/integrationId from each
|
|
10
|
+
// record body. For HTTP: pulls method+path. Never throws.
|
|
11
|
+
const summarizeLambdaEvent = (event) => {
|
|
12
|
+
if (!event) return {};
|
|
13
|
+
if (Array.isArray(event.Records)) {
|
|
14
|
+
return {
|
|
15
|
+
source: 'sqs',
|
|
16
|
+
records: event.Records.map((r) => {
|
|
17
|
+
let parsed = {};
|
|
18
|
+
try {
|
|
19
|
+
const body = JSON.parse(r.body);
|
|
20
|
+
parsed = {
|
|
21
|
+
event: body?.event,
|
|
22
|
+
processId: body?.data?.processId,
|
|
23
|
+
integrationId: body?.data?.integrationId,
|
|
24
|
+
};
|
|
25
|
+
} catch {
|
|
26
|
+
// ignore unparseable bodies
|
|
27
|
+
}
|
|
28
|
+
return {
|
|
29
|
+
messageId: r.messageId,
|
|
30
|
+
receiveCount: r.attributes?.ApproximateReceiveCount,
|
|
31
|
+
...parsed,
|
|
32
|
+
};
|
|
33
|
+
}),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
if (event.httpMethod || event.requestContext?.http) {
|
|
37
|
+
return {
|
|
38
|
+
source: 'http',
|
|
39
|
+
method:
|
|
40
|
+
event.httpMethod || event.requestContext?.http?.method,
|
|
41
|
+
path: event.path || event.rawPath,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
return { source: 'other' };
|
|
45
|
+
};
|
|
46
|
+
|
|
8
47
|
const createHandler = (optionByName = {}) => {
|
|
9
48
|
const {
|
|
10
49
|
eventName = 'Event',
|
|
@@ -17,7 +56,18 @@ const createHandler = (optionByName = {}) => {
|
|
|
17
56
|
}
|
|
18
57
|
|
|
19
58
|
return async (event, context) => {
|
|
59
|
+
const eventSummary = summarizeLambdaEvent(event);
|
|
60
|
+
|
|
20
61
|
try {
|
|
62
|
+
console.info(
|
|
63
|
+
`[createHandler] ${eventName}: handler entry`,
|
|
64
|
+
{
|
|
65
|
+
eventName,
|
|
66
|
+
awsRequestId: context?.awsRequestId,
|
|
67
|
+
...eventSummary,
|
|
68
|
+
}
|
|
69
|
+
);
|
|
70
|
+
|
|
21
71
|
initDebugLog(eventName, event);
|
|
22
72
|
|
|
23
73
|
const requestMethod = event.httpMethod;
|
|
@@ -62,7 +112,21 @@ const createHandler = (optionByName = {}) => {
|
|
|
62
112
|
// Handle server-to-server responses.
|
|
63
113
|
|
|
64
114
|
// Halt errors are logged but suceed and won't be retried.
|
|
115
|
+
// Log explicitly — silent suppression here previously made stuck
|
|
116
|
+
// messages invisible to observability tooling. Include
|
|
117
|
+
// eventSummary so operators can correlate across concurrent
|
|
118
|
+
// invocations (processId / messageIds / HTTP path).
|
|
65
119
|
if (error.isHaltError === true) {
|
|
120
|
+
console.warn(
|
|
121
|
+
`[createHandler] ${eventName}: halt error suppressed (no retry)`,
|
|
122
|
+
{
|
|
123
|
+
eventName,
|
|
124
|
+
errorName: error.name,
|
|
125
|
+
errorMessage: error.message,
|
|
126
|
+
statusCode: error.statusCode,
|
|
127
|
+
...eventSummary,
|
|
128
|
+
}
|
|
129
|
+
);
|
|
66
130
|
return;
|
|
67
131
|
}
|
|
68
132
|
|
|
@@ -42,6 +42,17 @@ const CORE_ENCRYPTION_SCHEMA = {
|
|
|
42
42
|
|
|
43
43
|
let customSchema = {};
|
|
44
44
|
|
|
45
|
+
/**
|
|
46
|
+
* Per-model write-side opt-out: fields registered here are NOT encrypted on
|
|
47
|
+
* write, but ARE still decrypted on read so legacy encrypted rows continue to
|
|
48
|
+
* deserialize. Lets apps migrate a model from encrypted to plain JSON without
|
|
49
|
+
* a data migration — touched rows naturally rewrite as plain on the next save,
|
|
50
|
+
* untouched rows stay encrypted-but-readable forever.
|
|
51
|
+
*
|
|
52
|
+
* Shape: `{ ModelName: ['field.path', ...] }`
|
|
53
|
+
*/
|
|
54
|
+
let encryptionOptOut = {};
|
|
55
|
+
|
|
45
56
|
/**
|
|
46
57
|
* Validates a custom encryption schema
|
|
47
58
|
* @returns {{valid: boolean, errors: string[]}}
|
|
@@ -218,6 +229,14 @@ function loadCustomEncryptionSchema() {
|
|
|
218
229
|
registerCustomSchema(customSchema);
|
|
219
230
|
}
|
|
220
231
|
|
|
232
|
+
// Load app-level encryption opt-out — apps can declare fields they
|
|
233
|
+
// don't want encrypted on write (decryption on read still works,
|
|
234
|
+
// so legacy data remains readable).
|
|
235
|
+
const disable = appDefinition.encryption?.disable;
|
|
236
|
+
if (disable && Object.keys(disable).length > 0) {
|
|
237
|
+
registerEncryptionOptOut(disable);
|
|
238
|
+
}
|
|
239
|
+
|
|
221
240
|
// Load module-level encryption schemas from integrations
|
|
222
241
|
const integrations = appDefinition.integrations;
|
|
223
242
|
if (integrations && Array.isArray(integrations)) {
|
|
@@ -240,6 +259,115 @@ function getEncryptedFields(modelName) {
|
|
|
240
259
|
return [...new Set(allFields)];
|
|
241
260
|
}
|
|
242
261
|
|
|
262
|
+
/**
|
|
263
|
+
* Validates an encryption opt-out config.
|
|
264
|
+
*
|
|
265
|
+
* Unlike custom schema validation, opt-out IS allowed to target paths that
|
|
266
|
+
* already live in CORE_ENCRYPTION_SCHEMA — that's the entire point.
|
|
267
|
+
*
|
|
268
|
+
* @param {Object} optOut - Map of `{ ModelName: ['field.path', ...] }`
|
|
269
|
+
* @returns {{valid: boolean, errors: string[]}}
|
|
270
|
+
*/
|
|
271
|
+
function validateOptOut(optOut) {
|
|
272
|
+
const errors = [];
|
|
273
|
+
|
|
274
|
+
if (!optOut || typeof optOut !== 'object') {
|
|
275
|
+
errors.push('Encryption opt-out must be an object');
|
|
276
|
+
return { valid: false, errors };
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
for (const [modelName, fields] of Object.entries(optOut)) {
|
|
280
|
+
if (typeof modelName !== 'string' || !modelName) {
|
|
281
|
+
errors.push(`Invalid model name in opt-out: ${modelName}`);
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
if (!Array.isArray(fields)) {
|
|
286
|
+
errors.push(
|
|
287
|
+
`Model "${modelName}" opt-out must be an array of field paths`
|
|
288
|
+
);
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
for (const fieldPath of fields) {
|
|
293
|
+
if (typeof fieldPath !== 'string' || !fieldPath) {
|
|
294
|
+
errors.push(
|
|
295
|
+
`Model "${modelName}" has invalid opt-out field path: ${fieldPath}`
|
|
296
|
+
);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
return { valid: errors.length === 0, errors };
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Registers an encryption opt-out config. Listed fields will be skipped during
|
|
306
|
+
* encryption on write while still being eligible for decryption on read (so
|
|
307
|
+
* legacy encrypted rows still deserialize correctly).
|
|
308
|
+
*
|
|
309
|
+
* Intended call site: `appDefinition.encryption.disable` via
|
|
310
|
+
* `loadCustomEncryptionSchema`.
|
|
311
|
+
*
|
|
312
|
+
* @param {Object} optOut - Map of `{ ModelName: ['field.path', ...] }`
|
|
313
|
+
* @throws {Error} If opt-out validation fails
|
|
314
|
+
*/
|
|
315
|
+
function registerEncryptionOptOut(optOut) {
|
|
316
|
+
if (!optOut || Object.keys(optOut).length === 0) {
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
const validation = validateOptOut(optOut);
|
|
321
|
+
if (!validation.valid) {
|
|
322
|
+
throw new Error(
|
|
323
|
+
`Invalid encryption opt-out:\n- ${validation.errors.join('\n- ')}`
|
|
324
|
+
);
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
encryptionOptOut = { ...optOut };
|
|
328
|
+
logger.info(
|
|
329
|
+
`Registered encryption opt-out for models: ${Object.keys(
|
|
330
|
+
encryptionOptOut
|
|
331
|
+
).join(', ')}`
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* Returns the field paths that should be encrypted when writing the given
|
|
337
|
+
* model. This is `getEncryptedFields` minus any paths the app has opted out
|
|
338
|
+
* of via `registerEncryptionOptOut`.
|
|
339
|
+
*
|
|
340
|
+
* Use this in the encrypt-on-write path of the FieldEncryptionService.
|
|
341
|
+
*/
|
|
342
|
+
function getFieldsToEncryptOnWrite(modelName) {
|
|
343
|
+
const allFields = getEncryptedFields(modelName);
|
|
344
|
+
const optedOut = new Set(encryptionOptOut[modelName] || []);
|
|
345
|
+
if (optedOut.size === 0) return allFields;
|
|
346
|
+
return allFields.filter((path) => !optedOut.has(path));
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Returns the field paths that should be checked for decryption when reading
|
|
351
|
+
* the given model. Always includes opted-out paths so legacy encrypted rows
|
|
352
|
+
* remain readable after an app opts a field out.
|
|
353
|
+
*
|
|
354
|
+
* `FieldEncryptionService._isEncrypted` already short-circuits for plain JSON
|
|
355
|
+
* values, so listing more fields than necessary here is harmless.
|
|
356
|
+
*
|
|
357
|
+
* Use this in the decrypt-on-read path of the FieldEncryptionService.
|
|
358
|
+
*/
|
|
359
|
+
function getFieldsToDecryptOnRead(modelName) {
|
|
360
|
+
return getEncryptedFields(modelName);
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Clears any registered encryption opt-outs. Test-helper; not intended for
|
|
365
|
+
* runtime use.
|
|
366
|
+
*/
|
|
367
|
+
function resetEncryptionOptOut() {
|
|
368
|
+
encryptionOptOut = {};
|
|
369
|
+
}
|
|
370
|
+
|
|
243
371
|
function hasEncryptedFields(modelName) {
|
|
244
372
|
return getEncryptedFields(modelName).length > 0;
|
|
245
373
|
}
|
|
@@ -257,12 +385,17 @@ function resetCustomSchema() {
|
|
|
257
385
|
module.exports = {
|
|
258
386
|
CORE_ENCRYPTION_SCHEMA,
|
|
259
387
|
getEncryptedFields,
|
|
388
|
+
getFieldsToEncryptOnWrite,
|
|
389
|
+
getFieldsToDecryptOnRead,
|
|
260
390
|
hasEncryptedFields,
|
|
261
391
|
getEncryptedModels,
|
|
262
392
|
registerCustomSchema,
|
|
393
|
+
registerEncryptionOptOut,
|
|
263
394
|
loadCustomEncryptionSchema,
|
|
264
395
|
loadModuleEncryptionSchemas,
|
|
265
396
|
extractCredentialFieldsFromModules,
|
|
266
397
|
validateCustomSchema,
|
|
398
|
+
validateOptOut,
|
|
267
399
|
resetCustomSchema,
|
|
400
|
+
resetEncryptionOptOut,
|
|
268
401
|
};
|
|
@@ -17,12 +17,40 @@ class FieldEncryptionService {
|
|
|
17
17
|
this.schema = schema;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
+
/**
|
|
21
|
+
* Resolve the field paths to encrypt on write. Prefers
|
|
22
|
+
* `schema.getFieldsToEncryptOnWrite` (which respects app opt-outs); falls
|
|
23
|
+
* back to `schema.getEncryptedFields` for backwards compatibility with
|
|
24
|
+
* older schema adapters.
|
|
25
|
+
* @private
|
|
26
|
+
*/
|
|
27
|
+
_getWriteFields(modelName) {
|
|
28
|
+
if (typeof this.schema.getFieldsToEncryptOnWrite === 'function') {
|
|
29
|
+
return this.schema.getFieldsToEncryptOnWrite(modelName);
|
|
30
|
+
}
|
|
31
|
+
return this.schema.getEncryptedFields(modelName);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Resolve the field paths to attempt decryption on read. Prefers
|
|
36
|
+
* `schema.getFieldsToDecryptOnRead` (which IGNORES opt-outs so legacy
|
|
37
|
+
* encrypted rows still deserialize); falls back to
|
|
38
|
+
* `schema.getEncryptedFields` for backwards compatibility.
|
|
39
|
+
* @private
|
|
40
|
+
*/
|
|
41
|
+
_getReadFields(modelName) {
|
|
42
|
+
if (typeof this.schema.getFieldsToDecryptOnRead === 'function') {
|
|
43
|
+
return this.schema.getFieldsToDecryptOnRead(modelName);
|
|
44
|
+
}
|
|
45
|
+
return this.schema.getEncryptedFields(modelName);
|
|
46
|
+
}
|
|
47
|
+
|
|
20
48
|
async encryptFields(modelName, document) {
|
|
21
49
|
if (!document || typeof document !== 'object') {
|
|
22
50
|
return document;
|
|
23
51
|
}
|
|
24
52
|
|
|
25
|
-
const fields = this.
|
|
53
|
+
const fields = this._getWriteFields(modelName);
|
|
26
54
|
if (fields.length === 0) {
|
|
27
55
|
return document;
|
|
28
56
|
}
|
|
@@ -58,7 +86,7 @@ class FieldEncryptionService {
|
|
|
58
86
|
return document;
|
|
59
87
|
}
|
|
60
88
|
|
|
61
|
-
const fields = this.
|
|
89
|
+
const fields = this._getReadFields(modelName);
|
|
62
90
|
if (fields.length === 0) {
|
|
63
91
|
return document;
|
|
64
92
|
}
|
|
@@ -3,7 +3,11 @@
|
|
|
3
3
|
* Intercepts Prisma queries to encrypt on write and decrypt on read.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
const {
|
|
6
|
+
const {
|
|
7
|
+
getEncryptedFields,
|
|
8
|
+
getFieldsToEncryptOnWrite,
|
|
9
|
+
getFieldsToDecryptOnRead,
|
|
10
|
+
} = require('./encryption-schema-registry');
|
|
7
11
|
const { FieldEncryptionService } = require('./field-encryption-service');
|
|
8
12
|
|
|
9
13
|
function createEncryptionExtension({ cryptor, enabled = true }) {
|
|
@@ -19,7 +23,11 @@ function createEncryptionExtension({ cryptor, enabled = true }) {
|
|
|
19
23
|
|
|
20
24
|
const encryptionService = new FieldEncryptionService({
|
|
21
25
|
cryptor,
|
|
22
|
-
schema: {
|
|
26
|
+
schema: {
|
|
27
|
+
getEncryptedFields,
|
|
28
|
+
getFieldsToEncryptOnWrite,
|
|
29
|
+
getFieldsToDecryptOnRead,
|
|
30
|
+
},
|
|
23
31
|
});
|
|
24
32
|
|
|
25
33
|
return {
|
|
@@ -141,8 +141,17 @@ const loadIntegrationForProcess = async (processId, integrationClass) => {
|
|
|
141
141
|
};
|
|
142
142
|
|
|
143
143
|
const createQueueWorker = (integrationClass) => {
|
|
144
|
+
const integrationName = integrationClass.Definition.name;
|
|
145
|
+
|
|
144
146
|
class QueueWorker extends Worker {
|
|
145
147
|
async _run(params, context) {
|
|
148
|
+
const logCtx = {
|
|
149
|
+
integration: integrationName,
|
|
150
|
+
event: params.event,
|
|
151
|
+
processId: params.data?.processId,
|
|
152
|
+
integrationId: params.data?.integrationId,
|
|
153
|
+
};
|
|
154
|
+
|
|
146
155
|
try {
|
|
147
156
|
let integrationInstance;
|
|
148
157
|
|
|
@@ -150,29 +159,46 @@ const createQueueWorker = (integrationClass) => {
|
|
|
150
159
|
// then integrationId (for ANY event type that needs hydration),
|
|
151
160
|
// fallback to unhydrated instance
|
|
152
161
|
if (params.data?.processId) {
|
|
162
|
+
console.log(
|
|
163
|
+
`[QueueWorker] hydrating by processId`,
|
|
164
|
+
logCtx
|
|
165
|
+
);
|
|
153
166
|
integrationInstance = await loadIntegrationForProcess(
|
|
154
167
|
params.data.processId,
|
|
155
168
|
integrationClass
|
|
156
169
|
);
|
|
170
|
+
console.log(`[QueueWorker] hydrated`, {
|
|
171
|
+
...logCtx,
|
|
172
|
+
integrationStatus: integrationInstance?.status,
|
|
173
|
+
hydratedIntegrationId: integrationInstance?.id,
|
|
174
|
+
});
|
|
157
175
|
if (['DISABLED', 'ERROR'].includes(integrationInstance?.status)) {
|
|
158
176
|
console.warn(
|
|
159
|
-
`[${
|
|
177
|
+
`[${integrationName}] Integration for process ${params.data.processId} is ${integrationInstance.status}. Discarding ${params.event} message.`
|
|
160
178
|
);
|
|
161
179
|
return;
|
|
162
180
|
}
|
|
163
181
|
} else if (params.data?.integrationId) {
|
|
182
|
+
console.log(
|
|
183
|
+
`[QueueWorker] hydrating by integrationId`,
|
|
184
|
+
logCtx
|
|
185
|
+
);
|
|
164
186
|
integrationInstance = await loadIntegrationForWebhook(
|
|
165
187
|
params.data.integrationId
|
|
166
188
|
);
|
|
167
189
|
if (!integrationInstance) {
|
|
168
190
|
console.warn(
|
|
169
|
-
`[${
|
|
191
|
+
`[${integrationName}] Integration ${params.data.integrationId} no longer exists. Discarding ${params.event} message.`
|
|
170
192
|
);
|
|
171
193
|
return;
|
|
172
194
|
}
|
|
195
|
+
console.log(`[QueueWorker] hydrated`, {
|
|
196
|
+
...logCtx,
|
|
197
|
+
integrationStatus: integrationInstance?.status,
|
|
198
|
+
});
|
|
173
199
|
if (['DISABLED', 'ERROR'].includes(integrationInstance.status)) {
|
|
174
200
|
console.warn(
|
|
175
|
-
`[${
|
|
201
|
+
`[${integrationName}] Integration ${params.data.integrationId} is ${integrationInstance.status}. Discarding ${params.event} message.`
|
|
176
202
|
);
|
|
177
203
|
return;
|
|
178
204
|
}
|
|
@@ -181,6 +207,10 @@ const createQueueWorker = (integrationClass) => {
|
|
|
181
207
|
// There will be cases where we need to use helpers that the api modules can export.
|
|
182
208
|
// Like for HubSpot, the answer is to do a reverse lookup for the integration by the entity external ID (HubSpot Portal ID),
|
|
183
209
|
// and then you'll have the integration ID available to hydrate from.
|
|
210
|
+
console.log(
|
|
211
|
+
`[QueueWorker] no processId/integrationId — running dry instance`,
|
|
212
|
+
logCtx
|
|
213
|
+
);
|
|
184
214
|
integrationInstance = new integrationClass();
|
|
185
215
|
}
|
|
186
216
|
|
|
@@ -188,14 +218,17 @@ const createQueueWorker = (integrationClass) => {
|
|
|
188
218
|
integrationInstance
|
|
189
219
|
);
|
|
190
220
|
|
|
191
|
-
|
|
221
|
+
console.log(`[QueueWorker] dispatching ${params.event}`, logCtx);
|
|
222
|
+
const result = await dispatcher.dispatchJob({
|
|
192
223
|
event: params.event,
|
|
193
224
|
data: params.data,
|
|
194
225
|
context: context,
|
|
195
226
|
});
|
|
227
|
+
console.log(`[QueueWorker] ${params.event} dispatched ok`, logCtx);
|
|
228
|
+
return result;
|
|
196
229
|
} catch (error) {
|
|
197
230
|
console.error(
|
|
198
|
-
`Error in ${params.event} for ${
|
|
231
|
+
`Error in ${params.event} for ${integrationName}:`,
|
|
199
232
|
error
|
|
200
233
|
);
|
|
201
234
|
|
|
@@ -207,7 +240,12 @@ const createQueueWorker = (integrationClass) => {
|
|
|
207
240
|
if (status && status >= 400 && status < 500 && status !== 408 && status !== 429) {
|
|
208
241
|
error.isHaltError = true;
|
|
209
242
|
console.warn(
|
|
210
|
-
`[${
|
|
243
|
+
`[${integrationName}] Permanent ${status} error for ${params.event} — message will be discarded (no retry)`,
|
|
244
|
+
{
|
|
245
|
+
...logCtx,
|
|
246
|
+
errorName: error.name,
|
|
247
|
+
errorMessage: error.message,
|
|
248
|
+
}
|
|
211
249
|
);
|
|
212
250
|
}
|
|
213
251
|
|
|
@@ -507,7 +507,8 @@ function setEntityRoutes(router, authenticateUser, useCases) {
|
|
|
507
507
|
const params = checkRequiredParams(req.query, ['entityType']);
|
|
508
508
|
const module = await getModuleInstanceFromType.execute(
|
|
509
509
|
userId,
|
|
510
|
-
params.entityType
|
|
510
|
+
params.entityType,
|
|
511
|
+
{ state: req.query.state }
|
|
511
512
|
);
|
|
512
513
|
const areRequirementsValid =
|
|
513
514
|
module.validateAuthorizationRequirements();
|
|
@@ -530,13 +531,33 @@ function setEntityRoutes(router, authenticateUser, useCases) {
|
|
|
530
531
|
'data',
|
|
531
532
|
]);
|
|
532
533
|
|
|
533
|
-
const
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
534
|
+
const dataKeys =
|
|
535
|
+
params.data && typeof params.data === 'object'
|
|
536
|
+
? Object.keys(params.data)
|
|
537
|
+
: [];
|
|
538
|
+
console.log(
|
|
539
|
+
`[Frigg] POST /api/authorize userId=${userId} entityType=${params.entityType} dataKeys=${JSON.stringify(dataKeys)}`
|
|
537
540
|
);
|
|
538
541
|
|
|
539
|
-
|
|
542
|
+
try {
|
|
543
|
+
const entityDetails =
|
|
544
|
+
await processAuthorizationCallback.execute(
|
|
545
|
+
userId,
|
|
546
|
+
params.entityType,
|
|
547
|
+
params.data
|
|
548
|
+
);
|
|
549
|
+
|
|
550
|
+
console.log(
|
|
551
|
+
`[Frigg] POST /api/authorize success userId=${userId} entityType=${params.entityType} credentialId=${entityDetails?.credential_id} entityId=${entityDetails?.entity_id}`
|
|
552
|
+
);
|
|
553
|
+
|
|
554
|
+
res.json(entityDetails);
|
|
555
|
+
} catch (err) {
|
|
556
|
+
console.error(
|
|
557
|
+
`[Frigg] POST /api/authorize failed userId=${userId} entityType=${params.entityType} error=${err?.message || err}`
|
|
558
|
+
);
|
|
559
|
+
throw err;
|
|
560
|
+
}
|
|
540
561
|
})
|
|
541
562
|
);
|
|
542
563
|
|
package/modules/module.js
CHANGED
|
@@ -20,8 +20,9 @@ class Module extends Delegate {
|
|
|
20
20
|
* @param {Object} params.definition The definition of the Api Module
|
|
21
21
|
* @param {string} params.userId The user id
|
|
22
22
|
* @param {Object} params.entity The entity record from the database
|
|
23
|
+
* @param {string} [params.state] Optional OAuth state value forwarded to the API client (round-trips through the OAuth provider).
|
|
23
24
|
*/
|
|
24
|
-
constructor({ definition, userId = null, entity: entityObj = null }) {
|
|
25
|
+
constructor({ definition, userId = null, entity: entityObj = null, state = null }) {
|
|
25
26
|
super({ definition, userId, entity: entityObj });
|
|
26
27
|
|
|
27
28
|
this.validateDefinition(definition);
|
|
@@ -46,6 +47,7 @@ class Module extends Delegate {
|
|
|
46
47
|
const apiParams = {
|
|
47
48
|
...this.definition.env,
|
|
48
49
|
delegate: this,
|
|
50
|
+
...(state ? { state } : {}),
|
|
49
51
|
...(this.credential?.data
|
|
50
52
|
? this.apiParamsFromCredential(this.credential.data)
|
|
51
53
|
: {}), // Handle case when credential is undefined
|
|
@@ -13,8 +13,10 @@ class GetModuleInstanceFromType {
|
|
|
13
13
|
* Retrieve a Module instance for a given user and entity/module type.
|
|
14
14
|
* @param {string} userId
|
|
15
15
|
* @param {string} type – human-readable module/entity type (e.g. "Hubspot")
|
|
16
|
+
* @param {Object} [options]
|
|
17
|
+
* @param {string} [options.state] – optional OAuth state value to be forwarded to the API client (round-trips through the OAuth provider).
|
|
16
18
|
*/
|
|
17
|
-
async execute(userId, type) {
|
|
19
|
+
async execute(userId, type, options = {}) {
|
|
18
20
|
const moduleDefinition = this.moduleDefinitions.find(
|
|
19
21
|
(def) => def.getName() === type
|
|
20
22
|
);
|
|
@@ -24,6 +26,7 @@ class GetModuleInstanceFromType {
|
|
|
24
26
|
return new Module({
|
|
25
27
|
userId,
|
|
26
28
|
definition: moduleDefinition,
|
|
29
|
+
state: options.state,
|
|
27
30
|
});
|
|
28
31
|
}
|
|
29
32
|
}
|
|
@@ -28,6 +28,11 @@ class ProcessAuthorizationCallback {
|
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
async execute(userId, entityType, params) {
|
|
31
|
+
const hasCode = Boolean(params && params.code);
|
|
32
|
+
console.log(
|
|
33
|
+
`[Frigg] processAuthorizationCallback start userId=${userId} entityType=${entityType} hasCode=${hasCode}`
|
|
34
|
+
);
|
|
35
|
+
|
|
31
36
|
const moduleDefinition = this.moduleDefinitions.find((def) => {
|
|
32
37
|
return entityType === def.moduleName;
|
|
33
38
|
});
|
|
@@ -38,12 +43,29 @@ class ProcessAuthorizationCallback {
|
|
|
38
43
|
);
|
|
39
44
|
}
|
|
40
45
|
|
|
41
|
-
//
|
|
42
|
-
|
|
46
|
+
// Bootstrap the Module with the existing entity (if any) so the API
|
|
47
|
+
// requester is preloaded with prior tokens. This enables a refresh
|
|
48
|
+
// fallback when callers lack a fresh OAuth code, and lets us match a
|
|
49
|
+
// re-auth back to the existing credential record by id.
|
|
50
|
+
const existingEntities =
|
|
51
|
+
await this.moduleRepository.findEntitiesByUserIdAndModuleName(
|
|
52
|
+
userId,
|
|
53
|
+
entityType
|
|
54
|
+
);
|
|
55
|
+
const existingEntity =
|
|
56
|
+
existingEntities && existingEntities.length > 0
|
|
57
|
+
? existingEntities[0]
|
|
58
|
+
: null;
|
|
59
|
+
|
|
60
|
+
if (existingEntity) {
|
|
61
|
+
console.log(
|
|
62
|
+
`[Frigg] processAuthorizationCallback found existing entity id=${existingEntity.id} credentialId=${existingEntity.credential?.id}`
|
|
63
|
+
);
|
|
64
|
+
}
|
|
43
65
|
|
|
44
66
|
const module = new Module({
|
|
45
67
|
userId,
|
|
46
|
-
entity,
|
|
68
|
+
entity: existingEntity,
|
|
47
69
|
definition: moduleDefinition,
|
|
48
70
|
});
|
|
49
71
|
|
|
@@ -53,6 +75,16 @@ class ProcessAuthorizationCallback {
|
|
|
53
75
|
module.api,
|
|
54
76
|
params
|
|
55
77
|
);
|
|
78
|
+
console.log(
|
|
79
|
+
`[Frigg] processAuthorizationCallback OAuth getToken complete userId=${userId} entityType=${entityType}`
|
|
80
|
+
);
|
|
81
|
+
// Belt-and-suspenders: persist tokens explicitly here rather than
|
|
82
|
+
// relying solely on the DLGT_TOKEN_UPDATE notification chain
|
|
83
|
+
// inside setTokens. The notification path remains in place but
|
|
84
|
+
// has been observed to no-op silently in some prod paths,
|
|
85
|
+
// leaving newly-issued tokens unsaved while the user-visible
|
|
86
|
+
// OAuth flow appears to succeed.
|
|
87
|
+
await this.onTokenUpdate(module, moduleDefinition, userId);
|
|
56
88
|
} else {
|
|
57
89
|
tokenResponse =
|
|
58
90
|
await moduleDefinition.requiredAuthMethods.setAuthParams(
|
|
@@ -62,6 +94,10 @@ class ProcessAuthorizationCallback {
|
|
|
62
94
|
await this.onTokenUpdate(module, moduleDefinition, userId);
|
|
63
95
|
}
|
|
64
96
|
|
|
97
|
+
console.log(
|
|
98
|
+
`[Frigg] processAuthorizationCallback credential persisted credentialId=${module.credential?.id} authIsValid=${module.credential?.authIsValid}`
|
|
99
|
+
);
|
|
100
|
+
|
|
65
101
|
const authRes = await module.testAuth();
|
|
66
102
|
if (!authRes) {
|
|
67
103
|
throw new Error('Authorization failed');
|
|
@@ -90,7 +126,12 @@ class ProcessAuthorizationCallback {
|
|
|
90
126
|
// credential + entity are already persisted. Operators can recover
|
|
91
127
|
// stuck integrations manually.
|
|
92
128
|
try {
|
|
93
|
-
await this.restoreIntegrationsForEntity(
|
|
129
|
+
const restoredCount = await this.restoreIntegrationsForEntity(
|
|
130
|
+
persistedEntity.id
|
|
131
|
+
);
|
|
132
|
+
console.log(
|
|
133
|
+
`[Frigg] processAuthorizationCallback restored ${restoredCount} integration(s) for entityId=${persistedEntity.id}`
|
|
134
|
+
);
|
|
94
135
|
} catch (err) {
|
|
95
136
|
console.error(
|
|
96
137
|
`[Frigg] Failed to restore integrations for entity ${persistedEntity.id} after successful re-auth — manual intervention may be needed`,
|
|
@@ -106,11 +147,12 @@ class ProcessAuthorizationCallback {
|
|
|
106
147
|
}
|
|
107
148
|
|
|
108
149
|
async restoreIntegrationsForEntity(entityId) {
|
|
109
|
-
if (!this.integrationRepository) return;
|
|
150
|
+
if (!this.integrationRepository) return 0;
|
|
110
151
|
const integrations =
|
|
111
152
|
await this.integrationRepository.findIntegrationsByEntityId(
|
|
112
153
|
entityId
|
|
113
154
|
);
|
|
155
|
+
let restored = 0;
|
|
114
156
|
for (const integration of integrations) {
|
|
115
157
|
if (STATUSES_RESET_ON_REAUTH.includes(integration.status)) {
|
|
116
158
|
console.log(
|
|
@@ -120,8 +162,10 @@ class ProcessAuthorizationCallback {
|
|
|
120
162
|
integration.id,
|
|
121
163
|
'ENABLED'
|
|
122
164
|
);
|
|
165
|
+
restored++;
|
|
123
166
|
}
|
|
124
167
|
}
|
|
168
|
+
return restored;
|
|
125
169
|
}
|
|
126
170
|
|
|
127
171
|
async onTokenUpdate(module, moduleDefinition, userId) {
|
|
@@ -162,6 +206,28 @@ class ProcessAuthorizationCallback {
|
|
|
162
206
|
});
|
|
163
207
|
|
|
164
208
|
if (existingEntity) {
|
|
209
|
+
// Repoint the entity's credentialId when re-auth produced a
|
|
210
|
+
// different credential than the one currently linked. This
|
|
211
|
+
// happens when the user re-authenticates against a different
|
|
212
|
+
// workspace/account of the same provider — `upsertCredential`
|
|
213
|
+
// matches/creates a credential keyed by externalId, but
|
|
214
|
+
// findEntity matches the entity by its own externalId, leaving
|
|
215
|
+
// the entity still linked to the prior workspace's credential
|
|
216
|
+
// unless we explicitly update the link.
|
|
217
|
+
const existingCredentialId = existingEntity.credential?.id;
|
|
218
|
+
if (
|
|
219
|
+
credentialId &&
|
|
220
|
+
String(existingCredentialId) !== String(credentialId)
|
|
221
|
+
) {
|
|
222
|
+
console.log(
|
|
223
|
+
`[Frigg] Repointing entity ${existingEntity.id} credentialId ${existingCredentialId} -> ${credentialId} after re-auth`
|
|
224
|
+
);
|
|
225
|
+
const updated = await this.moduleRepository.updateEntity(
|
|
226
|
+
existingEntity.id,
|
|
227
|
+
{ credential: credentialId }
|
|
228
|
+
);
|
|
229
|
+
if (updated) return updated;
|
|
230
|
+
}
|
|
165
231
|
return existingEntity;
|
|
166
232
|
}
|
|
167
233
|
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@friggframework/core",
|
|
3
3
|
"prettier": "@friggframework/prettier-config",
|
|
4
|
-
"version": "2.0.0--canary.
|
|
4
|
+
"version": "2.0.0--canary.585.8f515e1.0",
|
|
5
5
|
"dependencies": {
|
|
6
6
|
"@aws-sdk/client-apigatewaymanagementapi": "^3.588.0",
|
|
7
7
|
"@aws-sdk/client-kms": "^3.588.0",
|
|
@@ -38,9 +38,9 @@
|
|
|
38
38
|
}
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
|
-
"@friggframework/eslint-config": "2.0.0--canary.
|
|
42
|
-
"@friggframework/prettier-config": "2.0.0--canary.
|
|
43
|
-
"@friggframework/test": "2.0.0--canary.
|
|
41
|
+
"@friggframework/eslint-config": "2.0.0--canary.585.8f515e1.0",
|
|
42
|
+
"@friggframework/prettier-config": "2.0.0--canary.585.8f515e1.0",
|
|
43
|
+
"@friggframework/test": "2.0.0--canary.585.8f515e1.0",
|
|
44
44
|
"@prisma/client": "^6.17.0",
|
|
45
45
|
"@types/lodash": "4.17.15",
|
|
46
46
|
"@typescript-eslint/eslint-plugin": "^8.0.0",
|
|
@@ -80,5 +80,5 @@
|
|
|
80
80
|
"publishConfig": {
|
|
81
81
|
"access": "public"
|
|
82
82
|
},
|
|
83
|
-
"gitHead": "
|
|
83
|
+
"gitHead": "8f515e128f7c2f34f69acb14476a9189879417ec"
|
|
84
84
|
}
|
package/queues/queuer-util.js
CHANGED
|
@@ -18,13 +18,88 @@ const awsConfigOptions = () => {
|
|
|
18
18
|
|
|
19
19
|
const sqs = new SQSClient(awsConfigOptions());
|
|
20
20
|
|
|
21
|
+
// Best-effort extraction of the logical event/processId/integrationId from a
|
|
22
|
+
// JSON message body. Used only for log correlation — never throws.
|
|
23
|
+
const summarizeMessageBody = (bodyStr) => {
|
|
24
|
+
try {
|
|
25
|
+
const parsed = JSON.parse(bodyStr);
|
|
26
|
+
return {
|
|
27
|
+
event: parsed?.event,
|
|
28
|
+
processId: parsed?.data?.processId,
|
|
29
|
+
integrationId: parsed?.data?.integrationId,
|
|
30
|
+
};
|
|
31
|
+
} catch {
|
|
32
|
+
return {};
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
// Inspect SendMessageBatchResult for partial failures and log them.
|
|
37
|
+
// AWS SendMessageBatch can succeed at the HTTP level while individual entries
|
|
38
|
+
// are rejected (KMS errors, per-entry throttling, service errors). Callers that
|
|
39
|
+
// don't inspect result.Failed silently lose those messages. This logs the
|
|
40
|
+
// details — including the logical event/processId of the failed entry — so
|
|
41
|
+
// the loss is visible and correlatable in CloudWatch.
|
|
42
|
+
const inspectBatchResult = (result, queueUrl, buffer) => {
|
|
43
|
+
const bufferSize = buffer.length;
|
|
44
|
+
const failedCount = result?.Failed?.length ?? 0;
|
|
45
|
+
const successCount = result?.Successful?.length ?? 0;
|
|
46
|
+
|
|
47
|
+
// Index buffer by Id so we can attach event/processId to failures.
|
|
48
|
+
const bufferById = new Map(buffer.map((b) => [b.Id, b]));
|
|
49
|
+
|
|
50
|
+
if (failedCount > 0) {
|
|
51
|
+
console.error(
|
|
52
|
+
`[QueuerUtil] SendMessageBatch partial failure: ${failedCount}/${bufferSize} failed`,
|
|
53
|
+
{
|
|
54
|
+
queueUrl,
|
|
55
|
+
bufferSize,
|
|
56
|
+
successCount,
|
|
57
|
+
failedCount,
|
|
58
|
+
failed: result.Failed.map((f) => {
|
|
59
|
+
const bufEntry = bufferById.get(f.Id);
|
|
60
|
+
const summary = bufEntry
|
|
61
|
+
? summarizeMessageBody(bufEntry.MessageBody)
|
|
62
|
+
: {};
|
|
63
|
+
return {
|
|
64
|
+
Id: f.Id,
|
|
65
|
+
Code: f.Code,
|
|
66
|
+
SenderFault: f.SenderFault,
|
|
67
|
+
Message: f.Message,
|
|
68
|
+
...summary,
|
|
69
|
+
};
|
|
70
|
+
}),
|
|
71
|
+
}
|
|
72
|
+
);
|
|
73
|
+
} else if (successCount > 0) {
|
|
74
|
+
// Include a compact per-entry summary so operators can correlate
|
|
75
|
+
// "which send contained which logical message" during incident triage.
|
|
76
|
+
const entries = result.Successful.map((s) => {
|
|
77
|
+
const bufEntry = bufferById.get(s.Id);
|
|
78
|
+
const summary = bufEntry
|
|
79
|
+
? summarizeMessageBody(bufEntry.MessageBody)
|
|
80
|
+
: {};
|
|
81
|
+
return { MessageId: s.MessageId, ...summary };
|
|
82
|
+
});
|
|
83
|
+
console.log(
|
|
84
|
+
`[QueuerUtil] SendMessageBatch ok: ${successCount}/${bufferSize} to ${queueUrl}`,
|
|
85
|
+
{ entries }
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return result;
|
|
90
|
+
};
|
|
91
|
+
|
|
21
92
|
const QueuerUtil = {
|
|
22
93
|
send: async (message, queueUrl) => {
|
|
23
94
|
const command = new SendMessageCommand({
|
|
24
95
|
MessageBody: JSON.stringify(message),
|
|
25
96
|
QueueUrl: queueUrl,
|
|
26
97
|
});
|
|
27
|
-
|
|
98
|
+
const result = await sqs.send(command);
|
|
99
|
+
console.log(
|
|
100
|
+
`[QueuerUtil] SendMessage ok: MessageId=${result?.MessageId} to ${queueUrl}`
|
|
101
|
+
);
|
|
102
|
+
return result;
|
|
28
103
|
},
|
|
29
104
|
|
|
30
105
|
batchSend: async (entries = [], queueUrl) => {
|
|
@@ -42,7 +117,8 @@ const QueuerUtil = {
|
|
|
42
117
|
Entries: buffer,
|
|
43
118
|
QueueUrl: queueUrl,
|
|
44
119
|
});
|
|
45
|
-
await sqs.send(command);
|
|
120
|
+
const result = await sqs.send(command);
|
|
121
|
+
inspectBatchResult(result, queueUrl, buffer);
|
|
46
122
|
// Purge the buffer
|
|
47
123
|
buffer.splice(0, buffer.length);
|
|
48
124
|
}
|
|
@@ -54,7 +130,8 @@ const QueuerUtil = {
|
|
|
54
130
|
Entries: buffer,
|
|
55
131
|
QueueUrl: queueUrl,
|
|
56
132
|
});
|
|
57
|
-
|
|
133
|
+
const result = await sqs.send(command);
|
|
134
|
+
return inspectBatchResult(result, queueUrl, buffer);
|
|
58
135
|
}
|
|
59
136
|
|
|
60
137
|
// If we're exact... just return an empty object for now
|