@friggframework/core 2.0.0--canary.658.164d92b.0 → 2.0.0--canary.657.6a81b60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +76 -52
- package/core/CLAUDE.md +25 -0
- package/core/Worker.js +304 -9
- package/handlers/backend-utils.js +28 -0
- package/integrations/integration-base.js +0 -16
- package/integrations/repositories/integration-repository-documentdb.js +10 -23
- package/integrations/repositories/integration-repository-interface.js +7 -21
- package/integrations/repositories/integration-repository-mongo.js +16 -29
- package/integrations/repositories/integration-repository-postgres.js +16 -30
- package/integrations/tests/doubles/test-integration-repository.js +2 -15
- package/integrations/use-cases/clear-rate-limit-wait.js +23 -0
- package/integrations/use-cases/index.js +5 -1
- package/integrations/use-cases/record-rate-limit-wait.js +35 -0
- package/integrations/use-cases/update-integration-messages.js +9 -28
- package/modules/module.js +0 -27
- package/modules/requester/oauth-2.js +7 -0
- package/modules/requester/requester.js +0 -28
- package/package.json +7 -5
- package/queues/queue-deferral.js +69 -0
- package/queues/queue-delivery.js +1 -0
- package/queues/queuer-util.js +1 -1
- package/types/core/index.d.ts +15 -0
- package/types/integrations/index.d.ts +4 -20
- package/types/module-plugin/index.d.ts +1 -5
- package/integrations/repositories/message-item-shared.js +0 -4
- package/integrations/tests/doubles/prisma-message-row.js +0 -24
- package/integrations/use-cases/record-rate-limit-message.js +0 -110
package/CLAUDE.md
CHANGED
|
@@ -483,21 +483,22 @@ handlers (`{ req, res, next }`) and `this.on` events do not.
|
|
|
483
483
|
|
|
484
484
|
`isLastAttempt` is `false` when either count is unknown: a local or non-SQS
|
|
485
485
|
invocation, or a queue whose redrive policy the stack does not own
|
|
486
|
-
(`ownership.queue: 'external'`). The value
|
|
487
|
-
|
|
488
|
-
|
|
486
|
+
(`ownership.queue: 'external'`). The value does not change what core does: a
|
|
487
|
+
retryable error returns the message to SQS, a rate-limited error puts it back
|
|
488
|
+
to run at `retryAt` (see **Rate-limit deferral** below), and a halt error (4xx
|
|
489
|
+
except 408, 429 and errors with `isRateLimited`) discards it.
|
|
489
490
|
|
|
490
|
-
Use it to end a run or count lost work on the final try: when
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
message
|
|
491
|
+
Use it to end a run or count lost work on the final try. Check it first: when
|
|
492
|
+
it is `true`, mark the run failed or count the message's records as failed, and
|
|
493
|
+
return. Otherwise rethrow. Do not end the run and then rethrow: core can still
|
|
494
|
+
send a `RateLimitError` again, so the message would run again on an ended run.
|
|
494
495
|
|
|
495
496
|
```javascript
|
|
496
497
|
async processBatch({ data, delivery }) {
|
|
497
498
|
try {
|
|
498
499
|
await this.syncPage(data);
|
|
499
500
|
} catch (error) {
|
|
500
|
-
if (delivery?.isLastAttempt)
|
|
501
|
+
if (delivery?.isLastAttempt) return this.failRun(data.processId, error);
|
|
501
502
|
throw error;
|
|
502
503
|
}
|
|
503
504
|
}
|
|
@@ -508,6 +509,71 @@ The single source of the max receive count is
|
|
|
508
509
|
devtools integration builder uses it for the queue's `RedrivePolicy` and sets
|
|
509
510
|
it as `FRIGG_QUEUE_MAX_RECEIVE_COUNT` on the queue worker function.
|
|
510
511
|
|
|
512
|
+
**Rate-limit deferral** (ADR-049): when a queue handler throws an error with
|
|
513
|
+
`isRateLimited` and a `retryAt` (a `RateLimitError` from the Requester, or one a
|
|
514
|
+
module throws), `Worker.run` does not give the message back to SQS for a 30
|
|
515
|
+
minute redelivery. It puts the message back so it runs at `retryAt`:
|
|
516
|
+
|
|
517
|
+
| Wait | What the worker does | Record |
|
|
518
|
+
|---|---|---|
|
|
519
|
+
| Up to 900 s | Sends the same body again with `DelaySeconds`, then acknowledges the old message | handled |
|
|
520
|
+
| Over 900 s, and `SCHEDULER_ROLE_ARN` is set | Creates a one-time EventBridge schedule at `retryAt` that sends the body to the queue, then acknowledges | handled |
|
|
521
|
+
| Over 900 s, no scheduler | Extends the message's visibility timeout to `retryAt` (12 h less 20 minutes at most). This uses one receive | reported failed |
|
|
522
|
+
|
|
523
|
+
- The new message starts at receive count 1. A deferred body carries
|
|
524
|
+
`_frigg: { deferrals, firstDeferredAt }`; `_frigg` is reserved and the handler
|
|
525
|
+
never sees it.
|
|
526
|
+
- The visibility tier sets a timeout of at most 12 h less 20 minutes
|
|
527
|
+
(42,000 s). SQS counts its 12 h maximum from the receive, and an extension
|
|
528
|
+
does not reset it. The margin covers the longest Lambda run (900 s) plus the
|
|
529
|
+
longest batching window (300 s).
|
|
530
|
+
- Two caps bound it: `FRIGG_QUEUE_MAX_DEFERRALS` (default 30) and
|
|
531
|
+
`FRIGG_QUEUE_MAX_DEFERRED_MS` (default 26 h, from the first deferral to the
|
|
532
|
+
retry time). The defaults cover one full day of hourly waits, for a module
|
|
533
|
+
that probes a daily limit every hour. Past a cap, with no `retryAt`, with no
|
|
534
|
+
event source ARN or on a FIFO queue, the record fails like any retryable
|
|
535
|
+
error and `delivery` applies.
|
|
536
|
+
- The mock scheduler is never used for a deferral: a schedule kept in memory
|
|
537
|
+
would be lost with the acknowledged message.
|
|
538
|
+
- With a `processId` in the message, the worker writes
|
|
539
|
+
`Process.context.rateLimit = { status: 'WAITING' | 'EXHAUSTED', mechanism,
|
|
540
|
+
retryAt, reason, module, deferrals, updatedAt }`. `EXHAUSTED` means a cap ended
|
|
541
|
+
the deferrals, or a visibility change came on the last delivery. The value is
|
|
542
|
+
one per process, so it holds the latest known wait: a new wait replaces it only
|
|
543
|
+
when its `retryAt` is later, a `WAITING` value never hides an `EXHAUSTED` one,
|
|
544
|
+
and after a deferred or redelivered message ran the worker sets it to `null`
|
|
545
|
+
only when it is `WAITING` and its `retryAt` is past or within 5 s.
|
|
546
|
+
- Delivery stays at least once. The new message is sent before the old one is
|
|
547
|
+
acknowledged, so a crash between the two can run the work twice.
|
|
548
|
+
- Handlers check `delivery.isLastAttempt` first, then rethrow. The delay and
|
|
549
|
+
schedule tiers send a new message, so they never reach the last attempt. The
|
|
550
|
+
visibility tier and a cap use a receive: on the last one the worker writes an
|
|
551
|
+
ERROR (`frigg.worker.record_lost_rate_limited`), and the handler has this one
|
|
552
|
+
chance to end the run.
|
|
553
|
+
|
|
554
|
+
```javascript
|
|
555
|
+
async processBatch({ data, delivery }) {
|
|
556
|
+
try {
|
|
557
|
+
await this.syncPage(data);
|
|
558
|
+
} catch (error) {
|
|
559
|
+
if (delivery?.isLastAttempt) return this.failRun(data.processId, error);
|
|
560
|
+
throw error; // a RateLimitError too: core puts the message back at retryAt
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
```
|
|
564
|
+
|
|
565
|
+
Records: a rate-limited SQS record gets one log record, by its outcome.
|
|
566
|
+
|
|
567
|
+
| Outcome | Log record |
|
|
568
|
+
|---|---|
|
|
569
|
+
| Acknowledged | WARN `frigg.worker.record_deferred` (`mechanism` `delay` or `schedule`) |
|
|
570
|
+
| Not acknowledged, not the last attempt | One WARN: `record_deferral_capped`, `record_visibility_extended`, or `record_failed` with `deferral.skipped` |
|
|
571
|
+
| Not acknowledged, last attempt | One ERROR, `record_lost_rate_limited`, with the details, and no WARN |
|
|
572
|
+
|
|
573
|
+
A failing run state hook adds `rate_limit_state_failed`. A queue in a stack
|
|
574
|
+
that does not own it (`ownership.queue: 'external'`) needs its own
|
|
575
|
+
`sqs:ChangeMessageVisibility` and scheduler access.
|
|
576
|
+
|
|
511
577
|
### 8. Error Handling (`/errors`)
|
|
512
578
|
|
|
513
579
|
**Purpose**: Standardized error types with proper HTTP semantics.
|
|
@@ -530,50 +596,6 @@ if (!userId) {
|
|
|
530
596
|
}
|
|
531
597
|
```
|
|
532
598
|
|
|
533
|
-
**Rate-limit message** (ADR-049): when a wait is longer than the in-process cap
|
|
534
|
-
(`maxInProcessWaitMs`, default 5 min), the Requester tells its delegate
|
|
535
|
-
(`RATE_LIMITED`, with the `RateLimitError`) and then throws. A shorter wait
|
|
536
|
-
that throws sends no notification. The notification takes the path of
|
|
537
|
-
`CREDENTIAL_INVALIDATED`: Requester, `Module`, `IntegrationBase`.
|
|
538
|
-
|
|
539
|
-
- The `Module` sends `{ moduleName, reason, retryAt, policy, statusCode, links }`.
|
|
540
|
-
`links` is `static rateLimit.userHints[reason].links`, or `[]`. The message,
|
|
541
|
-
url, body and headers of the error are never sent: the message reaches end
|
|
542
|
-
users.
|
|
543
|
-
- `IntegrationBase` writes one `WARN` (`integration.<name>.rate_limited`), and
|
|
544
|
-
`RecordRateLimitMessage` appends one item to the stored `warnings`. It
|
|
545
|
-
changes no status. A rate limit is not an `ERROR`, and an `ERROR` status
|
|
546
|
-
makes the queue worker drop every webhook.
|
|
547
|
-
- A report for the same module whose reset time is within 60 s of a stored one
|
|
548
|
-
is skipped. The check reads the stored `warnings` with
|
|
549
|
-
`findIntegrationMessages`, not `this.messages`, which is per instance and
|
|
550
|
-
empty on Postgres. It is a read then a write, so workers that report at the
|
|
551
|
-
same instant can still store a few copies.
|
|
552
|
-
- Every hop is best effort. A failure is logged, and the caller still gets the
|
|
553
|
-
`RateLimitError`.
|
|
554
|
-
|
|
555
|
-
```javascript
|
|
556
|
-
{
|
|
557
|
-
title: 'Rate limit reached',
|
|
558
|
-
message: 'The hubspot API rate limit was reached and resets at 2026-09-28 14:31 UTC.',
|
|
559
|
-
timestamp: 1790000000000,
|
|
560
|
-
code: 'RATE_LIMITED',
|
|
561
|
-
module: 'hubspot',
|
|
562
|
-
reason: 'daily',
|
|
563
|
-
retryAt: '2026-09-28T14:30:15.000Z',
|
|
564
|
-
actions: [
|
|
565
|
-
{ type: 'RETRY_WHEN_READY' },
|
|
566
|
-
{ type: 'LINK', label: 'API usage limits', url: 'https://example.com/limits' },
|
|
567
|
-
],
|
|
568
|
-
}
|
|
569
|
-
```
|
|
570
|
-
|
|
571
|
-
The text is fixed. It holds the module name and the reset time in UTC, rounded
|
|
572
|
-
up to the minute, and nothing from the request.
|
|
573
|
-
|
|
574
|
-
Integration messages (the item form, `findIntegrationMessages`, the newest 50
|
|
575
|
-
per type): see "Integration Messages" in `docs/guides/INTEGRATION-PATTERNS.md`.
|
|
576
|
-
|
|
577
599
|
### 9. Logging System (`/logs`)
|
|
578
600
|
|
|
579
601
|
**Purpose**: One redacted JSON record per line to stdout (ADR-048). See `docs/guides/LOGGING.md`.
|
|
@@ -803,6 +825,8 @@ Use test doubles from `@friggframework/test` package for consistent mocking.
|
|
|
803
825
|
- `SECRET_ARN` - AWS Secrets Manager ARN for auto-injection
|
|
804
826
|
- `FRIGG_LOG_LEVEL` - Minimum log level (`TRACE` … `FATAL`, default `INFO`; `DEBUG` on local runs)
|
|
805
827
|
- `FRIGG_QUEUE_MAX_RECEIVE_COUNT` - Set by devtools on integration queue workers whose queue the stack owns; feeds `delivery.maxReceiveCount`
|
|
828
|
+
- `FRIGG_QUEUE_MAX_DEFERRALS` - Most rate-limit deferrals of one message (default 30)
|
|
829
|
+
- `FRIGG_QUEUE_MAX_DEFERRED_MS` - Most time in ms from a message's first rate-limit deferral to its retry time (default 26 h)
|
|
806
830
|
|
|
807
831
|
## Version Information
|
|
808
832
|
|
package/core/CLAUDE.md
CHANGED
|
@@ -79,6 +79,31 @@ await worker.send({
|
|
|
79
79
|
}, delaySeconds);
|
|
80
80
|
```
|
|
81
81
|
|
|
82
|
+
**Rate-limit deferral** (ADR-049): `run()` handles an error with `isRateLimited`
|
|
83
|
+
and a `retryAt` through `defer(record, body, error, delivery)`, which returns
|
|
84
|
+
`{ outcome }`:
|
|
85
|
+
- `acked`: the message was sent again with `DelaySeconds` (wait up to 900 s) or
|
|
86
|
+
scheduled with the one-time scheduler (longer wait). The record is handled.
|
|
87
|
+
- `failed`: the visibility timeout was extended to `retryAt`, or a cap
|
|
88
|
+
(`FRIGG_QUEUE_MAX_DEFERRALS`, `FRIGG_QUEUE_MAX_DEFERRED_MS`) ended the
|
|
89
|
+
deferrals. The record is reported failed.
|
|
90
|
+
- `skipped`: no `retryAt` (`no_retry_at`), no `eventSourceARN` or a FIFO queue
|
|
91
|
+
(`no_event_source`), a failed send (`send_failed`), or a failed visibility
|
|
92
|
+
change (`visibility_failed`). The record fails as any error does.
|
|
93
|
+
|
|
94
|
+
Each outcome writes one log record. `acked` writes WARN `record_deferred`.
|
|
95
|
+
`failed` and `skipped` write one WARN (`record_visibility_extended`,
|
|
96
|
+
`record_deferral_capped`, or `record_failed` with `deferral: { skipped }`). On
|
|
97
|
+
the last attempt they write one ERROR, `record_lost_rate_limited`, with the
|
|
98
|
+
details, and no WARN.
|
|
99
|
+
|
|
100
|
+
Subclasses override two no-op hooks to keep run state: `recordRateLimitWait(body,
|
|
101
|
+
error, state)` runs after the message was put back, and `clearRateLimitWait(body)`
|
|
102
|
+
runs after a deferred or redelivered message succeeded. A failing hook logs
|
|
103
|
+
`frigg.worker.rate_limit_state_failed` and never changes the outcome. The queue
|
|
104
|
+
worker of an integration (`createQueueWorker`) writes `Process.context.rateLimit`
|
|
105
|
+
in them. See `packages/core/CLAUDE.md` section 7.
|
|
106
|
+
|
|
82
107
|
### Delegate Pattern System (`Delegate.js:3-27`)
|
|
83
108
|
|
|
84
109
|
**Purpose**: Observer/delegation pattern for decoupled component communication
|
package/core/Worker.js
CHANGED
|
@@ -1,14 +1,38 @@
|
|
|
1
|
-
const {
|
|
1
|
+
const {
|
|
2
|
+
SQSClient,
|
|
3
|
+
GetQueueUrlCommand,
|
|
4
|
+
SendMessageCommand,
|
|
5
|
+
ChangeMessageVisibilityCommand,
|
|
6
|
+
} = require('@aws-sdk/client-sqs');
|
|
2
7
|
const _ = require('lodash');
|
|
3
8
|
const { RequiredPropertyError } = require('../errors');
|
|
4
9
|
const { get } = require('../assertions');
|
|
5
10
|
const { readQueueDelivery } = require('../queues/queue-delivery');
|
|
11
|
+
const { awsConfigOptions } = require('../queues/queuer-util');
|
|
12
|
+
const {
|
|
13
|
+
MAX_DEFERRED_VISIBILITY_SECONDS,
|
|
14
|
+
MAX_DELAY_SECONDS,
|
|
15
|
+
deferredMs,
|
|
16
|
+
isDeferralCapped,
|
|
17
|
+
nextDeferral,
|
|
18
|
+
readDeferral,
|
|
19
|
+
readDeferralLimits,
|
|
20
|
+
withDeferral,
|
|
21
|
+
} = require('../queues/queue-deferral');
|
|
6
22
|
const { runMessageScope } = require('./invocation-scope');
|
|
7
23
|
const { getLogger } = require('../logs');
|
|
8
24
|
|
|
9
|
-
const sqs = new SQSClient({
|
|
25
|
+
const sqs = new SQSClient({
|
|
26
|
+
region: process.env.AWS_REGION,
|
|
27
|
+
...awsConfigOptions(),
|
|
28
|
+
});
|
|
10
29
|
const log = getLogger('frigg.worker');
|
|
11
30
|
|
|
31
|
+
const validDate = (value) =>
|
|
32
|
+
value instanceof Date && !Number.isNaN(value.getTime()) ? value : null;
|
|
33
|
+
const queueNameOf = (arn) =>
|
|
34
|
+
typeof arn === 'string' && arn ? arn.split(':').pop() : undefined;
|
|
35
|
+
|
|
12
36
|
class Worker {
|
|
13
37
|
async getQueueURL(params) {
|
|
14
38
|
// Passing params in because there will be multiple QueueNames
|
|
@@ -33,12 +57,31 @@ class Worker {
|
|
|
33
57
|
// messageId, receiveCount and the event come from the scope.
|
|
34
58
|
log.debug('Record started', { eventName: 'frigg.worker.record_started' });
|
|
35
59
|
|
|
60
|
+
const delivery = readQueueDelivery(record);
|
|
61
|
+
let runParams;
|
|
36
62
|
try {
|
|
37
|
-
|
|
63
|
+
runParams = JSON.parse(record.body);
|
|
38
64
|
this._validateParams(runParams);
|
|
39
|
-
await this._run(runParams, context,
|
|
65
|
+
await this._run(runParams, context, delivery);
|
|
66
|
+
await this._clearDeferredState(runParams, delivery);
|
|
40
67
|
log.debug('Record succeeded', { eventName: 'frigg.worker.record_succeeded' });
|
|
41
68
|
} catch (error) {
|
|
69
|
+
let deferralResult;
|
|
70
|
+
if (error.isRateLimited && runParams) {
|
|
71
|
+
deferralResult = await this.defer(
|
|
72
|
+
record,
|
|
73
|
+
runParams,
|
|
74
|
+
error,
|
|
75
|
+
delivery
|
|
76
|
+
);
|
|
77
|
+
if (deferralResult.outcome === 'acked') return;
|
|
78
|
+
if (deferralResult.outcome === 'failed') {
|
|
79
|
+
batchItemFailures.push({
|
|
80
|
+
itemIdentifier: record.messageId,
|
|
81
|
+
});
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
42
85
|
if (error.isHaltError) {
|
|
43
86
|
// HaltError means "discard this message, don't retry".
|
|
44
87
|
// Treat as success so SQS deletes it from the queue.
|
|
@@ -51,11 +94,30 @@ class Worker {
|
|
|
51
94
|
});
|
|
52
95
|
return;
|
|
53
96
|
}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
97
|
+
const skippedDeferral = deferralResult && {
|
|
98
|
+
deferral: {
|
|
99
|
+
skipped: deferralResult.skipped,
|
|
100
|
+
...(deferralResult.cause && {
|
|
101
|
+
cause: deferralResult.cause,
|
|
102
|
+
}),
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
if (skippedDeferral && delivery.isLastAttempt) {
|
|
106
|
+
this._logLost({
|
|
107
|
+
retryAt: validDate(error.retryAt)?.toISOString(),
|
|
108
|
+
reason: error.reason,
|
|
109
|
+
module: error.module,
|
|
110
|
+
...skippedDeferral,
|
|
111
|
+
error,
|
|
112
|
+
});
|
|
113
|
+
} else {
|
|
114
|
+
// The message goes back to SQS, so WARN (ADR-048 §4).
|
|
115
|
+
log.warn('Record failed, returned for retry', {
|
|
116
|
+
eventName: 'frigg.worker.record_failed',
|
|
117
|
+
error,
|
|
118
|
+
...skippedDeferral,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
59
121
|
batchItemFailures.push({ itemIdentifier: record.messageId });
|
|
60
122
|
}
|
|
61
123
|
});
|
|
@@ -75,6 +137,239 @@ class Worker {
|
|
|
75
137
|
// parameters
|
|
76
138
|
}
|
|
77
139
|
|
|
140
|
+
async defer(record, body, error, delivery) {
|
|
141
|
+
const retryAt = validDate(error.retryAt);
|
|
142
|
+
if (!retryAt) return { outcome: 'skipped', skipped: 'no_retry_at' };
|
|
143
|
+
const queueName = queueNameOf(record.eventSourceARN);
|
|
144
|
+
if (!queueName || queueName.endsWith('.fifo')) {
|
|
145
|
+
return { outcome: 'skipped', skipped: 'no_event_source' };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const now = Date.now();
|
|
149
|
+
const counters = nextDeferral(body, now);
|
|
150
|
+
const deferral = {
|
|
151
|
+
record,
|
|
152
|
+
queueName,
|
|
153
|
+
body,
|
|
154
|
+
deferredBody: withDeferral(body, counters),
|
|
155
|
+
error,
|
|
156
|
+
delivery,
|
|
157
|
+
retryAt,
|
|
158
|
+
waitMs: Math.max(0, retryAt.getTime() - now),
|
|
159
|
+
counters,
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
const limits = readDeferralLimits();
|
|
163
|
+
if (isDeferralCapped({ ...counters, retryAt }, limits)) {
|
|
164
|
+
await this._recordWait(deferral, 'EXHAUSTED', 'none');
|
|
165
|
+
this._logUnacked(
|
|
166
|
+
deferral,
|
|
167
|
+
'record_deferral_capped',
|
|
168
|
+
'Rate-limit deferral capped',
|
|
169
|
+
{
|
|
170
|
+
maxDeferrals: limits.maxDeferrals,
|
|
171
|
+
maxDeferredMs: limits.maxDeferredMs,
|
|
172
|
+
deferredMs: deferredMs({ ...counters, retryAt }),
|
|
173
|
+
}
|
|
174
|
+
);
|
|
175
|
+
return { outcome: 'failed' };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
if (deferral.waitMs <= MAX_DELAY_SECONDS * 1000)
|
|
179
|
+
return this._deferWithDelay(deferral);
|
|
180
|
+
const scheduled = await this._deferWithSchedule(deferral);
|
|
181
|
+
if (scheduled?.outcome) return scheduled;
|
|
182
|
+
return this._deferWithVisibility(deferral, scheduled?.scheduleError);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async _deferWithDelay(deferral) {
|
|
186
|
+
const delaySeconds = Math.ceil(deferral.waitMs / 1000);
|
|
187
|
+
try {
|
|
188
|
+
const queueUrl = await this._queueUrl(deferral.queueName);
|
|
189
|
+
await this.send(
|
|
190
|
+
{ ...deferral.deferredBody, QueueUrl: queueUrl },
|
|
191
|
+
delaySeconds
|
|
192
|
+
);
|
|
193
|
+
} catch (cause) {
|
|
194
|
+
return { outcome: 'skipped', skipped: 'send_failed', cause };
|
|
195
|
+
}
|
|
196
|
+
return this._acknowledgeDeferral(deferral, 'delay', { delaySeconds });
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
async _deferWithSchedule(deferral) {
|
|
200
|
+
const { record, retryAt, deferredBody } = deferral;
|
|
201
|
+
const scheduleName = `frigg-defer-${record.messageId}`;
|
|
202
|
+
try {
|
|
203
|
+
const scheduler = this.getSchedulerService();
|
|
204
|
+
if (!scheduler) return null;
|
|
205
|
+
await scheduler.scheduleOneTime({
|
|
206
|
+
scheduleName,
|
|
207
|
+
scheduleAt: retryAt,
|
|
208
|
+
queueResourceId: record.eventSourceARN,
|
|
209
|
+
payload: deferredBody,
|
|
210
|
+
});
|
|
211
|
+
} catch (cause) {
|
|
212
|
+
if (cause?.name !== 'ConflictException')
|
|
213
|
+
return { scheduleError: cause };
|
|
214
|
+
}
|
|
215
|
+
return this._acknowledgeDeferral(deferral, 'schedule', {
|
|
216
|
+
scheduleName,
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async _deferWithVisibility(deferral, scheduleError) {
|
|
221
|
+
const { record, queueName, delivery, waitMs } = deferral;
|
|
222
|
+
const visibilityTimeout = Math.min(
|
|
223
|
+
Math.ceil(waitMs / 1000),
|
|
224
|
+
MAX_DEFERRED_VISIBILITY_SECONDS
|
|
225
|
+
);
|
|
226
|
+
try {
|
|
227
|
+
const queueUrl = await this._queueUrl(queueName);
|
|
228
|
+
await sqs.send(
|
|
229
|
+
new ChangeMessageVisibilityCommand({
|
|
230
|
+
QueueUrl: queueUrl,
|
|
231
|
+
ReceiptHandle: record.receiptHandle,
|
|
232
|
+
VisibilityTimeout: visibilityTimeout,
|
|
233
|
+
})
|
|
234
|
+
);
|
|
235
|
+
} catch (cause) {
|
|
236
|
+
return { outcome: 'skipped', skipped: 'visibility_failed', cause };
|
|
237
|
+
}
|
|
238
|
+
await this._recordWait(
|
|
239
|
+
deferral,
|
|
240
|
+
delivery.isLastAttempt ? 'EXHAUSTED' : 'WAITING',
|
|
241
|
+
'visibility'
|
|
242
|
+
);
|
|
243
|
+
this._logUnacked(
|
|
244
|
+
deferral,
|
|
245
|
+
'record_visibility_extended',
|
|
246
|
+
'Record visibility extended',
|
|
247
|
+
{
|
|
248
|
+
mechanism: 'visibility',
|
|
249
|
+
visibilityTimeout,
|
|
250
|
+
...(scheduleError && { scheduleError }),
|
|
251
|
+
}
|
|
252
|
+
);
|
|
253
|
+
return { outcome: 'failed' };
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
async _acknowledgeDeferral(deferral, mechanism, fields) {
|
|
257
|
+
await this._recordWait(deferral, 'WAITING', mechanism);
|
|
258
|
+
log.warn('Record deferred', {
|
|
259
|
+
eventName: 'frigg.worker.record_deferred',
|
|
260
|
+
mechanism,
|
|
261
|
+
...fields,
|
|
262
|
+
...this._deferralLogFields(deferral),
|
|
263
|
+
error: deferral.error,
|
|
264
|
+
});
|
|
265
|
+
return { outcome: 'acked' };
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
getSchedulerService() {
|
|
269
|
+
if (this._schedulerService !== undefined) return this._schedulerService;
|
|
270
|
+
if (
|
|
271
|
+
!process.env.SCHEDULER_ROLE_ARN ||
|
|
272
|
+
process.env.SCHEDULER_PROVIDER === 'mock'
|
|
273
|
+
) {
|
|
274
|
+
this._schedulerService = null;
|
|
275
|
+
return null;
|
|
276
|
+
}
|
|
277
|
+
const {
|
|
278
|
+
createSchedulerService,
|
|
279
|
+
SCHEDULER_PROVIDERS,
|
|
280
|
+
} = require('../infrastructure/scheduler/scheduler-service-factory');
|
|
281
|
+
this._schedulerService = createSchedulerService({
|
|
282
|
+
provider: SCHEDULER_PROVIDERS.EVENTBRIDGE,
|
|
283
|
+
});
|
|
284
|
+
return this._schedulerService;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
recordRateLimitWait() {
|
|
288
|
+
return Promise.resolve();
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
clearRateLimitWait() {
|
|
292
|
+
return Promise.resolve();
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
_queueUrl(queueName) {
|
|
296
|
+
this._queueUrls ??= new Map();
|
|
297
|
+
if (!this._queueUrls.has(queueName)) {
|
|
298
|
+
const lookup = this.getQueueURL({ QueueName: queueName }).catch(
|
|
299
|
+
(error) => {
|
|
300
|
+
this._queueUrls.delete(queueName);
|
|
301
|
+
throw error;
|
|
302
|
+
}
|
|
303
|
+
);
|
|
304
|
+
this._queueUrls.set(queueName, lookup);
|
|
305
|
+
}
|
|
306
|
+
return this._queueUrls.get(queueName);
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
async _recordWait({ body, error, counters, retryAt }, status, mechanism) {
|
|
310
|
+
await this._runStateHook('record', () =>
|
|
311
|
+
this.recordRateLimitWait(body, error, {
|
|
312
|
+
status,
|
|
313
|
+
mechanism,
|
|
314
|
+
deferrals: counters.deferrals,
|
|
315
|
+
retryAt,
|
|
316
|
+
})
|
|
317
|
+
);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
async _clearDeferredState(body, delivery) {
|
|
321
|
+
const redelivered = delivery.receiveCount > 1;
|
|
322
|
+
if (readDeferral(body).deferrals === 0 && !redelivered) return;
|
|
323
|
+
await this._runStateHook('clear', () => this.clearRateLimitWait(body));
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
async _runStateHook(operation, hook) {
|
|
327
|
+
try {
|
|
328
|
+
await hook();
|
|
329
|
+
} catch (error) {
|
|
330
|
+
log.warn('Rate-limit run state not written', {
|
|
331
|
+
eventName: 'frigg.worker.rate_limit_state_failed',
|
|
332
|
+
operation,
|
|
333
|
+
error,
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
_deferralLogFields({ error, retryAt, waitMs, counters }) {
|
|
339
|
+
return {
|
|
340
|
+
waitMs,
|
|
341
|
+
retryAt: retryAt.toISOString(),
|
|
342
|
+
deferrals: counters.deferrals,
|
|
343
|
+
firstDeferredAt: counters.firstDeferredAt,
|
|
344
|
+
reason: error.reason,
|
|
345
|
+
module: error.module,
|
|
346
|
+
statusCode: error.statusCode,
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
_logUnacked(deferral, eventName, message, fields) {
|
|
351
|
+
const logFields = {
|
|
352
|
+
...fields,
|
|
353
|
+
...this._deferralLogFields(deferral),
|
|
354
|
+
error: deferral.error,
|
|
355
|
+
};
|
|
356
|
+
if (deferral.delivery.isLastAttempt) {
|
|
357
|
+
this._logLost(logFields);
|
|
358
|
+
return;
|
|
359
|
+
}
|
|
360
|
+
log.warn(message, {
|
|
361
|
+
eventName: `frigg.worker.${eventName}`,
|
|
362
|
+
...logFields,
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
_logLost(fields) {
|
|
367
|
+
log.error(
|
|
368
|
+
'Rate-limited record lost: the message goes to the dead-letter queue next',
|
|
369
|
+
{ eventName: 'frigg.worker.record_lost_rate_limited', ...fields }
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
|
|
78
373
|
// returns the message id
|
|
79
374
|
async send(params, delay = 0) {
|
|
80
375
|
this._validateParams(params);
|
|
@@ -21,6 +21,12 @@ const {
|
|
|
21
21
|
getModulesDefinitionFromIntegrationClasses,
|
|
22
22
|
} = require('../integrations/utils/map-integration-dto');
|
|
23
23
|
const { processNotFound } = require('../integrations/use-cases/process-errors');
|
|
24
|
+
const {
|
|
25
|
+
RecordRateLimitWait,
|
|
26
|
+
} = require('../integrations/use-cases/record-rate-limit-wait');
|
|
27
|
+
const {
|
|
28
|
+
ClearRateLimitWait,
|
|
29
|
+
} = require('../integrations/use-cases/clear-rate-limit-wait');
|
|
24
30
|
|
|
25
31
|
const loadRouterFromObject = (IntegrationClass, routerObject) => {
|
|
26
32
|
const router = Router();
|
|
@@ -165,6 +171,28 @@ const createQueueWorker = (integrationClass) => {
|
|
|
165
171
|
const integrationName = integrationClass.Definition.name;
|
|
166
172
|
|
|
167
173
|
class QueueWorker extends Worker {
|
|
174
|
+
async recordRateLimitWait(body, error, state) {
|
|
175
|
+
const processId = body.data?.processId;
|
|
176
|
+
if (!processId) return;
|
|
177
|
+
const recordRateLimitWait = new RecordRateLimitWait({
|
|
178
|
+
processRepository: createProcessRepository(),
|
|
179
|
+
});
|
|
180
|
+
await recordRateLimitWait.execute(processId, {
|
|
181
|
+
...state,
|
|
182
|
+
reason: error.reason,
|
|
183
|
+
module: error.module,
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
async clearRateLimitWait(body) {
|
|
188
|
+
const processId = body?.data?.processId;
|
|
189
|
+
if (!processId) return;
|
|
190
|
+
const clearRateLimitWait = new ClearRateLimitWait({
|
|
191
|
+
processRepository: createProcessRepository(),
|
|
192
|
+
});
|
|
193
|
+
await clearRateLimitWait.execute(processId);
|
|
194
|
+
}
|
|
195
|
+
|
|
168
196
|
async _run(params, context, delivery) {
|
|
169
197
|
const logCtx = {
|
|
170
198
|
integration: integrationName,
|
|
@@ -17,9 +17,6 @@ const {
|
|
|
17
17
|
const {
|
|
18
18
|
UpdateIntegrationConfig,
|
|
19
19
|
} = require('./use-cases/update-integration-config');
|
|
20
|
-
const {
|
|
21
|
-
RecordRateLimitMessage,
|
|
22
|
-
} = require('./use-cases/record-rate-limit-message');
|
|
23
20
|
const { validateExtensionBinding } = require('./extension');
|
|
24
21
|
const { getTelemetry } = require('../telemetry/telemetry-runtime');
|
|
25
22
|
const { instrumentHandler } = require('../telemetry/instrument-handler');
|
|
@@ -62,9 +59,6 @@ class IntegrationBase {
|
|
|
62
59
|
updateIntegrationConfig = new UpdateIntegrationConfig({
|
|
63
60
|
integrationRepository: this.integrationRepository,
|
|
64
61
|
});
|
|
65
|
-
recordRateLimitMessage = new RecordRateLimitMessage({
|
|
66
|
-
integrationRepository: this.integrationRepository,
|
|
67
|
-
});
|
|
68
62
|
|
|
69
63
|
// this.telemetry.count('records.synced', n, { entity: 'contact' })
|
|
70
64
|
telemetry = bindTelemetryContext(getTelemetry(), () =>
|
|
@@ -914,16 +908,6 @@ class IntegrationBase {
|
|
|
914
908
|
});
|
|
915
909
|
await this.persistStatus('ENABLED');
|
|
916
910
|
}
|
|
917
|
-
|
|
918
|
-
if (delegateString === 'RATE_LIMITED') {
|
|
919
|
-
this.logger.warn('Module reported a rate limit', {
|
|
920
|
-
eventName: `${this.logger.name}.rate_limited`,
|
|
921
|
-
moduleName: object?.moduleName,
|
|
922
|
-
reason: object?.reason,
|
|
923
|
-
retryAt: object?.retryAt,
|
|
924
|
-
});
|
|
925
|
-
await this.recordRateLimitMessage.execute(this.id, object);
|
|
926
|
-
}
|
|
927
911
|
}
|
|
928
912
|
|
|
929
913
|
/**
|
|
@@ -14,7 +14,6 @@ const {
|
|
|
14
14
|
IntegrationRepositoryInterface,
|
|
15
15
|
} = require('./integration-repository-interface');
|
|
16
16
|
const { validateConfigPatch } = require('./config-patch-shared');
|
|
17
|
-
const { messagesOfType } = require('./message-item-shared');
|
|
18
17
|
|
|
19
18
|
class IntegrationRepositoryDocumentDB extends IntegrationRepositoryInterface {
|
|
20
19
|
constructor() {
|
|
@@ -95,8 +94,9 @@ class IntegrationRepositoryDocumentDB extends IntegrationRepositoryInterface {
|
|
|
95
94
|
async updateIntegrationMessages(
|
|
96
95
|
integrationId,
|
|
97
96
|
messageType,
|
|
98
|
-
|
|
99
|
-
|
|
97
|
+
messageTitle,
|
|
98
|
+
messageBody,
|
|
99
|
+
messageTimestamp
|
|
100
100
|
) {
|
|
101
101
|
const objectId = toObjectId(integrationId);
|
|
102
102
|
if (!objectId) {
|
|
@@ -107,12 +107,13 @@ class IntegrationRepositoryDocumentDB extends IntegrationRepositoryInterface {
|
|
|
107
107
|
throw new Error(`Integration ${integrationId} not found`);
|
|
108
108
|
}
|
|
109
109
|
const messages = this._extractMessages(existing);
|
|
110
|
-
const list = [...
|
|
111
|
-
list.push({
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
};
|
|
110
|
+
const list = Array.isArray(messages[messageType]) ? [...messages[messageType]] : [];
|
|
111
|
+
list.push({
|
|
112
|
+
title: messageTitle ?? null,
|
|
113
|
+
message: messageBody,
|
|
114
|
+
timestamp: messageTimestamp,
|
|
115
|
+
});
|
|
116
|
+
const updatedMessages = { ...messages, [messageType]: list };
|
|
116
117
|
await updateOne(
|
|
117
118
|
this.prisma,
|
|
118
119
|
'Integration',
|
|
@@ -131,20 +132,6 @@ class IntegrationRepositoryDocumentDB extends IntegrationRepositoryInterface {
|
|
|
131
132
|
return true;
|
|
132
133
|
}
|
|
133
134
|
|
|
134
|
-
async findIntegrationMessages(integrationId, messageType) {
|
|
135
|
-
const objectId = toObjectId(integrationId);
|
|
136
|
-
if (!objectId) {
|
|
137
|
-
throw new Error(`Integration ${integrationId} not found`);
|
|
138
|
-
}
|
|
139
|
-
const existing = await findOne(this.prisma, 'Integration', {
|
|
140
|
-
_id: objectId,
|
|
141
|
-
});
|
|
142
|
-
if (!existing) {
|
|
143
|
-
throw new Error(`Integration ${integrationId} not found`);
|
|
144
|
-
}
|
|
145
|
-
return messagesOfType(this._extractMessages(existing), messageType);
|
|
146
|
-
}
|
|
147
|
-
|
|
148
135
|
async createIntegration(entities, userId, config) {
|
|
149
136
|
const now = new Date();
|
|
150
137
|
const document = {
|