@friggframework/core 2.0.0--canary.693.d8859c2.0 → 2.0.0--canary.694.f66a64a.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/database/documentdb-utils.js +2 -0
- package/handlers/workers/integration-triggers.js +25 -0
- package/infrastructure/scheduler/recurrence.js +204 -0
- package/integrations/data-defined/README.md +19 -3
- package/integrations/data-defined/compile.js +8 -5
- package/integrations/data-defined/config-form.js +27 -0
- package/integrations/data-defined/data-defined-integration.js +54 -3
- package/integrations/data-defined/schema/installation.v0.1.schema.json +47 -0
- package/integrations/data-defined/schema/integration-definition.v0.1.schema.json +60 -6
- package/integrations/data-defined/validate.js +15 -1
- package/integrations/data-defined/versioning.js +16 -1
- package/integrations/integration-base.js +88 -0
- package/integrations/repositories/integration-repository-documentdb.js +6 -2
- package/integrations/repositories/integration-repository-interface.js +3 -0
- package/integrations/repositories/integration-repository-mongo.js +19 -4
- package/integrations/repositories/integration-repository-postgres.js +5 -1
- package/integrations/triggers.js +244 -0
- package/integrations/use-cases/run-trigger-tick.js +132 -0
- package/integrations/use-cases/run-trigger.js +77 -0
- package/package.json +5 -5
- package/queues/queuer-util.js +9 -2
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
const { BUILTINS, isBuiltin, splitCapability } = require('./steps');
|
|
14
14
|
const { parseCondition, referencedPaths, templateTags, FILTERS } = require('./expressions');
|
|
15
15
|
const { TRANSFORMS: TRANSFORM_FNS } = require('./transforms');
|
|
16
|
+
const { validateRecurrence } = require('../../infrastructure/scheduler/recurrence');
|
|
16
17
|
|
|
17
18
|
const ID = /^[a-z][a-z0-9-]*$/;
|
|
18
19
|
const SEMVER = /^\d+\.\d+\.\d+(-[\w.]+)?$/;
|
|
@@ -26,6 +27,14 @@ function validate(def, manifests = {}) {
|
|
|
26
27
|
const errors = [], warnings = [];
|
|
27
28
|
const err = (path, message) => errors.push({ path, message });
|
|
28
29
|
const warn = (path, message) => warnings.push({ path, message });
|
|
30
|
+
/** A schedule's own timing must be valid, and must itself respect the floor it sets for installations. */
|
|
31
|
+
const checkSchedule = (recurrence, configurable, path) => {
|
|
32
|
+
const where = recurrence.every !== undefined ? `${path}.every` : `${path}.cron`;
|
|
33
|
+
for (const m of validateRecurrence(recurrence)) err(where, m);
|
|
34
|
+
if (configurable && typeof configurable === 'object' && configurable.min !== undefined) {
|
|
35
|
+
for (const m of validateRecurrence(recurrence, { min: configurable.min })) if (!validateRecurrence(recurrence).includes(m)) err(`${path}.configurable.min`, `The default schedule breaks its own minimum: ${m}`);
|
|
36
|
+
}
|
|
37
|
+
};
|
|
29
38
|
|
|
30
39
|
if (!def || typeof def !== 'object') return { valid: false, errors: [{ path: '', message: 'Definition must be an object' }], warnings };
|
|
31
40
|
if (def.schemaVersion !== '0.1') err('schemaVersion', 'Must be "0.1"');
|
|
@@ -154,6 +163,8 @@ function validate(def, manifests = {}) {
|
|
|
154
163
|
const hasEvents = (k, o) => Object.values(manifestOf[k]?.events || {}).some((e) => e.object === o);
|
|
155
164
|
if (manifestOf[src.k] && !hasEvents(src.k, src.o)) warn(`${p}.ongoing`, `${s.source} declares no webhook events; ongoing sync will need polling`);
|
|
156
165
|
}
|
|
166
|
+
if (s.ongoing?.mode === 'poll') checkSchedule({ every: s.ongoing.every || '15m' }, s.ongoing.configurable, `${p}.ongoing`);
|
|
167
|
+
else if (s.ongoing?.configurable) err(`${p}.ongoing.configurable`, 'Only a polling sync has an interval to configure');
|
|
157
168
|
if (s.filter) checkCondition(s.filter, `${p}.filter`);
|
|
158
169
|
});
|
|
159
170
|
|
|
@@ -273,7 +284,10 @@ function validate(def, manifests = {}) {
|
|
|
273
284
|
if (!manifestOf[t.module]) err(`${p}.trigger.module`, `Unknown module key "${t.module}"`);
|
|
274
285
|
else if (!manifestOf[t.module].events?.[t.topic]) err(`${p}.trigger.topic`, `Module "${t.module}" emits no event "${t.topic}"`);
|
|
275
286
|
}
|
|
276
|
-
if (t.type === 'schedule'
|
|
287
|
+
if (t.type === 'schedule') {
|
|
288
|
+
if ((t.cron === undefined) === (t.every === undefined)) err(`${p}.trigger`, 'Give exactly one of cron or every');
|
|
289
|
+
else checkSchedule(t.every !== undefined ? { every: t.every } : { cron: t.cron, ...(t.timezone ? { timezone: t.timezone } : {}) }, t.configurable, `${p}.trigger`);
|
|
290
|
+
}
|
|
277
291
|
if (t.type === 'userAction' && t.from && !uis[t.from]) err(`${p}.trigger.from`, `Unknown UI surface "${t.from}"`);
|
|
278
292
|
if (t.type === 'syncRecord' && !syncs[t.sync]) err(`${p}.trigger.sync`, `Unknown sync "${t.sync}"`);
|
|
279
293
|
if (t.filter) checkCondition(t.filter, `${p}.trigger.filter`);
|
|
@@ -51,7 +51,8 @@ function diff(from, to) {
|
|
|
51
51
|
if (o.direction !== s.direction) push('major', `syncs.${id}.direction`, 'Sync direction changed');
|
|
52
52
|
if (!same(o.identity, s.identity)) push('major', `syncs.${id}.identity`, 'Identity keys changed: existing links must be rebuilt');
|
|
53
53
|
if (!same(o.mapping, s.mapping)) push('patch', `syncs.${id}.mapping`, 'Default mapping changed (installations with their own mapping keep it)');
|
|
54
|
-
if (
|
|
54
|
+
if (o.ongoing?.configurable && !s.ongoing?.configurable) push('major', `syncs.${id}.ongoing.configurable`, 'Poll interval no longer configurable: installations lose their chosen interval');
|
|
55
|
+
else if (!same(o.ongoing, s.ongoing) || !same(o.backfill, s.backfill) || o.filter !== s.filter) push('minor', `syncs.${id}`, 'Sync schedule or filter changed');
|
|
55
56
|
}
|
|
56
57
|
|
|
57
58
|
const la = byId(from.links), lb = byId(to.links);
|
|
@@ -70,6 +71,7 @@ function diff(from, to) {
|
|
|
70
71
|
const o = wa[id];
|
|
71
72
|
if (!o) { push('minor', `workflows.${id}`, `New workflow (default ${w.default || 'on'})`); continue; }
|
|
72
73
|
if (o.trigger?.type !== w.trigger?.type || o.trigger?.module !== w.trigger?.module) push('major', `workflows.${id}.trigger`, 'Trigger source changed');
|
|
74
|
+
else if (o.trigger?.configurable && !w.trigger?.configurable) push('major', `workflows.${id}.trigger.configurable`, 'Schedule no longer configurable: installations lose their chosen timing');
|
|
73
75
|
else if (!same(o.trigger, w.trigger)) push('minor', `workflows.${id}.trigger`, 'Trigger topic, schedule or filter changed');
|
|
74
76
|
diffSettings(o.settings, w.settings, `workflows.${id}.settings`, push);
|
|
75
77
|
if (!same(o.steps, w.steps)) push('patch', `workflows.${id}.steps`, 'Steps changed');
|
|
@@ -152,6 +154,16 @@ function planUpgrade(installation, from, to) {
|
|
|
152
154
|
const la = byId(from.links), lb = byId(to.links);
|
|
153
155
|
for (const l of to.links || []) if (la[l.id] && !same(la[l.id].identity, l.identity)) actions.push({ kind: 'reindex', id: l.id, message: `${l.label || l.id}: links by id still resolve; lookups by the new identity keys miss until each pair is linked again` });
|
|
154
156
|
for (const id of Object.keys(la)) if (!lb[id]) actions.push({ kind: 'drop-links', id, message: `Link set "${la[id].label || id}" is gone; its stored links are no longer read` });
|
|
157
|
+
// An installation's own timing survives an upgrade only where the new version still allows it.
|
|
158
|
+
const keepTiming = (chosen, allowed, path, label) => {
|
|
159
|
+
if (!chosen?.schedule) return;
|
|
160
|
+
const problems = !allowed ? ['not configurable'] : validateRecurrence(recurrenceOf(chosen.schedule), { min: allowed === true ? undefined : allowed.min });
|
|
161
|
+
if (!problems.length) return;
|
|
162
|
+
delete chosen.schedule;
|
|
163
|
+
actions.push({ kind: 'reschedule', id: path, message: `${label}: back to the definition's schedule (${problems.join('; ')})` });
|
|
164
|
+
};
|
|
165
|
+
for (const w of to.workflows || []) if (w.trigger?.type === 'schedule') keepTiming(config.workflows?.[w.id], w.trigger.configurable, `workflows.${w.id}`, w.label || w.id);
|
|
166
|
+
for (const sy of to.syncs || []) keepTiming(config.syncs?.[sy.id], sy.ongoing?.mode === 'poll' && sy.ongoing.configurable, `syncs.${sy.id}`, sy.label || sy.id);
|
|
155
167
|
if (!same(from.lifecycle?.onInstall, to.lifecycle?.onInstall)) actions.push({ kind: 'reinstall', message: 'Re-run install steps (webhook subscriptions)' });
|
|
156
168
|
actions.push({ kind: 'drain', message: `Messages already queued under ${from.version} finish on ${from.version}; new events run on ${to.version}` });
|
|
157
169
|
|
|
@@ -159,3 +171,6 @@ function planUpgrade(installation, from, to) {
|
|
|
159
171
|
}
|
|
160
172
|
|
|
161
173
|
module.exports = { diff, checkBump, planUpgrade, bumpOf };
|
|
174
|
+
|
|
175
|
+
const { validateRecurrence } = require('../../infrastructure/scheduler/recurrence');
|
|
176
|
+
const { recurrenceOf } = require('../triggers');
|
|
@@ -22,6 +22,7 @@ const { getTelemetry } = require('../telemetry/telemetry-runtime');
|
|
|
22
22
|
const { instrumentHandler } = require('../telemetry/instrument-handler');
|
|
23
23
|
const { bindTelemetryContext } = require('../telemetry/bind-telemetry-context');
|
|
24
24
|
const { getLogger } = require('../logs');
|
|
25
|
+
const { TRIGGER_RUN, TRIGGER_TICK, resolveTrigger } = require('./triggers');
|
|
25
26
|
|
|
26
27
|
const constantsToBeMigrated = {
|
|
27
28
|
defaultEvents: {
|
|
@@ -35,6 +36,8 @@ const constantsToBeMigrated = {
|
|
|
35
36
|
REFRESH_USER_ACTION_OPTIONS: 'REFRESH_USER_ACTION_OPTIONS',
|
|
36
37
|
WEBHOOK_RECEIVED: 'WEBHOOK_RECEIVED', // HTTP handler, no DB
|
|
37
38
|
ON_WEBHOOK: 'ON_WEBHOOK', // Queue worker, DB-connected
|
|
39
|
+
[TRIGGER_RUN]: TRIGGER_RUN, // Queue worker: one recurring trigger run (ADR-055)
|
|
40
|
+
[TRIGGER_TICK]: TRIGGER_TICK, // Queue worker: a trigger tick carrying on from its page cursor
|
|
38
41
|
// etc...
|
|
39
42
|
},
|
|
40
43
|
types: {
|
|
@@ -91,6 +94,9 @@ class IntegrationBase {
|
|
|
91
94
|
extensions: {},
|
|
92
95
|
// usage: { canonical: ['records.synced'], custom: { 'deals.enriched': { unit, label } } }
|
|
93
96
|
usage: {},
|
|
97
|
+
// Recurring triggers (ADR-055) — see integrations/triggers.js
|
|
98
|
+
// Shape: { [name]: { every | at | cron, timezone?, tick?, when?, overlap?, configurable?, handler } }
|
|
99
|
+
triggers: {},
|
|
94
100
|
display: {
|
|
95
101
|
name: 'Integration Name',
|
|
96
102
|
logo: '',
|
|
@@ -171,6 +177,14 @@ class IntegrationBase {
|
|
|
171
177
|
type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT,
|
|
172
178
|
handler: this.onWebhook,
|
|
173
179
|
},
|
|
180
|
+
[TRIGGER_RUN]: {
|
|
181
|
+
type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT,
|
|
182
|
+
handler: this.onTriggerRun,
|
|
183
|
+
},
|
|
184
|
+
[TRIGGER_TICK]: {
|
|
185
|
+
type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT,
|
|
186
|
+
handler: this.onTriggerTick,
|
|
187
|
+
},
|
|
174
188
|
};
|
|
175
189
|
}
|
|
176
190
|
|
|
@@ -522,10 +536,84 @@ class IntegrationBase {
|
|
|
522
536
|
} else if (this.status === 'NEEDS_CONFIG') {
|
|
523
537
|
await this.persistStatus('ENABLED');
|
|
524
538
|
}
|
|
539
|
+
await this.reportTriggerChoices();
|
|
525
540
|
}
|
|
526
541
|
|
|
527
542
|
async onDelete(params) {}
|
|
528
543
|
|
|
544
|
+
/**
|
|
545
|
+
* The recurring triggers declared for one installation (ADR-055): `Definition.triggers`, each with its `name`.
|
|
546
|
+
* Static so the tick can read them from a record without hydrating the installation; a class whose triggers
|
|
547
|
+
* depend on the installation (data-defined integrations, `Definition.instanceTriggers`) overrides it.
|
|
548
|
+
* @param {{ config?: Object, record?: Object }} [installation]
|
|
549
|
+
* @returns {Promise<Array<Object>>}
|
|
550
|
+
*/
|
|
551
|
+
static async triggersFor(installation = {}) {
|
|
552
|
+
return Object.entries(this.Definition?.triggers || {}).map(([name, t]) => ({ ...t, name }));
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
/** This installation's trigger declarations (the static `triggersFor`, unless a subclass knows better). */
|
|
556
|
+
async triggerDeclarations() {
|
|
557
|
+
return this.constructor.triggersFor({ config: this.config || {}, record: this.record });
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/**
|
|
561
|
+
* Run one recurring trigger: the queue worker dispatches `FRIGG_TRIGGER_RUN` here with
|
|
562
|
+
* `{ integrationId, trigger, scheduledFor }` after hydrating the installation. The handler is called as
|
|
563
|
+
* `handler(payload, ctx)` with `payload = { trigger, scheduledFor }` and `ctx = { modules, config, log, integration }`
|
|
564
|
+
* (ADR-054); a string handler names a method on this instance. Overlap and the run record: use-cases/run-trigger.js.
|
|
565
|
+
*/
|
|
566
|
+
async onTriggerRun({ data = {} } = {}) {
|
|
567
|
+
const declarations = await this.triggerDeclarations();
|
|
568
|
+
const declaration = declarations.find((t) => t.name === data.trigger);
|
|
569
|
+
const resolved = declaration ? resolveTrigger(declaration.name, declaration, this.config || {}) : null;
|
|
570
|
+
const payload = { trigger: data.trigger, scheduledFor: data.scheduledFor };
|
|
571
|
+
const invoke = async (p) => {
|
|
572
|
+
const handler = typeof declaration.handler === 'string' ? this[declaration.handler] : declaration.handler;
|
|
573
|
+
if (typeof handler !== 'function') throw new Error(`Trigger "${declaration.name}" handler "${declaration.handler}" is not a method of ${this.constructor.Definition?.name}`);
|
|
574
|
+
return handler.call(this, p, { modules: this.modules, config: this.config || {}, log: this.logger, integration: this });
|
|
575
|
+
};
|
|
576
|
+
const { RunTrigger } = require('./use-cases/run-trigger');
|
|
577
|
+
if (!this.triggerProcessRepository) {
|
|
578
|
+
const { createProcessRepository } = require('./repositories/process-repository-factory');
|
|
579
|
+
this.triggerProcessRepository = createProcessRepository();
|
|
580
|
+
}
|
|
581
|
+
const outcome = await new RunTrigger({ processRepository: this.triggerProcessRepository }).execute({ integration: this, resolved, invoke, payload });
|
|
582
|
+
if (outcome.status !== 'ran') {
|
|
583
|
+
this.logger.info('Trigger run not started', { eventName: `${this.logger.name}.trigger_${outcome.status}`, trigger: data.trigger, reason: outcome.reason });
|
|
584
|
+
}
|
|
585
|
+
return outcome;
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
/** A tick that ran out of time continues here, on a dry instance, from its page cursor. */
|
|
589
|
+
async onTriggerTick({ data = {}, context } = {}) {
|
|
590
|
+
const { runTriggerTick } = require('./use-cases/run-trigger-tick');
|
|
591
|
+
return runTriggerTick(this.constructor, data, context);
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
/**
|
|
595
|
+
* Report each trigger timing this installation chose that cannot be used (not configurable, or under the
|
|
596
|
+
* floor): the trigger keeps its default, and the integration gets a warning message saying why. No-op for an
|
|
597
|
+
* integration without triggers.
|
|
598
|
+
*/
|
|
599
|
+
async reportTriggerChoices() {
|
|
600
|
+
if (this.id === undefined || this.id === null) return [];
|
|
601
|
+
try {
|
|
602
|
+
const declarations = await this.triggerDeclarations();
|
|
603
|
+
const problems = declarations
|
|
604
|
+
.map((t) => ({ name: t.name, problems: resolveTrigger(t.name, t, this.config || {}).problems }))
|
|
605
|
+
.filter((p) => p.problems.length);
|
|
606
|
+
for (const p of problems) {
|
|
607
|
+
await this.updateIntegrationMessages.execute(this.id, 'warnings', 'Trigger timing not changed', `${p.name}: ${p.problems.join('; ')}`, Date.now());
|
|
608
|
+
}
|
|
609
|
+
return problems;
|
|
610
|
+
} catch (error) {
|
|
611
|
+
// A report, never a reason to fail the update.
|
|
612
|
+
this.logger.warn('Trigger choices not checked', { eventName: `${this.logger.name}.trigger_choices_unchecked`, error });
|
|
613
|
+
return [];
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
|
|
529
617
|
async getConfigOptions() {
|
|
530
618
|
const options = {
|
|
531
619
|
jsonSchema: {},
|
|
@@ -28,7 +28,7 @@ class IntegrationRepositoryDocumentDB extends IntegrationRepositoryInterface {
|
|
|
28
28
|
return records.map((doc) => this._mapIntegration(doc));
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
-
async findIntegrations({ type, status } = {}) {
|
|
31
|
+
async findIntegrations({ type, status, limit, after } = {}) {
|
|
32
32
|
const filter = {};
|
|
33
33
|
if (type) {
|
|
34
34
|
filter['config.type'] = type;
|
|
@@ -36,7 +36,11 @@ class IntegrationRepositoryDocumentDB extends IntegrationRepositoryInterface {
|
|
|
36
36
|
if (status) {
|
|
37
37
|
filter.status = status;
|
|
38
38
|
}
|
|
39
|
-
|
|
39
|
+
if (limit && after !== undefined && after !== null) {
|
|
40
|
+
// Extended JSON: $runCommandRaw does not match a BSON ObjectId instance inside an operator.
|
|
41
|
+
filter._id = { $gt: { $oid: fromObjectId(toObjectId(after)) } };
|
|
42
|
+
}
|
|
43
|
+
const records = await findMany(this.prisma, 'Integration', filter, limit ? { sort: { _id: 1 }, limit, batchSize: limit } : {});
|
|
40
44
|
return records.map((doc) => this._mapIntegration(doc));
|
|
41
45
|
}
|
|
42
46
|
|
|
@@ -28,6 +28,9 @@ class IntegrationRepositoryInterface {
|
|
|
28
28
|
* @param {Object} [filter={}]
|
|
29
29
|
* @param {string} [filter.type] - Integration type (config.type)
|
|
30
30
|
* @param {string} [filter.status] - Integration status
|
|
31
|
+
* @param {number} [filter.limit] - Page size: at most this many rows, in id order
|
|
32
|
+
* @param {string|number} [filter.after] - With limit: only rows whose id sorts after this one (the last id of the
|
|
33
|
+
* previous page), so a caller can page through every row (the trigger tick does, ADR-055)
|
|
31
34
|
* @returns {Promise<Array>} Array of integration objects (possibly empty)
|
|
32
35
|
* @abstract
|
|
33
36
|
*/
|
|
@@ -61,20 +61,35 @@ class IntegrationRepositoryMongo extends IntegrationRepositoryInterface {
|
|
|
61
61
|
* @param {string} [filter.status] - Integration status
|
|
62
62
|
* @returns {Promise<Array>} Array of integration objects (possibly empty)
|
|
63
63
|
*/
|
|
64
|
-
async findIntegrations({ type, status } = {}) {
|
|
64
|
+
async findIntegrations({ type, status, limit, after } = {}) {
|
|
65
65
|
const where = {};
|
|
66
|
-
if (type) {
|
|
67
|
-
where.config = { path: ['type'], equals: type };
|
|
68
|
-
}
|
|
69
66
|
if (status) {
|
|
70
67
|
where.status = status;
|
|
71
68
|
}
|
|
69
|
+
if (limit && after !== undefined && after !== null) {
|
|
70
|
+
where.id = { gt: after };
|
|
71
|
+
}
|
|
72
|
+
if (type) {
|
|
73
|
+
// Prisma's MongoDB connector has no JSON path filter (`config: { path }` is refused), so the ids of the
|
|
74
|
+
// matching rows come from a raw find on config.type, then the rows load as usual.
|
|
75
|
+
const filter = { 'config.type': type };
|
|
76
|
+
if (status) filter.status = status;
|
|
77
|
+
if (where.id) filter._id = { $gt: { $oid: String(after) } };
|
|
78
|
+
const found = await this.prisma.integration.findRaw({
|
|
79
|
+
filter,
|
|
80
|
+
options: { projection: { _id: 1 }, sort: { _id: 1 }, ...(limit ? { limit } : {}) },
|
|
81
|
+
});
|
|
82
|
+
const ids = found.map((doc) => doc._id?.$oid ?? String(doc._id));
|
|
83
|
+
if (!ids.length) return [];
|
|
84
|
+
where.id = { in: ids };
|
|
85
|
+
}
|
|
72
86
|
|
|
73
87
|
const integrations = await this.prisma.integration.findMany({
|
|
74
88
|
where,
|
|
75
89
|
include: {
|
|
76
90
|
entities: true,
|
|
77
91
|
},
|
|
92
|
+
...(limit ? { orderBy: { id: 'asc' }, take: limit } : {}),
|
|
78
93
|
});
|
|
79
94
|
|
|
80
95
|
return integrations.map((integration) => ({
|
|
@@ -99,7 +99,7 @@ class IntegrationRepositoryPostgres extends IntegrationRepositoryInterface {
|
|
|
99
99
|
* @param {string} [filter.status] - Integration status
|
|
100
100
|
* @returns {Promise<Array>} Array of integration objects (possibly empty)
|
|
101
101
|
*/
|
|
102
|
-
async findIntegrations({ type, status } = {}) {
|
|
102
|
+
async findIntegrations({ type, status, limit, after } = {}) {
|
|
103
103
|
const where = {};
|
|
104
104
|
if (type) {
|
|
105
105
|
where.config = { path: ['type'], equals: type };
|
|
@@ -107,12 +107,16 @@ class IntegrationRepositoryPostgres extends IntegrationRepositoryInterface {
|
|
|
107
107
|
if (status) {
|
|
108
108
|
where.status = status;
|
|
109
109
|
}
|
|
110
|
+
if (limit && after !== undefined && after !== null) {
|
|
111
|
+
where.id = { gt: this._convertId(after) };
|
|
112
|
+
}
|
|
110
113
|
|
|
111
114
|
const integrations = await this.prisma.integration.findMany({
|
|
112
115
|
where,
|
|
113
116
|
include: {
|
|
114
117
|
entities: true,
|
|
115
118
|
},
|
|
119
|
+
...(limit ? { orderBy: { id: 'asc' }, take: limit } : {}),
|
|
116
120
|
});
|
|
117
121
|
|
|
118
122
|
return integrations.map((integration) => {
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recurring triggers (ADR-055): what an integration declares, how one installation resolves it, and whether it is
|
|
3
|
+
* due at a tick. Pure; no I/O. The tick itself is `use-cases/run-trigger-tick.js`; the run is
|
|
4
|
+
* `IntegrationBase.onTriggerRun`.
|
|
5
|
+
*
|
|
6
|
+
* static Definition = {
|
|
7
|
+
* name: 'housecallpro',
|
|
8
|
+
* triggers: {
|
|
9
|
+
* pollErp: { every: '15m', handler: 'pollErp' }, // every installation
|
|
10
|
+
* ongoingSync: { every: '1h', configurable: { min: '15m' }, handler: sync }, // an installation may retime it
|
|
11
|
+
* perPlan: { every: (config) => config.pollEvery, when: (config) => config.poll !== false, handler },
|
|
12
|
+
* digest: { cron: '0 8 * * 1-5', timezone: 'America/New_York', configurable: true, handler },
|
|
13
|
+
* nightly: { at: 'cron(0 3 * * ? *)', timezone: 'Europe/London', handler }, // same time for everyone
|
|
14
|
+
* },
|
|
15
|
+
* };
|
|
16
|
+
*
|
|
17
|
+
* Nothing is created per installation. Devtools provisions one function per integration with one schedule per
|
|
18
|
+
* distinct tick and one per `at` trigger (`provisioningOf`). At each tick the function pages through the ENABLED
|
|
19
|
+
* installations and sends `{ event: 'FRIGG_TRIGGER_RUN', data: { integrationId, trigger, scheduledFor } }` for each
|
|
20
|
+
* one that is due. A disabled or deleted installation is simply not due, and a config change applies at the next tick.
|
|
21
|
+
*
|
|
22
|
+
* Two additions to ADR-055 (its "Amendment (ADR-063)"):
|
|
23
|
+
* - `configurable: true | { min }` on an `every` or `cron` trigger lets an installation choose its own timing at
|
|
24
|
+
* `config.triggers.<name>` (`{ every }`, `{ cron, timezone }` or `{ enabled: false }`), never more often than `min`.
|
|
25
|
+
* A choice that breaks the floor, or a trigger that is not configurable, keeps the default (and is reported).
|
|
26
|
+
* - `cron` (five fields, with `timezone`) is a calendar trigger that may differ per installation. It runs off the
|
|
27
|
+
* shared tick: due when a firing falls in (previous tick, this tick]. `at` stays the ADR-055 calendar trigger,
|
|
28
|
+
* the same for every installation, with an exact schedule of its own.
|
|
29
|
+
*/
|
|
30
|
+
const { parseEvery, validateRecurrence, minGapMinutes, lastFiringIn, isTimeZone } = require('../infrastructure/scheduler/recurrence');
|
|
31
|
+
|
|
32
|
+
const TRIGGER_RUN = 'FRIGG_TRIGGER_RUN';
|
|
33
|
+
/** A tick that ran out of time sends this to the integration's queue to carry on from its page cursor. */
|
|
34
|
+
const TRIGGER_TICK = 'FRIGG_TRIGGER_TICK';
|
|
35
|
+
|
|
36
|
+
const DEFAULT_TICK_MINUTES = 5;
|
|
37
|
+
const MAX_EVERY_MINUTES = 1440; // longer cadences use a calendar (`at` / `cron`)
|
|
38
|
+
const AT = /^cron\(\S+ \S+ \S+ \S+ \S+ \S+\)$/;
|
|
39
|
+
const OVERLAP = ['skip', 'allow'];
|
|
40
|
+
|
|
41
|
+
const configurableOf = (c) => (c ? (c === true ? {} : c) : false);
|
|
42
|
+
const kindOf = (t) => (t.at !== undefined ? 'at' : t.cron !== undefined ? 'cron' : 'every');
|
|
43
|
+
const recurrenceOf = (r) => (r?.every !== undefined ? { every: r.every } : r?.cron !== undefined ? { cron: r.cron, ...(r.timezone ? { timezone: r.timezone } : {}) } : null);
|
|
44
|
+
|
|
45
|
+
/** A minutes count as the shortest duration string: 90 -> '90m', 60 -> '1h', 1440 -> '1d'. */
|
|
46
|
+
function durationOf(minutes) {
|
|
47
|
+
if (minutes % 1440 === 0) return `${minutes / 1440}d`;
|
|
48
|
+
if (minutes % 60 === 0) return `${minutes / 60}h`;
|
|
49
|
+
return `${minutes}m`;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The tick a trigger runs on, in minutes. `tick` when set. Otherwise: for a constant `every`, that interval; for an
|
|
54
|
+
* `every` given as a function, 5 minutes; for a `cron`, 5 minutes, or less when the cron itself fires more often.
|
|
55
|
+
* A configurable trigger never ticks slower than its floor (or 5 minutes, without one), so the fastest choice it
|
|
56
|
+
* allows can run. `at` triggers have no tick (null): each has its own schedule.
|
|
57
|
+
*/
|
|
58
|
+
function tickMinutesOf(trigger) {
|
|
59
|
+
if (kindOf(trigger) === 'at') return null;
|
|
60
|
+
if (trigger.tick !== undefined) return parseEvery(trigger.tick);
|
|
61
|
+
const configurable = configurableOf(trigger.configurable);
|
|
62
|
+
const floor = configurable ? (configurable.min !== undefined ? parseEvery(configurable.min) : DEFAULT_TICK_MINUTES) : Infinity;
|
|
63
|
+
let natural;
|
|
64
|
+
if (kindOf(trigger) === 'cron') {
|
|
65
|
+
const gap = minGapMinutes({ cron: trigger.cron });
|
|
66
|
+
natural = Math.max(1, Math.min(DEFAULT_TICK_MINUTES, gap ?? DEFAULT_TICK_MINUTES));
|
|
67
|
+
} else {
|
|
68
|
+
natural = typeof trigger.every === 'function' ? DEFAULT_TICK_MINUTES : parseEvery(trigger.every) ?? DEFAULT_TICK_MINUTES;
|
|
69
|
+
}
|
|
70
|
+
return Math.max(1, Math.min(natural, floor ?? DEFAULT_TICK_MINUTES, MAX_EVERY_MINUTES));
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Problems with a trigger declaration, found without an installation (definition time and build time).
|
|
75
|
+
* @returns {string[]}
|
|
76
|
+
*/
|
|
77
|
+
function validateTrigger(name, t) {
|
|
78
|
+
const at = (m) => `${name}: ${m}`;
|
|
79
|
+
if (!t || typeof t !== 'object') return [at('A trigger is an object')];
|
|
80
|
+
const problems = [];
|
|
81
|
+
const kinds = ['every', 'at', 'cron'].filter((k) => t[k] !== undefined);
|
|
82
|
+
if (kinds.length !== 1) return [at('Give exactly one of every, at or cron')];
|
|
83
|
+
if (typeof t.handler !== 'function' && typeof t.handler !== 'string') problems.push(at('handler must be a function or a method name'));
|
|
84
|
+
if (t.overlap !== undefined && !OVERLAP.includes(t.overlap)) problems.push(at(`overlap must be ${OVERLAP.join(' or ')}`));
|
|
85
|
+
if (t.when !== undefined && typeof t.when !== 'function') problems.push(at('when must be a function of the installation config'));
|
|
86
|
+
if (t.tick !== undefined) {
|
|
87
|
+
const tick = parseEvery(t.tick);
|
|
88
|
+
if (tick === null || tick > MAX_EVERY_MINUTES) problems.push(at(`tick "${t.tick}" is not a duration between 1m and 1d`));
|
|
89
|
+
}
|
|
90
|
+
const kind = kinds[0];
|
|
91
|
+
if (kind === 'at') {
|
|
92
|
+
if (typeof t.at !== 'string' || !AT.test(t.at)) problems.push(at(`at "${t.at}" must be an EventBridge cron(...) expression with six fields`));
|
|
93
|
+
if (t.timezone !== undefined && !isTimeZone(t.timezone)) problems.push(at(`Unknown time zone "${t.timezone}"`));
|
|
94
|
+
if (t.configurable) problems.push(at('An at trigger runs at one time for every installation; use cron to let installations choose'));
|
|
95
|
+
if (t.tick !== undefined) problems.push(at('An at trigger has its own schedule; tick does not apply'));
|
|
96
|
+
return problems;
|
|
97
|
+
}
|
|
98
|
+
const configurable = configurableOf(t.configurable);
|
|
99
|
+
if (configurable && configurable.min !== undefined && parseEvery(configurable.min) === null) problems.push(at(`configurable.min "${configurable.min}" is not a duration (15m, 2h, 1d)`));
|
|
100
|
+
if (kind === 'every' && typeof t.every === 'function') return problems;
|
|
101
|
+
const recurrence = recurrenceOf(t);
|
|
102
|
+
for (const p of validateRecurrence(recurrence)) problems.push(at(p));
|
|
103
|
+
if (kind === 'every' && parseEvery(t.every) > MAX_EVERY_MINUTES) problems.push(at(`every "${t.every}" is longer than 1d; use at or cron`));
|
|
104
|
+
if (!problems.length && configurable && configurable.min !== undefined) {
|
|
105
|
+
for (const p of validateRecurrence(recurrence, { min: configurable.min })) problems.push(at(`The default breaks its own minimum: ${p}`));
|
|
106
|
+
}
|
|
107
|
+
return problems;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Problems with every trigger a Definition declares. */
|
|
111
|
+
function validateTriggers(triggers = {}) {
|
|
112
|
+
return Object.entries(triggers || {}).flatMap(([name, t]) => validateTrigger(name, t));
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* What devtools provisions for an integration class: the distinct ticks (minutes) of its `every`/`cron` triggers,
|
|
117
|
+
* including the tick of the triggers it resolves per installation (`Definition.instanceTriggers`), and one entry per
|
|
118
|
+
* `at` trigger. Throws on an invalid declaration, so a bad trigger fails the build rather than the first tick.
|
|
119
|
+
*
|
|
120
|
+
* @returns {{ ticks: number[], at: Array<{ name, at, timezone? }> }}
|
|
121
|
+
*/
|
|
122
|
+
function provisioningOf(IntegrationClass) {
|
|
123
|
+
const Definition = IntegrationClass?.Definition || {};
|
|
124
|
+
const triggers = Definition.triggers || {};
|
|
125
|
+
const problems = validateTriggers(triggers);
|
|
126
|
+
const instance = Definition.instanceTriggers;
|
|
127
|
+
if (instance && parseEvery(instance.tick) === null) problems.push(`instanceTriggers.tick "${instance.tick}" is not a duration`);
|
|
128
|
+
if (problems.length) throw new Error(`Integration "${Definition.name}" declares invalid triggers: ${problems.join('; ')}`);
|
|
129
|
+
const ticks = new Set();
|
|
130
|
+
const at = [];
|
|
131
|
+
for (const [name, t] of Object.entries(triggers)) {
|
|
132
|
+
if (kindOf(t) === 'at') at.push({ name, at: t.at, ...(t.timezone ? { timezone: t.timezone } : {}) });
|
|
133
|
+
else ticks.add(tickMinutesOf(t));
|
|
134
|
+
}
|
|
135
|
+
if (instance) ticks.add(parseEvery(instance.tick));
|
|
136
|
+
return { ticks: [...ticks].sort((a, b) => a - b), at };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Does this class declare any triggers (fixed or per installation)? */
|
|
140
|
+
function hasTriggers(IntegrationClass) {
|
|
141
|
+
const Definition = IntegrationClass?.Definition || {};
|
|
142
|
+
return Object.keys(Definition.triggers || {}).length > 0 || Boolean(Definition.instanceTriggers);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* One trigger, resolved for one installation.
|
|
147
|
+
*
|
|
148
|
+
* @param {string} name
|
|
149
|
+
* @param {Object} t - the declaration (plus `choice` when the caller found the installation's choice elsewhere)
|
|
150
|
+
* @param {Object} config - the installation's config
|
|
151
|
+
* @returns {{ name, kind, recurrence?, at?, tickMinutes, cadence?, overlap, active: boolean, inactive?: string,
|
|
152
|
+
* source: 'definition'|'installation', problems: string[] }}
|
|
153
|
+
* `active: false` when `when` excludes the installation or it paused a configurable trigger.
|
|
154
|
+
* An invalid choice keeps the default and is listed in `problems`, so a bad edit never stops a trigger.
|
|
155
|
+
*/
|
|
156
|
+
function resolveTrigger(name, t, config = {}) {
|
|
157
|
+
const kind = kindOf(t);
|
|
158
|
+
const out = { name, kind, tickMinutes: tickMinutesOf(t), overlap: t.overlap || 'skip', active: true, source: 'definition', problems: [] };
|
|
159
|
+
if (typeof t.when === 'function' && !t.when(config)) return { ...out, active: false, inactive: 'when' };
|
|
160
|
+
if (kind === 'at') return { ...out, at: t.at, timezone: t.timezone };
|
|
161
|
+
|
|
162
|
+
let recurrence = kind === 'every' ? { every: typeof t.every === 'function' ? t.every(config) : t.every } : recurrenceOf(t);
|
|
163
|
+
const configurable = configurableOf(t.configurable);
|
|
164
|
+
const choice = t.choice !== undefined ? t.choice : config?.triggers?.[name];
|
|
165
|
+
if (choice && typeof choice === 'object' && (recurrenceOf(choice) || choice.enabled !== undefined)) {
|
|
166
|
+
if (!configurable) out.problems.push('This trigger is not configurable');
|
|
167
|
+
else {
|
|
168
|
+
const picked = recurrenceOf(choice);
|
|
169
|
+
const issues = picked ? validateRecurrence(picked, { min: configurable.min }) : [];
|
|
170
|
+
if (picked?.every !== undefined && !issues.length && parseEvery(picked.every) > MAX_EVERY_MINUTES) issues.push(`every "${picked.every}" is longer than 1d`);
|
|
171
|
+
if (issues.length) out.problems.push(...issues);
|
|
172
|
+
else {
|
|
173
|
+
if (picked) { recurrence = picked; out.source = 'installation'; }
|
|
174
|
+
if (choice.enabled === false) return { ...out, recurrence, active: false, inactive: 'paused' };
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
const own = validateRecurrence(recurrence);
|
|
179
|
+
if (recurrence.every !== undefined && !own.length && parseEvery(recurrence.every) > MAX_EVERY_MINUTES) own.push(`every "${recurrence.every}" is longer than 1d`);
|
|
180
|
+
if (own.length) return { ...out, recurrence, active: false, inactive: 'invalid', problems: [...out.problems, ...own] };
|
|
181
|
+
out.recurrence = recurrence;
|
|
182
|
+
// An interval runs every n ticks (rounded up to a whole number of ticks).
|
|
183
|
+
if (recurrence.every !== undefined) out.cadence = Math.max(1, Math.ceil(parseEvery(recurrence.every) / out.tickMinutes));
|
|
184
|
+
return out;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** A stable, evenly spread number for an installation id (FNV-1a, 32 bits). */
|
|
188
|
+
function hashId(value) {
|
|
189
|
+
let h = 0x811c9dc5;
|
|
190
|
+
for (const ch of String(value)) {
|
|
191
|
+
h ^= ch.codePointAt(0);
|
|
192
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
193
|
+
}
|
|
194
|
+
return h;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** The start of the tick slot an instant falls in: tick runs are keyed by slot, so a late or repeated tick agrees. */
|
|
198
|
+
function slotStart(timeMs, tickMinutes) {
|
|
199
|
+
const size = tickMinutes * 60000;
|
|
200
|
+
return Math.floor(timeMs / size) * size;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Is a resolved trigger due for this installation at this tick slot?
|
|
205
|
+
*
|
|
206
|
+
* - every: due when `slot mod n == hash(integrationId) mod n` (n = cadence in ticks), so it is stateless and spreads
|
|
207
|
+
* installations evenly across the cadence. scheduledFor is the slot start.
|
|
208
|
+
* - cron: due when it fired in (slot start - tick, slot start]. scheduledFor is that firing, so two ticks in one slot
|
|
209
|
+
* (a redelivery, a retried continuation) name the same run.
|
|
210
|
+
* - at: due whenever its own schedule invoked the tick; scheduledFor is that time, to the minute.
|
|
211
|
+
*
|
|
212
|
+
* @param {Object} resolved - from resolveTrigger
|
|
213
|
+
* @param {{ integrationId, slotMs: number, firing?: (recurrence, from, to) => number|null }} at
|
|
214
|
+
* @returns {{ scheduledFor: string } | null}
|
|
215
|
+
*/
|
|
216
|
+
function dueRun(resolved, { integrationId, slotMs, firing = lastFiringIn }) {
|
|
217
|
+
if (!resolved?.active) return null;
|
|
218
|
+
if (resolved.kind === 'at') return { scheduledFor: new Date(Math.floor(slotMs / 60000) * 60000).toISOString() };
|
|
219
|
+
if (resolved.recurrence.every !== undefined) {
|
|
220
|
+
const n = resolved.cadence;
|
|
221
|
+
const slot = Math.floor(slotMs / (resolved.tickMinutes * 60000));
|
|
222
|
+
return slot % n === hashId(integrationId) % n ? { scheduledFor: new Date(slotMs).toISOString() } : null;
|
|
223
|
+
}
|
|
224
|
+
const fired = firing(resolved.recurrence, slotMs - resolved.tickMinutes * 60000, slotMs);
|
|
225
|
+
return fired === null ? null : { scheduledFor: new Date(fired).toISOString() };
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
module.exports = {
|
|
229
|
+
TRIGGER_RUN,
|
|
230
|
+
TRIGGER_TICK,
|
|
231
|
+
DEFAULT_TICK_MINUTES,
|
|
232
|
+
configurableOf,
|
|
233
|
+
recurrenceOf,
|
|
234
|
+
durationOf,
|
|
235
|
+
tickMinutesOf,
|
|
236
|
+
validateTrigger,
|
|
237
|
+
validateTriggers,
|
|
238
|
+
provisioningOf,
|
|
239
|
+
hasTriggers,
|
|
240
|
+
resolveTrigger,
|
|
241
|
+
hashId,
|
|
242
|
+
slotStart,
|
|
243
|
+
dueRun,
|
|
244
|
+
};
|