@capybara-hq/kernel 0.1.0-rc.24 → 0.1.0-rc.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin/admin.module.js +2 -0
- package/dist/admin/health.service.d.ts +2 -2
- package/dist/admin/health.service.js +6 -2
- package/dist/admin/outbox-dead.service.d.ts +7 -3
- package/dist/admin/outbox-dead.service.js +17 -6
- package/dist/admin/overview.service.d.ts +6 -6
- package/dist/admin/overview.service.js +7 -7
- package/dist/agents/agents.module.js +3 -0
- package/dist/agents/agents.service.d.ts +6 -1
- package/dist/agents/agents.service.js +11 -2
- package/dist/background/wake-rules.d.ts +5 -0
- package/dist/background/wake-rules.js +16 -5
- package/dist/catalog/catalog-import.module.js +0 -4
- package/dist/catalog/catalog-scope.service.d.ts +63 -26
- package/dist/catalog/catalog-scope.service.js +146 -71
- package/dist/catalog/catalog.module.js +4 -3
- package/dist/catalog/catalog.service.d.ts +11 -4
- package/dist/catalog/catalog.service.js +21 -5
- package/dist/catalog/cli.d.ts +4 -3
- package/dist/catalog/cli.js +8 -13
- package/dist/catalog/job-ids.d.ts +8 -20
- package/dist/catalog/job-ids.js +11 -16
- package/dist/catalog/marketplaces.service.d.ts +1 -1
- package/dist/catalog/org-plugin-uploads.d.ts +11 -4
- package/dist/catalog/org-plugin-uploads.js +21 -6
- package/dist/catalog/owned-items.d.ts +6 -13
- package/dist/catalog/owned-items.js +11 -29
- package/dist/catalog/owned-plugin-cleanup.d.ts +8 -17
- package/dist/catalog/owned-plugin-cleanup.js +12 -66
- package/dist/catalog/platform-components.d.ts +56 -34
- package/dist/catalog/platform-components.js +68 -78
- package/dist/catalog/plugin-import.d.ts +28 -25
- package/dist/catalog/plugin-import.js +90 -76
- package/dist/catalog/plugin-upgrade.d.ts +4 -4
- package/dist/catalog/plugin-upgrade.js +7 -19
- package/dist/catalog/scope-jobs.d.ts +59 -0
- package/dist/catalog/scope-jobs.js +65 -0
- package/dist/catalog/scope-sync-workers.d.ts +22 -0
- package/dist/catalog/scope-sync-workers.js +58 -2
- package/dist/catalog/scope-sync.d.ts +40 -65
- package/dist/catalog/scope-sync.js +114 -122
- package/dist/channels/channels.module.d.ts +9 -8
- package/dist/channels/channels.module.js +11 -10
- package/dist/channels/connections.service.d.ts +3 -3
- package/dist/channels/connections.service.js +6 -6
- package/dist/channels/deliver-job.d.ts +10 -3
- package/dist/channels/deliver-job.js +7 -3
- package/dist/channels/inbound.js +2 -2
- package/dist/channels/jobs.d.ts +11 -4
- package/dist/channels/jobs.js +18 -3
- package/dist/channels/outbound.d.ts +5 -2
- package/dist/channels/outbound.js +9 -2
- package/dist/channels/threads.d.ts +4 -3
- package/dist/channels/threads.js +4 -3
- package/dist/conversations/conversations.controller.d.ts +4 -2
- package/dist/conversations/conversations.controller.js +8 -4
- package/dist/conversations/conversations.service.d.ts +54 -9
- package/dist/conversations/conversations.service.js +94 -25
- package/dist/conversations/events.d.ts +27 -0
- package/dist/conversations/events.js +30 -1
- package/dist/conversations/source.d.ts +10 -0
- package/dist/conversations/source.js +17 -0
- package/dist/conversations/tree.d.ts +10 -0
- package/dist/conversations/tree.js +35 -0
- package/dist/db/kernel-schema.d.ts +19 -15
- package/dist/devices/devices.service.d.ts +25 -14
- package/dist/devices/devices.service.js +36 -30
- package/dist/evals/evals.module.js +5 -1
- package/dist/evals/jobs.d.ts +14 -6
- package/dist/evals/jobs.js +21 -19
- package/dist/evals/run-replay.d.ts +13 -2
- package/dist/evals/run-replay.js +28 -1
- package/dist/evals/runs.service.d.ts +5 -6
- package/dist/evals/runs.service.js +11 -33
- package/dist/evals/snapshot.d.ts +7 -1
- package/dist/evals/snapshot.js +2 -1
- package/dist/extensions/hooks.d.ts +22 -3
- package/dist/extensions/hooks.js +15 -1
- package/dist/extensions/lifecycle-hooks.d.ts +99 -0
- package/dist/extensions/lifecycle-hooks.js +273 -0
- package/dist/extensions/org-config.d.ts +13 -1
- package/dist/extensions/org-config.js +19 -2
- package/dist/extensions/permission-view.d.ts +9 -5
- package/dist/extensions/permission-view.js +9 -4
- package/dist/extensions/point-defaults.d.ts +28 -1
- package/dist/extensions/point-defaults.js +44 -13
- package/dist/inbox/inbox.service.d.ts +6 -20
- package/dist/inbox/inbox.service.js +0 -3
- package/dist/inbox/org-notices.d.ts +1 -1
- package/dist/inbox/platform-inbox.service.d.ts +3 -32
- package/dist/inbox/platform-inbox.service.js +5 -54
- package/dist/infra/outbox/outbox.service.d.ts +6 -0
- package/dist/infra/outbox/outbox.service.js +51 -17
- package/dist/infra/outbox/registry.d.ts +176 -0
- package/dist/infra/outbox/registry.js +183 -0
- package/dist/infra/queue/queue-names.d.ts +1 -1
- package/dist/infra/queue/queue-names.js +1 -0
- package/dist/local-devices/device-policy.service.d.ts +131 -26
- package/dist/local-devices/device-policy.service.js +341 -145
- package/dist/local-devices/machines.d.ts +17 -2
- package/dist/local-devices/machines.js +66 -5
- package/dist/local-devices/policy-op-jobs.d.ts +15 -0
- package/dist/local-devices/policy-op-jobs.js +12 -0
- package/dist/local-devices/policy-summary.d.ts +36 -3
- package/dist/local-devices/policy-summary.js +47 -1
- package/dist/orgs/directory-sync.js +1 -1
- package/dist/orgs/members.service.js +1 -1
- package/dist/orgs/offboarding.d.ts +50 -57
- package/dist/orgs/offboarding.js +117 -162
- package/dist/orgs/orgs.module.js +3 -0
- package/dist/orgs/orgs.service.d.ts +11 -5
- package/dist/orgs/orgs.service.js +19 -7
- package/dist/overrides.d.ts +7 -0
- package/dist/overrides.js +4 -0
- package/dist/platform/ops/conversations.ops.js +1 -1
- package/dist/platform/ops/inbox.ops.js +4 -20
- package/dist/platform/ops/platform.ops.d.ts +1 -0
- package/dist/platform/ops/platform.ops.js +6 -3
- package/dist/roles/worker-role.d.ts +10 -5
- package/dist/roles/worker-role.js +19 -9
- package/dist/roles/worker.module.js +2 -4
- package/dist/runtime/context/build.d.ts +3 -0
- package/dist/runtime/context/build.js +15 -0
- package/dist/runtime/context/plugin-context.d.ts +20 -0
- package/dist/runtime/context/plugin-context.js +30 -0
- package/dist/runtime/langgraph-loop.js +4 -0
- package/dist/runtime/loop.d.ts +8 -0
- package/dist/runtime/maintenance.d.ts +9 -4
- package/dist/runtime/maintenance.js +52 -13
- package/dist/runtime/model-call.d.ts +6 -1
- package/dist/runtime/model-call.js +1 -0
- package/dist/runtime/plugin-context.service.d.ts +75 -0
- package/dist/runtime/plugin-context.service.js +220 -0
- package/dist/runtime/run-context.service.d.ts +7 -3
- package/dist/runtime/run-context.service.js +12 -2
- package/dist/runtime/run-control.d.ts +6 -1
- package/dist/runtime/run-control.js +13 -0
- package/dist/runtime/run-finish.d.ts +2 -2
- package/dist/runtime/run-finish.js +11 -6
- package/dist/runtime/run-jobs.d.ts +21 -4
- package/dist/runtime/run-jobs.js +29 -6
- package/dist/runtime/run-orphan-check.job.d.ts +14 -11
- package/dist/runtime/run-orphan-check.job.js +16 -11
- package/dist/runtime/run-queue.d.ts +29 -24
- package/dist/runtime/run-queue.js +118 -57
- package/dist/runtime/run-status.d.ts +12 -0
- package/dist/runtime/run-status.js +21 -0
- package/dist/runtime/run-worker.d.ts +32 -3
- package/dist/runtime/run-worker.js +143 -21
- package/dist/runtime/runtime.module.js +2 -0
- package/dist/runtime/step-log.d.ts +6 -5
- package/dist/runtime/step-log.js +6 -5
- package/dist/runtime/subagents/dispatch.js +1 -0
- package/dist/runtime/subagents/skill-fork.js +3 -0
- package/dist/runtime/usage-report.d.ts +3 -1
- package/dist/runtime/usage-report.js +4 -2
- package/dist/runtime/waits.d.ts +16 -6
- package/dist/runtime/waits.js +37 -15
- package/dist/sandbox/jobs.d.ts +20 -10
- package/dist/sandbox/jobs.js +23 -17
- package/dist/schedules/fire.d.ts +25 -4
- package/dist/schedules/fire.js +87 -13
- package/dist/schedules/job-ids.d.ts +66 -17
- package/dist/schedules/job-ids.js +81 -13
- package/dist/schedules/run-tracker.js +2 -1
- package/dist/schedules/scan.job.d.ts +16 -35
- package/dist/schedules/scan.job.js +38 -147
- package/dist/schedules/scheduled-tasks.service.d.ts +6 -6
- package/dist/schedules/scheduled-tasks.service.js +17 -14
- package/dist/schedules/schedules.module.js +0 -4
- package/dist/skills/catalog.d.ts +7 -3
- package/dist/skills/catalog.js +12 -2
- package/dist/skills/changes.controller.d.ts +1 -1
- package/dist/skills/changes.controller.js +1 -1
- package/dist/tools/approvals.service.d.ts +111 -23
- package/dist/tools/approvals.service.js +326 -77
- package/dist/tools/builtins/sandbox/task-output.js +21 -6
- package/dist/tools/device-op-resolver.d.ts +40 -0
- package/dist/tools/device-op-resolver.js +121 -0
- package/dist/tools/network-asks.d.ts +1 -1
- package/dist/tools/network-asks.js +4 -3
- package/dist/tools/pipeline.d.ts +11 -2
- package/dist/tools/pipeline.js +5 -0
- package/dist/tools/questions.service.d.ts +15 -4
- package/dist/tools/questions.service.js +21 -19
- package/dist/tools/registry.d.ts +8 -1
- package/dist/tools/registry.js +9 -0
- package/dist/tools/tools.module.js +4 -0
- package/package.json +3 -3
- package/src/migrations/0121_drop_platform_migration_runs.sql +12 -0
- package/src/migrations/0122_drop_scheduled_task_runs_rechecked_at.sql +5 -0
- package/src/migrations/0123_pgvector.sql +27 -0
- package/src/migrations/0124_conversation_source.sql +36 -0
- package/src/migrations/0125_conversation_tree_stopped_at.sql +5 -0
- package/dist/catalog/platform-migration.d.ts +0 -115
- package/dist/catalog/platform-migration.js +0 -583
- package/dist/infra/queue/add-job-once.d.ts +0 -14
- package/dist/infra/queue/add-job-once.js +0 -16
|
@@ -1,13 +1,12 @@
|
|
|
1
|
-
import { WorkerHost } from "@nestjs/bullmq";
|
|
2
|
-
import type { Job, Queue, Worker } from "bullmq";
|
|
3
1
|
import { ExtensionLifecycle } from "../extensions/lifecycle.js";
|
|
4
2
|
import { ExtensionRegistry } from "../extensions/registry.js";
|
|
5
3
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
6
4
|
import { TransactionHost } from "../infra/database/transaction-host.js";
|
|
7
5
|
import { LockService } from "../infra/lock/lock.service.js";
|
|
8
|
-
import
|
|
6
|
+
import { OutboxService } from "../infra/outbox/outbox.service.js";
|
|
9
7
|
import type { KernelLogger } from "../log.js";
|
|
10
8
|
import { PluginInstaller } from "./plugin-install.js";
|
|
9
|
+
import { type ScopeJob } from "./scope-jobs.js";
|
|
11
10
|
/**
|
|
12
11
|
* The plugin scope sync (plugin and component management spec §5, phase 1): converges every org
|
|
13
12
|
* to each platform plugin's scope. In phase 1 "on in an org" is the org's plugin install, so for
|
|
@@ -30,55 +29,29 @@ import { PluginInstaller } from "./plugin-install.js";
|
|
|
30
29
|
* super admin's inbox, until a later sync of the pair does not fail) and not retried until the next
|
|
31
30
|
* enqueue. An error outside those steps (a read, the failure write itself) ends the attempt without
|
|
32
31
|
* touching the pair's row, so an older failure stays open.
|
|
33
|
-
* Every worker start enqueues every live item once ({@link PluginScopeSync.enqueueAll}), which
|
|
34
|
-
* also catches a lost enqueue.
|
|
35
32
|
*
|
|
36
33
|
* Jobs (queue {@link PLUGIN_SCOPE_QUEUE}): one per item and scope version
|
|
37
34
|
* (`scope-item-<itemId>-<scope_version>`, every org), one per org (`scope-org-<orgId>`, every item,
|
|
38
|
-
*
|
|
35
|
+
* owed by the org's creation). Both are owed in the outbox by the transaction that changes what
|
|
36
|
+
* they converge to (transactional outbox spec §8.1), so none is lost; a migration that changes a
|
|
37
|
+
* scope inserts its outbox rows itself. A confirmation (import, permissions, org upload) owes the
|
|
38
|
+
* immediate item job after its deploy and, in its commit, a backstop `scope-item-backstop` due
|
|
39
|
+
* five minutes later (ruling R24) that waits, retrying, for the extension to be enabled. Both are removed when done, so the next add with the same
|
|
39
40
|
* id is not swallowed. Pairs of one org are serialized across replicas by a Redis lock per org; the
|
|
40
41
|
* installer's org row lock and its one-live-install rule are the backstop.
|
|
41
42
|
*/
|
|
42
|
-
|
|
43
|
-
export declare const PLUGIN_SCOPE_QUEUE = "cap-kernel-plugin-scope";
|
|
44
|
-
/** The job converging one item in every org. */
|
|
45
|
-
export declare const SCOPE_ITEM_JOB = "scope-item";
|
|
46
|
-
/** The job converging every item in one org. */
|
|
47
|
-
export declare const SCOPE_ORG_JOB = "scope-org";
|
|
48
|
-
/** One item job per scope version (no `:`: item ids are UUIDs). */
|
|
49
|
-
export declare const scopeItemJobId: (itemId: string, version: number) => string;
|
|
50
|
-
/** One org job at a time per org. */
|
|
51
|
-
export declare const scopeOrgJobId: (orgId: string) => string;
|
|
52
|
-
/**
|
|
53
|
-
* The delayed retry `n` of an item job whose extension this replica had not loaded yet (ruling
|
|
54
|
-
* P3-38); deterministic, so replicas deferring the same item add it once.
|
|
55
|
-
*/
|
|
56
|
-
export declare const scopeRetryJobId: (itemId: string, version: number, n: number) => string;
|
|
43
|
+
export { PLUGIN_SCOPE_QUEUE, SCOPE_BACKSTOP_DELAY, SCOPE_BACKSTOP_DELAY_MS, SCOPE_BACKSTOP_JOB, SCOPE_ITEM_JOB, SCOPE_ORG_JOB, type ScopeJob, scopeBackstopJobId, scopeItemJobId, scopeOrgJobId, scopeRetryJobId, } from "./scope-jobs.js";
|
|
57
44
|
/** How many delayed retries an item job gets while its extension is not loaded here (P3-38). */
|
|
58
45
|
export declare const SCOPE_LOAD_RETRIES = 6;
|
|
59
46
|
/** The delay of each such retry. */
|
|
60
47
|
export declare const SCOPE_LOAD_RETRY_DELAY_MS = 5000;
|
|
61
48
|
/** How long a pair waits for this replica's reconcile before deferring (P3-38). */
|
|
62
49
|
export declare const SCOPE_RECONCILE_WAIT_MS = 10000;
|
|
63
|
-
/**
|
|
64
|
-
* The job's data. `retry`: the item job's delayed retry number (P3-38); at
|
|
65
|
-
* {@link SCOPE_LOAD_RETRIES} an extension still not loaded is a failure as before.
|
|
66
|
-
*/
|
|
67
|
-
export type ScopeJob = {
|
|
68
|
-
kind: "item";
|
|
69
|
-
itemId: string;
|
|
70
|
-
retry?: number;
|
|
71
|
-
} | {
|
|
72
|
-
kind: "org";
|
|
73
|
-
orgId: string;
|
|
74
|
-
};
|
|
75
50
|
/**
|
|
76
51
|
* Awaits this replica's extension reconcile for at most `ms` (a slower one carries on in the
|
|
77
52
|
* background); a failure only warns.
|
|
78
53
|
*/
|
|
79
54
|
export declare function reconcileWithin(lifecycle: Pick<ExtensionLifecycle, "reconcile">, reason: string, ms: number, log: Pick<KernelLogger, "warn">): Promise<void>;
|
|
80
|
-
/** The item job of a scope version (what {@link PluginScopeSync.enqueueItem} adds). */
|
|
81
|
-
export declare function scopeItemJob(itemId: string, version: number): OneJob<ScopeJob>;
|
|
82
55
|
/**
|
|
83
56
|
* What a sync of one (item, org) did. `deferred`: the plugin's extension is enabled on the
|
|
84
57
|
* deployment but not loaded on this replica yet (P3-38); the item job retries later.
|
|
@@ -88,7 +61,7 @@ export type ScopeSyncResult = "installed" | "removed" | "needs_config" | "unchan
|
|
|
88
61
|
export declare const PLUGIN_SCOPE_CONCURRENCY = 2;
|
|
89
62
|
export declare class PluginScopeSync {
|
|
90
63
|
private readonly kdb;
|
|
91
|
-
private readonly
|
|
64
|
+
private readonly outbox;
|
|
92
65
|
private readonly installer;
|
|
93
66
|
private readonly registry;
|
|
94
67
|
/** This replica's reconcile, awaited (bounded) before deferring an install (P3-38). */
|
|
@@ -96,23 +69,44 @@ export declare class PluginScopeSync {
|
|
|
96
69
|
private readonly locks;
|
|
97
70
|
private readonly log;
|
|
98
71
|
readonly txHost: TransactionHost;
|
|
99
|
-
|
|
72
|
+
/** The backstop's delay (a test seam; default {@link SCOPE_BACKSTOP_DELAY_MS}). */
|
|
73
|
+
private readonly backstopDelayMs;
|
|
74
|
+
constructor(kdb: KernelDb, outbox: Pick<OutboxService, "add">, installer: PluginInstaller, registry: ExtensionRegistry,
|
|
100
75
|
/** This replica's reconcile, awaited (bounded) before deferring an install (P3-38). */
|
|
101
|
-
lifecycle: ExtensionLifecycle, locks: LockService, log: KernelLogger, txHost: TransactionHost
|
|
76
|
+
lifecycle: ExtensionLifecycle, locks: LockService, log: KernelLogger, txHost: TransactionHost,
|
|
77
|
+
/** The backstop's delay (a test seam; default {@link SCOPE_BACKSTOP_DELAY_MS}). */
|
|
78
|
+
backstopDelayMs?: number);
|
|
102
79
|
private get db();
|
|
103
|
-
/**
|
|
80
|
+
/**
|
|
81
|
+
* Owes the sync of an item in every org, once per scope version (the version read in the
|
|
82
|
+
* caller's transaction, so after its own bump), in the transaction that changes the scope.
|
|
83
|
+
*/
|
|
104
84
|
enqueueItem(itemId: string): Promise<void>;
|
|
105
85
|
/**
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
86
|
+
* Owes the delayed backstop of the item's sync (ruling R24) in the transaction in context: a
|
|
87
|
+
* confirmation owes it before (or with) its deploy, so a crash before its post-deploy add still
|
|
88
|
+
* converges the orgs.
|
|
109
89
|
*/
|
|
110
|
-
|
|
111
|
-
/**
|
|
90
|
+
enqueueBackstop(itemId: string): Promise<void>;
|
|
91
|
+
/** Owes the sync of every item in an org, in the transaction that creates the org. */
|
|
112
92
|
enqueueOrg(orgId: string): Promise<void>;
|
|
113
|
-
/**
|
|
93
|
+
/**
|
|
94
|
+
* One job: every org of an item, or every item of an org; `signal` stops between pairs. A
|
|
95
|
+
* backstop job first checks the item's extension is deployed ({@link assertDeployed}).
|
|
96
|
+
*/
|
|
114
97
|
process(data: ScopeJob, signal?: AbortSignal): Promise<void>;
|
|
115
|
-
/**
|
|
98
|
+
/**
|
|
99
|
+
* A backstop job (ruling R24) runs only once the item's extension is loaded here or enabled on
|
|
100
|
+
* the deployment; otherwise it throws, so BullMQ retries it with its backoff (bounded by its
|
|
101
|
+
* attempts) and no pair records a `catalog.not_deployed` failure that nothing would retry. An item without an
|
|
102
|
+
* extension, or one no longer live (deleted, disabled, still waiting, builtin), passes: its pairs
|
|
103
|
+
* converge (or change nothing) as for an immediate job.
|
|
104
|
+
*/
|
|
105
|
+
private assertDeployed;
|
|
106
|
+
/**
|
|
107
|
+
* Owes the delayed retry `n` of an item job (deterministic id, P3-38) in a transaction of its
|
|
108
|
+
* own: the job's own follow-up, delivered by the relay once due.
|
|
109
|
+
*/
|
|
116
110
|
private enqueueRetry;
|
|
117
111
|
/**
|
|
118
112
|
* Whether the install of the item's `version` may go ahead on this replica (P3-38): its
|
|
@@ -155,22 +149,3 @@ export declare class PluginScopeSync {
|
|
|
155
149
|
private failed;
|
|
156
150
|
private recordFailure;
|
|
157
151
|
}
|
|
158
|
-
/** The consumer of {@link PLUGIN_SCOPE_QUEUE}: {@link PluginScopeSync}. */
|
|
159
|
-
export declare class PluginScopeSyncProcessor extends WorkerHost<Worker<ScopeJob>> {
|
|
160
|
-
private readonly sync;
|
|
161
|
-
private readonly log;
|
|
162
|
-
private readonly stopping;
|
|
163
|
-
constructor(sync: PluginScopeSync, log: Pick<KernelLogger, "error">);
|
|
164
|
-
process(job: Job<ScopeJob>): Promise<void>;
|
|
165
|
-
/** Starts consuming (worker role, after the extensions loaded; built with `autorun: false`). */
|
|
166
|
-
start(): void;
|
|
167
|
-
/**
|
|
168
|
-
* Enqueues every live item's sync once (the worker role, after the start gate opened), in the
|
|
169
|
-
* background; a failure is only logged (the next worker start, or the next change, enqueues).
|
|
170
|
-
*/
|
|
171
|
-
resync(): void;
|
|
172
|
-
/** Stops between pairs and closes the worker. */
|
|
173
|
-
close(): Promise<void>;
|
|
174
|
-
failed(job: Job<ScopeJob> | undefined, err: Error): void;
|
|
175
|
-
error(err: Error): void;
|
|
176
|
-
}
|
|
@@ -7,8 +7,7 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
7
7
|
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
8
8
|
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
9
|
};
|
|
10
|
-
import {
|
|
11
|
-
import { Inject, Injectable } from "@nestjs/common";
|
|
10
|
+
import { Inject, Injectable, Optional } from "@nestjs/common";
|
|
12
11
|
import { ApiError } from "../errors.js";
|
|
13
12
|
import { ExtensionLifecycle } from "../extensions/lifecycle.js";
|
|
14
13
|
import { ExtensionRegistry } from "../extensions/registry.js";
|
|
@@ -17,12 +16,13 @@ import { KernelDb } from "../infra/database/kernel-db.js";
|
|
|
17
16
|
import { TransactionHost } from "../infra/database/transaction-host.js";
|
|
18
17
|
import { Transactional } from "../infra/database/transactional.decorator.js";
|
|
19
18
|
import { LockService } from "../infra/lock/lock.service.js";
|
|
20
|
-
import {
|
|
19
|
+
import { OutboxService } from "../infra/outbox/outbox.service.js";
|
|
21
20
|
import { KERNEL_LOGGER } from "../infra/tokens.js";
|
|
22
21
|
import { connectorTemplatesOn, templateIdOfPath } from "./component-state.js";
|
|
23
22
|
import { isUsable } from "./ownership.js";
|
|
24
23
|
import { PluginInstaller } from "./plugin-install.js";
|
|
25
24
|
import { wantedOn } from "./scope.js";
|
|
25
|
+
import { oweScopeItem, PLUGIN_SCOPE_QUEUE, SCOPE_BACKSTOP_DELAY, SCOPE_BACKSTOP_DELAY_MS, SCOPE_ITEM_JOB, SCOPE_ORG_JOB, scopeOrgJobId, scopeRetryJobId, } from "./scope-jobs.js";
|
|
26
26
|
import { ScopeNoLongerWanted, ScopeStillWanted } from "./scope-state.js";
|
|
27
27
|
/**
|
|
28
28
|
* The plugin scope sync (plugin and component management spec §5, phase 1): converges every org
|
|
@@ -46,30 +46,18 @@ import { ScopeNoLongerWanted, ScopeStillWanted } from "./scope-state.js";
|
|
|
46
46
|
* super admin's inbox, until a later sync of the pair does not fail) and not retried until the next
|
|
47
47
|
* enqueue. An error outside those steps (a read, the failure write itself) ends the attempt without
|
|
48
48
|
* touching the pair's row, so an older failure stays open.
|
|
49
|
-
* Every worker start enqueues every live item once ({@link PluginScopeSync.enqueueAll}), which
|
|
50
|
-
* also catches a lost enqueue.
|
|
51
49
|
*
|
|
52
50
|
* Jobs (queue {@link PLUGIN_SCOPE_QUEUE}): one per item and scope version
|
|
53
51
|
* (`scope-item-<itemId>-<scope_version>`, every org), one per org (`scope-org-<orgId>`, every item,
|
|
54
|
-
*
|
|
52
|
+
* owed by the org's creation). Both are owed in the outbox by the transaction that changes what
|
|
53
|
+
* they converge to (transactional outbox spec §8.1), so none is lost; a migration that changes a
|
|
54
|
+
* scope inserts its outbox rows itself. A confirmation (import, permissions, org upload) owes the
|
|
55
|
+
* immediate item job after its deploy and, in its commit, a backstop `scope-item-backstop` due
|
|
56
|
+
* five minutes later (ruling R24) that waits, retrying, for the extension to be enabled. Both are removed when done, so the next add with the same
|
|
55
57
|
* id is not swallowed. Pairs of one org are serialized across replicas by a Redis lock per org; the
|
|
56
58
|
* installer's org row lock and its one-live-install rule are the backstop.
|
|
57
59
|
*/
|
|
58
|
-
|
|
59
|
-
export const PLUGIN_SCOPE_QUEUE = "cap-kernel-plugin-scope";
|
|
60
|
-
/** The job converging one item in every org. */
|
|
61
|
-
export const SCOPE_ITEM_JOB = "scope-item";
|
|
62
|
-
/** The job converging every item in one org. */
|
|
63
|
-
export const SCOPE_ORG_JOB = "scope-org";
|
|
64
|
-
/** One item job per scope version (no `:`: item ids are UUIDs). */
|
|
65
|
-
export const scopeItemJobId = (itemId, version) => `scope-item-${itemId}-${version}`;
|
|
66
|
-
/** One org job at a time per org. */
|
|
67
|
-
export const scopeOrgJobId = (orgId) => `scope-org-${orgId}`;
|
|
68
|
-
/**
|
|
69
|
-
* The delayed retry `n` of an item job whose extension this replica had not loaded yet (ruling
|
|
70
|
-
* P3-38); deterministic, so replicas deferring the same item add it once.
|
|
71
|
-
*/
|
|
72
|
-
export const scopeRetryJobId = (itemId, version, n) => `scope-item-${itemId}-${version}-r${n}`;
|
|
60
|
+
export { PLUGIN_SCOPE_QUEUE, SCOPE_BACKSTOP_DELAY, SCOPE_BACKSTOP_DELAY_MS, SCOPE_BACKSTOP_JOB, SCOPE_ITEM_JOB, SCOPE_ORG_JOB, scopeBackstopJobId, scopeItemJobId, scopeOrgJobId, scopeRetryJobId, } from "./scope-jobs.js";
|
|
73
61
|
/** How many delayed retries an item job gets while its extension is not loaded here (P3-38). */
|
|
74
62
|
export const SCOPE_LOAD_RETRIES = 6;
|
|
75
63
|
/** The delay of each such retry. */
|
|
@@ -98,19 +86,6 @@ export async function reconcileWithin(lifecycle, reason, ms, log) {
|
|
|
98
86
|
clearTimeout(timer);
|
|
99
87
|
}
|
|
100
88
|
}
|
|
101
|
-
/** The item job of a scope version (what {@link PluginScopeSync.enqueueItem} adds). */
|
|
102
|
-
export function scopeItemJob(itemId, version) {
|
|
103
|
-
return {
|
|
104
|
-
name: SCOPE_ITEM_JOB,
|
|
105
|
-
data: { kind: "item", itemId },
|
|
106
|
-
opts: {
|
|
107
|
-
jobId: scopeItemJobId(itemId, version),
|
|
108
|
-
attempts: 1,
|
|
109
|
-
removeOnComplete: true,
|
|
110
|
-
removeOnFail: true,
|
|
111
|
-
},
|
|
112
|
-
};
|
|
113
|
-
}
|
|
114
89
|
/** How long one org's pairs wait for each other (an install can take a while). */
|
|
115
90
|
const ORG_LOCK_TTL_MS = 60_000;
|
|
116
91
|
const ORG_LOCK_WAIT_MS = 10 * 60_000;
|
|
@@ -119,62 +94,67 @@ export const PLUGIN_SCOPE_CONCURRENCY = 2;
|
|
|
119
94
|
const LIVE = ["installing", "installed", "upgrading"];
|
|
120
95
|
let PluginScopeSync = class PluginScopeSync {
|
|
121
96
|
kdb;
|
|
122
|
-
|
|
97
|
+
outbox;
|
|
123
98
|
installer;
|
|
124
99
|
registry;
|
|
125
100
|
lifecycle;
|
|
126
101
|
locks;
|
|
127
102
|
log;
|
|
128
103
|
txHost;
|
|
129
|
-
|
|
104
|
+
backstopDelayMs;
|
|
105
|
+
constructor(kdb, outbox, installer, registry,
|
|
130
106
|
/** This replica's reconcile, awaited (bounded) before deferring an install (P3-38). */
|
|
131
|
-
lifecycle, locks, log, txHost
|
|
107
|
+
lifecycle, locks, log, txHost,
|
|
108
|
+
/** The backstop's delay (a test seam; default {@link SCOPE_BACKSTOP_DELAY_MS}). */
|
|
109
|
+
backstopDelayMs = SCOPE_BACKSTOP_DELAY_MS) {
|
|
132
110
|
this.kdb = kdb;
|
|
133
|
-
this.
|
|
111
|
+
this.outbox = outbox;
|
|
134
112
|
this.installer = installer;
|
|
135
113
|
this.registry = registry;
|
|
136
114
|
this.lifecycle = lifecycle;
|
|
137
115
|
this.locks = locks;
|
|
138
116
|
this.log = log;
|
|
139
117
|
this.txHost = txHost;
|
|
118
|
+
this.backstopDelayMs = backstopDelayMs;
|
|
140
119
|
}
|
|
141
120
|
get db() {
|
|
142
121
|
return this.kdb.db;
|
|
143
122
|
}
|
|
144
|
-
/**
|
|
123
|
+
/**
|
|
124
|
+
* Owes the sync of an item in every org, once per scope version (the version read in the
|
|
125
|
+
* caller's transaction, so after its own bump), in the transaction that changes the scope.
|
|
126
|
+
*/
|
|
145
127
|
async enqueueItem(itemId) {
|
|
146
|
-
|
|
147
|
-
.selectFrom("catalog_items")
|
|
148
|
-
.select("scope_version")
|
|
149
|
-
.where("id", "=", itemId)
|
|
150
|
-
.executeTakeFirst();
|
|
151
|
-
if (!row)
|
|
152
|
-
return;
|
|
153
|
-
const job = scopeItemJob(itemId, row.scope_version);
|
|
154
|
-
await this.queue.add(job.name, job.data, job.opts);
|
|
128
|
+
await oweScopeItem(this.outbox, this.db, itemId);
|
|
155
129
|
}
|
|
156
130
|
/**
|
|
157
|
-
*
|
|
158
|
-
*
|
|
159
|
-
*
|
|
131
|
+
* Owes the delayed backstop of the item's sync (ruling R24) in the transaction in context: a
|
|
132
|
+
* confirmation owes it before (or with) its deploy, so a crash before its post-deploy add still
|
|
133
|
+
* converges the orgs.
|
|
160
134
|
*/
|
|
161
|
-
async
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
.
|
|
165
|
-
|
|
166
|
-
.where("builtin", "=", false)
|
|
167
|
-
.orderBy("id")
|
|
168
|
-
.execute();
|
|
169
|
-
for (const i of items)
|
|
170
|
-
await this.enqueueItem(i.id);
|
|
135
|
+
async enqueueBackstop(itemId) {
|
|
136
|
+
await oweScopeItem(this.outbox, this.db, itemId, {
|
|
137
|
+
backstop: true,
|
|
138
|
+
backstopDelayMs: this.backstopDelayMs,
|
|
139
|
+
});
|
|
171
140
|
}
|
|
172
|
-
/**
|
|
141
|
+
/** Owes the sync of every item in an org, in the transaction that creates the org. */
|
|
173
142
|
async enqueueOrg(orgId) {
|
|
174
|
-
await this.
|
|
143
|
+
await this.outbox.add({
|
|
144
|
+
queue: PLUGIN_SCOPE_QUEUE,
|
|
145
|
+
name: SCOPE_ORG_JOB,
|
|
146
|
+
jobId: scopeOrgJobId(orgId),
|
|
147
|
+
data: { kind: "org", orgId },
|
|
148
|
+
orgId,
|
|
149
|
+
});
|
|
175
150
|
}
|
|
176
|
-
/**
|
|
151
|
+
/**
|
|
152
|
+
* One job: every org of an item, or every item of an org; `signal` stops between pairs. A
|
|
153
|
+
* backstop job first checks the item's extension is deployed ({@link assertDeployed}).
|
|
154
|
+
*/
|
|
177
155
|
async process(data, signal) {
|
|
156
|
+
if (data.kind === "item" && data.backstop === true)
|
|
157
|
+
await this.assertDeployed(data.itemId);
|
|
178
158
|
const pairs = data.kind === "item"
|
|
179
159
|
? (await this.db
|
|
180
160
|
.selectFrom("orgs")
|
|
@@ -214,7 +194,55 @@ let PluginScopeSync = class PluginScopeSync {
|
|
|
214
194
|
}
|
|
215
195
|
}
|
|
216
196
|
}
|
|
217
|
-
/**
|
|
197
|
+
/**
|
|
198
|
+
* A backstop job (ruling R24) runs only once the item's extension is loaded here or enabled on
|
|
199
|
+
* the deployment; otherwise it throws, so BullMQ retries it with its backoff (bounded by its
|
|
200
|
+
* attempts) and no pair records a `catalog.not_deployed` failure that nothing would retry. An item without an
|
|
201
|
+
* extension, or one no longer live (deleted, disabled, still waiting, builtin), passes: its pairs
|
|
202
|
+
* converge (or change nothing) as for an immediate job.
|
|
203
|
+
*/
|
|
204
|
+
async assertDeployed(itemId) {
|
|
205
|
+
const item = await this.db
|
|
206
|
+
.selectFrom("catalog_items")
|
|
207
|
+
.select([
|
|
208
|
+
"builtin",
|
|
209
|
+
"disabled",
|
|
210
|
+
"deleted_at",
|
|
211
|
+
"awaiting_confirmation",
|
|
212
|
+
"latest_version",
|
|
213
|
+
"platform_synced_version",
|
|
214
|
+
])
|
|
215
|
+
.where("id", "=", itemId)
|
|
216
|
+
.executeTakeFirst();
|
|
217
|
+
if (!item || item.builtin || !isUsable(item))
|
|
218
|
+
return;
|
|
219
|
+
const version = item.platform_synced_version ?? item.latest_version;
|
|
220
|
+
const v = await this.db
|
|
221
|
+
.selectFrom("catalog_item_versions")
|
|
222
|
+
.select("manifest")
|
|
223
|
+
.where("item_id", "=", itemId)
|
|
224
|
+
.where("version", "=", version)
|
|
225
|
+
.executeTakeFirst();
|
|
226
|
+
const m = v?.manifest;
|
|
227
|
+
if (!m?.extension)
|
|
228
|
+
return;
|
|
229
|
+
// loaded here (a config-declared extension has no install record), or enabled on the
|
|
230
|
+
// deployment (a replica that has not loaded it yet defers as P3-38 says)
|
|
231
|
+
if (this.registry.manifests().some((x) => x.name === m.name))
|
|
232
|
+
return;
|
|
233
|
+
const install = await this.db
|
|
234
|
+
.selectFrom("extension_installs")
|
|
235
|
+
.select("desired_state")
|
|
236
|
+
.where("name", "=", m.name)
|
|
237
|
+
.executeTakeFirst();
|
|
238
|
+
if (install?.desired_state === "enabled")
|
|
239
|
+
return;
|
|
240
|
+
throw new Error(`the extension ${m.name} of catalog item ${itemId} is not enabled on the deployment yet; the scope sync backstop is retried`);
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Owes the delayed retry `n` of an item job (deterministic id, P3-38) in a transaction of its
|
|
244
|
+
* own: the job's own follow-up, delivered by the relay once due.
|
|
245
|
+
*/
|
|
218
246
|
async enqueueRetry(itemId, n) {
|
|
219
247
|
const row = await this.db
|
|
220
248
|
.selectFrom("catalog_items")
|
|
@@ -223,12 +251,12 @@ let PluginScopeSync = class PluginScopeSync {
|
|
|
223
251
|
.executeTakeFirst();
|
|
224
252
|
if (!row)
|
|
225
253
|
return;
|
|
226
|
-
await this.
|
|
254
|
+
await this.outbox.add({
|
|
255
|
+
queue: PLUGIN_SCOPE_QUEUE,
|
|
256
|
+
name: SCOPE_ITEM_JOB,
|
|
227
257
|
jobId: scopeRetryJobId(itemId, row.scope_version, n),
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
removeOnComplete: true,
|
|
231
|
-
removeOnFail: true,
|
|
258
|
+
data: { kind: "item", itemId, retry: n },
|
|
259
|
+
runAt: new Date(Date.now() + SCOPE_LOAD_RETRY_DELAY_MS),
|
|
232
260
|
});
|
|
233
261
|
}
|
|
234
262
|
/**
|
|
@@ -509,68 +537,32 @@ let PluginScopeSync = class PluginScopeSync {
|
|
|
509
537
|
.execute();
|
|
510
538
|
}
|
|
511
539
|
};
|
|
540
|
+
__decorate([
|
|
541
|
+
Transactional({ propagation: "mandatory" })
|
|
542
|
+
], PluginScopeSync.prototype, "enqueueItem", null);
|
|
543
|
+
__decorate([
|
|
544
|
+
Transactional({ propagation: "mandatory" })
|
|
545
|
+
], PluginScopeSync.prototype, "enqueueBackstop", null);
|
|
546
|
+
__decorate([
|
|
547
|
+
Transactional({ propagation: "mandatory" })
|
|
548
|
+
], PluginScopeSync.prototype, "enqueueOrg", null);
|
|
549
|
+
__decorate([
|
|
550
|
+
Transactional()
|
|
551
|
+
], PluginScopeSync.prototype, "enqueueRetry", null);
|
|
512
552
|
__decorate([
|
|
513
553
|
Transactional({ propagation: "requires_new" })
|
|
514
554
|
], PluginScopeSync.prototype, "recordFailure", null);
|
|
515
555
|
PluginScopeSync = __decorate([
|
|
516
556
|
Injectable(),
|
|
517
557
|
__param(0, Inject(KernelDb)),
|
|
518
|
-
__param(1, Inject(
|
|
558
|
+
__param(1, Inject(OutboxService)),
|
|
519
559
|
__param(2, Inject(PluginInstaller)),
|
|
520
560
|
__param(3, Inject(ExtensionRegistry)),
|
|
521
561
|
__param(4, Inject(ExtensionLifecycle)),
|
|
522
562
|
__param(5, Inject(LockService)),
|
|
523
563
|
__param(6, Inject(KERNEL_LOGGER)),
|
|
524
|
-
__param(7, Inject(TransactionHost))
|
|
564
|
+
__param(7, Inject(TransactionHost)),
|
|
565
|
+
__param(8, Optional()),
|
|
566
|
+
__param(8, Inject(SCOPE_BACKSTOP_DELAY))
|
|
525
567
|
], PluginScopeSync);
|
|
526
568
|
export { PluginScopeSync };
|
|
527
|
-
/** The consumer of {@link PLUGIN_SCOPE_QUEUE}: {@link PluginScopeSync}. */
|
|
528
|
-
let PluginScopeSyncProcessor = class PluginScopeSyncProcessor extends WorkerHost {
|
|
529
|
-
sync;
|
|
530
|
-
log;
|
|
531
|
-
stopping = new AbortController();
|
|
532
|
-
constructor(sync, log) {
|
|
533
|
-
super();
|
|
534
|
-
this.sync = sync;
|
|
535
|
-
this.log = log;
|
|
536
|
-
}
|
|
537
|
-
process(job) {
|
|
538
|
-
return runJob(PLUGIN_SCOPE_QUEUE, job, () => this.sync.process(job.data, this.stopping.signal));
|
|
539
|
-
}
|
|
540
|
-
/** Starts consuming (worker role, after the extensions loaded; built with `autorun: false`). */
|
|
541
|
-
start() {
|
|
542
|
-
void this.worker.run().catch((err) => this.error(err));
|
|
543
|
-
}
|
|
544
|
-
/**
|
|
545
|
-
* Enqueues every live item's sync once (the worker role, after the start gate opened), in the
|
|
546
|
-
* background; a failure is only logged (the next worker start, or the next change, enqueues).
|
|
547
|
-
*/
|
|
548
|
-
resync() {
|
|
549
|
-
void this.sync
|
|
550
|
-
.enqueueAll()
|
|
551
|
-
.catch((err) => this.log.error("plugin scope resync failed", { err }));
|
|
552
|
-
}
|
|
553
|
-
/** Stops between pairs and closes the worker. */
|
|
554
|
-
async close() {
|
|
555
|
-
this.stopping.abort();
|
|
556
|
-
await this.worker.close();
|
|
557
|
-
}
|
|
558
|
-
failed(job, err) {
|
|
559
|
-
this.log.error("plugin scope job failed", { name: job?.name, jobId: job?.id, err });
|
|
560
|
-
}
|
|
561
|
-
error(err) {
|
|
562
|
-
this.log.error("plugin scope worker error", { err });
|
|
563
|
-
}
|
|
564
|
-
};
|
|
565
|
-
__decorate([
|
|
566
|
-
OnWorkerEvent("failed")
|
|
567
|
-
], PluginScopeSyncProcessor.prototype, "failed", null);
|
|
568
|
-
__decorate([
|
|
569
|
-
OnWorkerEvent("error")
|
|
570
|
-
], PluginScopeSyncProcessor.prototype, "error", null);
|
|
571
|
-
PluginScopeSyncProcessor = __decorate([
|
|
572
|
-
Injectable(),
|
|
573
|
-
__param(0, Inject(PluginScopeSync)),
|
|
574
|
-
__param(1, Inject(KERNEL_LOGGER))
|
|
575
|
-
], PluginScopeSyncProcessor);
|
|
576
|
-
export { PluginScopeSyncProcessor };
|
|
@@ -9,14 +9,15 @@ import { ChannelOutbound } from "./outbound.js";
|
|
|
9
9
|
* Channels on every role: the org's channel connections, inbound (dedupe and enqueue; the inbound
|
|
10
10
|
* jobs), the outbound delivery, the producer of the channel queue `cap-kernel-channels`, the
|
|
11
11
|
* anonymous guests. The roles' parts are the role modules': the queue's consumer
|
|
12
|
-
* ({@link ChannelWorkersModule}) and the gateway check job are the worker role's, the
|
|
13
|
-
* supervisor and the webhook inbound the gateway role's (`GatewayModule`).
|
|
12
|
+
* ({@link ChannelWorkersModule}) and the gateway check job are the worker role's, the
|
|
13
|
+
* long-connection supervisor and the webhook inbound the gateway role's (`GatewayModule`).
|
|
14
14
|
*/
|
|
15
15
|
export declare class ChannelsModule {
|
|
16
16
|
}
|
|
17
17
|
/**
|
|
18
18
|
* The consumer of `cap-kernel-channels`: deliveries (`DELIVER_JOB`, owed through the outbox or
|
|
19
|
-
* hinted) and inbound jobs (`INBOUND_JOB`),
|
|
19
|
+
* hinted), a connection's catch-up (`RESUME_JOB`, owed) and inbound jobs (`INBOUND_JOB`), all
|
|
20
|
+
* handled by {@link ChannelOutbound}.
|
|
20
21
|
*/
|
|
21
22
|
export declare class ChannelsProcessor extends WorkerHost<Worker<ChannelJobData>> {
|
|
22
23
|
private readonly outbound;
|
|
@@ -32,11 +33,11 @@ export declare class ChannelsProcessor extends WorkerHost<Worker<ChannelJobData>
|
|
|
32
33
|
*/
|
|
33
34
|
export declare function channelsProcessorOptions(config: Pick<KernelConfig, "runtime" | "channels">): ProcessorWorkerOptions;
|
|
34
35
|
/**
|
|
35
|
-
* The consumer of the channel queue (concurrency `channels.deliveryConcurrency`, the run queue's
|
|
36
|
-
* lock and stalled detection, the delivery backoff). Imported only by the worker role
|
|
37
|
-
* (`WorkerModule`), so other roles get no processor and no BullMQ worker. It never starts by
|
|
38
|
-
* the worker role (`WorkerRole`) starts it after the extension load;
|
|
39
|
-
* on stop.
|
|
36
|
+
* The consumer of the channel queue (concurrency `channels.deliveryConcurrency`, the run queue's
|
|
37
|
+
* job lock and stalled detection, the delivery backoff). Imported only by the worker role
|
|
38
|
+
* (`WorkerModule`), so other roles get no processor and no BullMQ worker. It never starts by
|
|
39
|
+
* itself: the worker role (`WorkerRole`) starts it after the extension load;
|
|
40
|
+
* `ChannelOutbound.close` closes it on stop.
|
|
40
41
|
*/
|
|
41
42
|
export declare class ChannelWorkersModule {
|
|
42
43
|
static forRoot(config: KernelConfig): DynamicModule;
|
|
@@ -26,7 +26,7 @@ import { SchedulesModule } from "../schedules/schedules.module.js";
|
|
|
26
26
|
import { ToolsModule } from "../tools/tools.module.js";
|
|
27
27
|
import { ChannelConnections } from "./connections.service.js";
|
|
28
28
|
import { ChannelInbound } from "./inbound.js";
|
|
29
|
-
import { CHANNEL_QUEUE_NAME, ChannelJobs, DELIVER_JOB, INBOUND_JOB, } from "./jobs.js";
|
|
29
|
+
import { CHANNEL_QUEUE_NAME, ChannelJobs, DELIVER_JOB, INBOUND_JOB, RESUME_JOB, } from "./jobs.js";
|
|
30
30
|
import { ChannelOutbound, deliverBackoff } from "./outbound.js";
|
|
31
31
|
import { AnonymousGuestLock, AnonymousGuests } from "./threads.js";
|
|
32
32
|
/** The channel queue's registration (its producer here; the admin health reads its counts). */
|
|
@@ -35,8 +35,8 @@ const CHANNEL_QUEUE_MODULE = QueueModule.registerQueue(CHANNEL_QUEUE_NAME);
|
|
|
35
35
|
* Channels on every role: the org's channel connections, inbound (dedupe and enqueue; the inbound
|
|
36
36
|
* jobs), the outbound delivery, the producer of the channel queue `cap-kernel-channels`, the
|
|
37
37
|
* anonymous guests. The roles' parts are the role modules': the queue's consumer
|
|
38
|
-
* ({@link ChannelWorkersModule}) and the gateway check job are the worker role's, the
|
|
39
|
-
* supervisor and the webhook inbound the gateway role's (`GatewayModule`).
|
|
38
|
+
* ({@link ChannelWorkersModule}) and the gateway check job are the worker role's, the
|
|
39
|
+
* long-connection supervisor and the webhook inbound the gateway role's (`GatewayModule`).
|
|
40
40
|
*/
|
|
41
41
|
let ChannelsModule = class ChannelsModule {
|
|
42
42
|
};
|
|
@@ -77,7 +77,8 @@ ChannelsModule = __decorate([
|
|
|
77
77
|
export { ChannelsModule };
|
|
78
78
|
/**
|
|
79
79
|
* The consumer of `cap-kernel-channels`: deliveries (`DELIVER_JOB`, owed through the outbox or
|
|
80
|
-
* hinted) and inbound jobs (`INBOUND_JOB`),
|
|
80
|
+
* hinted), a connection's catch-up (`RESUME_JOB`, owed) and inbound jobs (`INBOUND_JOB`), all
|
|
81
|
+
* handled by {@link ChannelOutbound}.
|
|
81
82
|
*/
|
|
82
83
|
let ChannelsProcessor = class ChannelsProcessor extends WorkerHost {
|
|
83
84
|
outbound;
|
|
@@ -86,7 +87,7 @@ let ChannelsProcessor = class ChannelsProcessor extends WorkerHost {
|
|
|
86
87
|
this.outbound = outbound;
|
|
87
88
|
}
|
|
88
89
|
process(job, token) {
|
|
89
|
-
if (job.name !== DELIVER_JOB && job.name !== INBOUND_JOB) {
|
|
90
|
+
if (job.name !== DELIVER_JOB && job.name !== INBOUND_JOB && job.name !== RESUME_JOB) {
|
|
90
91
|
return Promise.reject(new UnrecoverableError(`unknown channel job "${job.name}"`));
|
|
91
92
|
}
|
|
92
93
|
return runJob(CHANNEL_QUEUE_NAME, job, () => this.outbound.process(job, token));
|
|
@@ -124,11 +125,11 @@ export function channelsProcessorOptions(config) {
|
|
|
124
125
|
};
|
|
125
126
|
}
|
|
126
127
|
/**
|
|
127
|
-
* The consumer of the channel queue (concurrency `channels.deliveryConcurrency`, the run queue's
|
|
128
|
-
* lock and stalled detection, the delivery backoff). Imported only by the worker role
|
|
129
|
-
* (`WorkerModule`), so other roles get no processor and no BullMQ worker. It never starts by
|
|
130
|
-
* the worker role (`WorkerRole`) starts it after the extension load;
|
|
131
|
-
* on stop.
|
|
128
|
+
* The consumer of the channel queue (concurrency `channels.deliveryConcurrency`, the run queue's
|
|
129
|
+
* job lock and stalled detection, the delivery backoff). Imported only by the worker role
|
|
130
|
+
* (`WorkerModule`), so other roles get no processor and no BullMQ worker. It never starts by
|
|
131
|
+
* itself: the worker role (`WorkerRole`) starts it after the extension load;
|
|
132
|
+
* `ChannelOutbound.close` closes it on stop.
|
|
132
133
|
*/
|
|
133
134
|
let ChannelWorkersModule = ChannelWorkersModule_1 = class ChannelWorkersModule {
|
|
134
135
|
static forRoot(config) {
|
|
@@ -123,7 +123,7 @@ export declare class ChannelConnections implements OnApplicationShutdown {
|
|
|
123
123
|
private readonly registry;
|
|
124
124
|
private readonly bus;
|
|
125
125
|
private readonly redis;
|
|
126
|
-
/** A connection that can deliver again
|
|
126
|
+
/** A connection that can deliver again owes its conversations' catch-up (`oweResume`). */
|
|
127
127
|
private readonly jobs;
|
|
128
128
|
/** `channels.webhookBaseUrl`: without it webhook mode is unavailable. */
|
|
129
129
|
private readonly config;
|
|
@@ -138,8 +138,8 @@ export declare class ChannelConnections implements OnApplicationShutdown {
|
|
|
138
138
|
constructor(db: KernelDb,
|
|
139
139
|
/** The org row lock (orgs/org-lock.ts): joins the transaction of the write. */
|
|
140
140
|
orgLock: Pick<OrgLock, "lockAndAuthorize">, keyring: Keyring, registry: Pick<ExtensionRegistry, "list">, bus: ExtensionContext["bus"], redis: Pick<Redis, "del">,
|
|
141
|
-
/** A connection that can deliver again
|
|
142
|
-
jobs: Pick<ChannelJobs, "
|
|
141
|
+
/** A connection that can deliver again owes its conversations' catch-up (`oweResume`). */
|
|
142
|
+
jobs: Pick<ChannelJobs, "oweResume">,
|
|
143
143
|
/** `channels.webhookBaseUrl`: without it webhook mode is unavailable. */
|
|
144
144
|
config: Pick<KernelConfigService, "get">, events: Pick<KernelEvents, "on">,
|
|
145
145
|
/** Secret values handled here (written or decrypted) are redacted from the logs (`addOrgSecrets`). */
|
|
@@ -154,7 +154,7 @@ let ChannelConnections = class ChannelConnections {
|
|
|
154
154
|
constructor(db,
|
|
155
155
|
/** The org row lock (orgs/org-lock.ts): joins the transaction of the write. */
|
|
156
156
|
orgLock, keyring, registry, bus, redis,
|
|
157
|
-
/** A connection that can deliver again
|
|
157
|
+
/** A connection that can deliver again owes its conversations' catch-up (`oweResume`). */
|
|
158
158
|
jobs,
|
|
159
159
|
/** `channels.webhookBaseUrl`: without it webhook mode is unavailable. */
|
|
160
160
|
config, events,
|
|
@@ -546,11 +546,6 @@ let ChannelConnections = class ChannelConnections {
|
|
|
546
546
|
}
|
|
547
547
|
// A mode switch also reconnects: the holder drops the long connection when going to webhook (F9)
|
|
548
548
|
await this.changed(id, rotated !== undefined || patch.status !== undefined || modeChanged);
|
|
549
|
-
if ((rotated || patch.status === "active") && row.status === "active") {
|
|
550
|
-
await this.jobs
|
|
551
|
-
.resumeConnection(id)
|
|
552
|
-
.catch((err) => this.log.warn("cannot resume channel deliveries", { err }));
|
|
553
|
-
}
|
|
554
549
|
return { channel: await this.viewOf(row), verifySkipped };
|
|
555
550
|
}
|
|
556
551
|
/** The rows of {@link update}, in their own transaction (the caller publishes after the commit). */
|
|
@@ -592,6 +587,11 @@ let ChannelConnections = class ChannelConnections {
|
|
|
592
587
|
.executeTakeFirst();
|
|
593
588
|
if (!updated)
|
|
594
589
|
throw notFound();
|
|
590
|
+
// can deliver again (re-enabled, or new credentials while active): the conversations that fell
|
|
591
|
+
// behind meanwhile are caught up by a job owed here (transactional outbox spec §3.1)
|
|
592
|
+
if ((rotated || patch.status === "active") && updated.status === "active") {
|
|
593
|
+
await this.jobs.oweResume(id, orgId, updated.updated_at);
|
|
594
|
+
}
|
|
595
595
|
await audit(this.db.db, {
|
|
596
596
|
actorAccountId,
|
|
597
597
|
...(actorTokenId !== undefined && { actorTokenId }),
|