@capybara-hq/kernel 0.1.0-rc.23 → 0.1.0-rc.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bootstrap.json +25 -25
- package/dist/admin/admin.module.d.ts +2 -1
- package/dist/admin/admin.module.js +9 -3
- package/dist/admin/health.service.d.ts +2 -2
- package/dist/admin/health.service.js +6 -2
- package/dist/admin/outbox-dead.controller.d.ts +23 -0
- package/dist/admin/outbox-dead.controller.js +62 -0
- package/dist/admin/outbox-dead.schemas.d.ts +8 -0
- package/dist/admin/outbox-dead.schemas.js +18 -0
- package/dist/admin/outbox-dead.service.d.ts +66 -0
- package/dist/admin/outbox-dead.service.js +179 -0
- package/dist/admin/overview.service.d.ts +40 -5
- package/dist/admin/overview.service.js +41 -7
- package/dist/agents/agents.module.js +3 -0
- package/dist/agents/agents.service.d.ts +6 -1
- package/dist/agents/agents.service.js +11 -2
- package/dist/agents/from-template.js +1 -1
- package/dist/background/finish.d.ts +5 -2
- package/dist/background/finish.js +14 -3
- package/dist/background/launch.d.ts +2 -1
- package/dist/background/launch.js +5 -2
- package/dist/background/store.d.ts +8 -0
- package/dist/background/store.js +24 -0
- package/dist/catalog/admin-catalog.controller.d.ts +21 -2
- package/dist/catalog/admin-catalog.controller.js +45 -6
- package/dist/catalog/admin-catalog.schemas.d.ts +5 -0
- package/dist/catalog/admin-catalog.schemas.js +8 -0
- package/dist/catalog/catalog-import.module.js +0 -4
- package/dist/catalog/catalog-scope.service.d.ts +63 -26
- package/dist/catalog/catalog-scope.service.js +146 -71
- package/dist/catalog/catalog.module.d.ts +2 -1
- package/dist/catalog/catalog.module.js +9 -4
- package/dist/catalog/catalog.service.d.ts +11 -4
- package/dist/catalog/catalog.service.js +21 -5
- package/dist/catalog/cli.d.ts +4 -3
- package/dist/catalog/cli.js +8 -13
- package/dist/catalog/job-ids.d.ts +8 -20
- package/dist/catalog/job-ids.js +11 -16
- package/dist/catalog/marketplaces.service.d.ts +1 -1
- package/dist/catalog/org-plugin-uploads.d.ts +11 -4
- package/dist/catalog/org-plugin-uploads.js +21 -6
- package/dist/catalog/owned-items.d.ts +6 -13
- package/dist/catalog/owned-items.js +11 -29
- package/dist/catalog/owned-plugin-cleanup.d.ts +8 -17
- package/dist/catalog/owned-plugin-cleanup.js +12 -66
- package/dist/catalog/platform-components.d.ts +56 -34
- package/dist/catalog/platform-components.js +68 -78
- package/dist/catalog/platform-import-update.d.ts +50 -0
- package/dist/catalog/platform-import-update.js +116 -0
- package/dist/catalog/plugin-import.d.ts +64 -26
- package/dist/catalog/plugin-import.js +113 -77
- package/dist/catalog/plugin-upgrade.d.ts +22 -4
- package/dist/catalog/plugin-upgrade.js +26 -19
- package/dist/catalog/scope-jobs.d.ts +59 -0
- package/dist/catalog/scope-jobs.js +65 -0
- package/dist/catalog/scope-sync-workers.d.ts +22 -0
- package/dist/catalog/scope-sync-workers.js +58 -2
- package/dist/catalog/scope-sync.d.ts +40 -65
- package/dist/catalog/scope-sync.js +114 -122
- package/dist/channels/channels.module.d.ts +5 -2
- package/dist/channels/channels.module.js +9 -2
- package/dist/channels/deliver-job.d.ts +25 -0
- package/dist/channels/deliver-job.js +25 -0
- package/dist/channels/inbound.d.ts +1 -1
- package/dist/channels/inbound.js +4 -17
- package/dist/channels/jobs.d.ts +28 -18
- package/dist/channels/jobs.js +63 -26
- package/dist/channels/threads.d.ts +4 -3
- package/dist/channels/threads.js +4 -3
- package/dist/config/schema.d.ts +0 -3
- package/dist/config/schema.js +11 -20
- package/dist/conversations/agent-tree.d.ts +16 -0
- package/dist/conversations/agent-tree.js +36 -0
- package/dist/conversations/conversations.controller.js +4 -2
- package/dist/conversations/conversations.service.d.ts +61 -19
- package/dist/conversations/conversations.service.js +69 -43
- package/dist/conversations/event-log.d.ts +7 -10
- package/dist/conversations/event-log.js +24 -15
- package/dist/conversations/events.d.ts +21 -0
- package/dist/conversations/events.js +26 -2
- package/dist/conversations/queue.service.d.ts +8 -3
- package/dist/conversations/queue.service.js +37 -13
- package/dist/conversations/source.d.ts +10 -0
- package/dist/conversations/source.js +17 -0
- package/dist/conversations/titles.d.ts +21 -8
- package/dist/conversations/titles.js +42 -15
- package/dist/conversations/tree.d.ts +10 -0
- package/dist/conversations/tree.js +35 -0
- package/dist/db/kernel-schema.d.ts +35 -22
- package/dist/devices/devices.service.d.ts +23 -11
- package/dist/devices/devices.service.js +34 -26
- package/dist/errors.d.ts +11 -0
- package/dist/errors.js +11 -0
- package/dist/evals/run-replay.d.ts +13 -2
- package/dist/evals/run-replay.js +42 -9
- package/dist/evals/snapshot.d.ts +7 -1
- package/dist/evals/snapshot.js +2 -1
- package/dist/extensions/extensions.module.js +3 -0
- package/dist/extensions/hooks.d.ts +22 -3
- package/dist/extensions/hooks.js +15 -1
- package/dist/extensions/lifecycle-hooks.d.ts +99 -0
- package/dist/extensions/lifecycle-hooks.js +273 -0
- package/dist/extensions/lifecycle.d.ts +6 -1
- package/dist/extensions/lifecycle.js +9 -2
- package/dist/extensions/org-config.d.ts +13 -1
- package/dist/extensions/org-config.js +19 -2
- package/dist/extensions/permission-view.d.ts +9 -5
- package/dist/extensions/permission-view.js +9 -4
- package/dist/extensions/point-defaults.d.ts +28 -1
- package/dist/extensions/point-defaults.js +44 -13
- package/dist/extensions/usage-subscribers.d.ts +43 -0
- package/dist/extensions/usage-subscribers.js +108 -0
- package/dist/http/api.module.js +2 -0
- package/dist/inbox/inbox.service.d.ts +6 -20
- package/dist/inbox/inbox.service.js +0 -3
- package/dist/inbox/org-notices.d.ts +1 -1
- package/dist/inbox/platform-inbox.service.d.ts +3 -32
- package/dist/inbox/platform-inbox.service.js +5 -54
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/infra/cli/kernel-cli.module.js +5 -0
- package/dist/infra/events/kernel-events.d.ts +9 -35
- package/dist/infra/outbox/outbox-relay.module.d.ts +6 -0
- package/dist/infra/outbox/outbox-relay.module.js +18 -0
- package/dist/infra/outbox/outbox.dispatcher.d.ts +103 -0
- package/dist/infra/outbox/outbox.dispatcher.js +310 -0
- package/dist/infra/outbox/outbox.module.d.ts +24 -0
- package/dist/infra/outbox/outbox.module.js +50 -0
- package/dist/infra/outbox/outbox.relay.d.ts +31 -0
- package/dist/infra/outbox/outbox.relay.js +76 -0
- package/dist/infra/outbox/outbox.service.d.ts +51 -0
- package/dist/infra/outbox/outbox.service.js +131 -0
- package/dist/infra/outbox/registry.d.ts +244 -0
- package/dist/infra/outbox/registry.js +234 -0
- package/dist/infra/queue/queue-names.d.ts +8 -0
- package/dist/infra/queue/queue-names.js +17 -0
- package/dist/kernel.module.js +3 -0
- package/dist/local-devices/device-policy.service.d.ts +4 -0
- package/dist/local-devices/device-policy.service.js +61 -3
- package/dist/local-devices/machines.d.ts +13 -5
- package/dist/local-devices/machines.js +45 -7
- package/dist/local-devices/policy-summary.d.ts +44 -1
- package/dist/local-devices/policy-summary.js +69 -0
- package/dist/models/embeddings.d.ts +9 -9
- package/dist/models/embeddings.js +11 -18
- package/dist/models/models.module.js +3 -0
- package/dist/orgs/directory-sync.js +1 -1
- package/dist/orgs/members.service.js +1 -1
- package/dist/orgs/offboarding.d.ts +50 -57
- package/dist/orgs/offboarding.js +117 -162
- package/dist/orgs/orgs.module.js +3 -0
- package/dist/orgs/orgs.service.d.ts +11 -5
- package/dist/orgs/orgs.service.js +22 -6
- package/dist/overrides.d.ts +23 -3
- package/dist/overrides.js +12 -2
- package/dist/platform/exclusions.js +5 -0
- package/dist/platform/ops/admin.ops.js +18 -2
- package/dist/platform/ops/catalog-admin.ops.d.ts +3 -1
- package/dist/platform/ops/catalog-admin.ops.js +32 -4
- package/dist/platform/ops/inbox.ops.js +4 -20
- package/dist/platform/ops/outbox.ops.d.ts +21 -0
- package/dist/platform/ops/outbox.ops.js +125 -0
- package/dist/platform/ops/platform.ops.d.ts +1 -0
- package/dist/platform/ops/platform.ops.js +6 -3
- package/dist/platform/ops/schedules.ops.js +40 -1
- package/dist/platform/platform-ops.module.d.ts +2 -1
- package/dist/platform/platform-ops.module.js +3 -0
- package/dist/platform/route-schemas.js +3 -0
- package/dist/roles/worker-role.d.ts +10 -5
- package/dist/roles/worker-role.js +19 -9
- package/dist/roles/worker.module.js +11 -8
- package/dist/runtime/context/build.d.ts +3 -0
- package/dist/runtime/context/build.js +15 -0
- package/dist/runtime/context/plugin-context.d.ts +20 -0
- package/dist/runtime/context/plugin-context.js +30 -0
- package/dist/runtime/langgraph-loop.js +17 -5
- package/dist/runtime/limits.d.ts +15 -0
- package/dist/runtime/limits.js +25 -0
- package/dist/runtime/loop.d.ts +16 -3
- package/dist/runtime/loop.js +2 -1
- package/dist/runtime/maintenance.d.ts +7 -3
- package/dist/runtime/maintenance.js +52 -12
- package/dist/runtime/model-call.d.ts +20 -17
- package/dist/runtime/model-call.js +15 -24
- package/dist/runtime/plugin-context.service.d.ts +69 -0
- package/dist/runtime/plugin-context.service.js +208 -0
- package/dist/runtime/queue-lost-jobs.d.ts +24 -0
- package/dist/runtime/queue-lost-jobs.js +51 -0
- package/dist/runtime/run-control.d.ts +47 -5
- package/dist/runtime/run-control.js +79 -12
- package/dist/runtime/run-finish.d.ts +39 -12
- package/dist/runtime/run-finish.js +97 -14
- package/dist/runtime/run-jobs.d.ts +95 -13
- package/dist/runtime/run-jobs.js +199 -28
- package/dist/runtime/run-orphan-check.job.d.ts +142 -0
- package/dist/runtime/run-orphan-check.job.js +339 -0
- package/dist/runtime/run-processors.d.ts +16 -9
- package/dist/runtime/run-processors.js +1 -1
- package/dist/runtime/run-queue.d.ts +75 -53
- package/dist/runtime/run-queue.js +189 -263
- package/dist/runtime/run-steps.d.ts +1 -2
- package/dist/runtime/run-steps.js +8 -11
- package/dist/runtime/run-worker.d.ts +26 -16
- package/dist/runtime/run-worker.js +132 -37
- package/dist/runtime/runtime.module.d.ts +2 -3
- package/dist/runtime/runtime.module.js +7 -6
- package/dist/runtime/step-log.d.ts +16 -4
- package/dist/runtime/step-log.js +26 -6
- package/dist/runtime/stream.controller.d.ts +2 -4
- package/dist/runtime/stream.controller.js +5 -11
- package/dist/runtime/subagents/child.js +1 -1
- package/dist/runtime/subagents/definition.d.ts +2 -1
- package/dist/runtime/subagents/definition.js +4 -2
- package/dist/runtime/subagents/deliveries.d.ts +6 -3
- package/dist/runtime/subagents/deliveries.js +16 -7
- package/dist/runtime/subagents/dispatch.d.ts +12 -12
- package/dist/runtime/subagents/dispatch.js +14 -38
- package/dist/runtime/subagents/messages.d.ts +5 -1
- package/dist/runtime/subagents/messages.js +11 -3
- package/dist/runtime/subagents/skill-fork.d.ts +3 -3
- package/dist/runtime/subagents/skill-fork.js +8 -5
- package/dist/runtime/subagents/subagents.service.js +1 -1
- package/dist/runtime/subagents/tree-jobs.d.ts +61 -0
- package/dist/runtime/subagents/tree-jobs.js +86 -0
- package/dist/runtime/usage-report-jobs.d.ts +23 -48
- package/dist/runtime/usage-report-jobs.js +67 -119
- package/dist/runtime/usage-report.d.ts +28 -77
- package/dist/runtime/usage-report.js +35 -213
- package/dist/runtime/waits.d.ts +48 -198
- package/dist/runtime/waits.js +103 -805
- package/dist/sandbox/jobs.d.ts +20 -10
- package/dist/sandbox/jobs.js +23 -17
- package/dist/sandbox/sandbox.service.d.ts +17 -4
- package/dist/sandbox/sandbox.service.js +55 -14
- package/dist/schedules/calendar.d.ts +134 -0
- package/dist/schedules/calendar.js +410 -0
- package/dist/schedules/fire.d.ts +26 -7
- package/dist/schedules/fire.js +91 -22
- package/dist/schedules/job-ids.d.ts +66 -17
- package/dist/schedules/job-ids.js +81 -13
- package/dist/schedules/run-tracker.d.ts +0 -5
- package/dist/schedules/run-tracker.js +2 -14
- package/dist/schedules/scan.job.d.ts +13 -32
- package/dist/schedules/scan.job.js +50 -164
- package/dist/schedules/schedule-math.d.ts +42 -0
- package/dist/schedules/schedule-math.js +52 -0
- package/dist/schedules/scheduled-tasks.service.d.ts +22 -6
- package/dist/schedules/scheduled-tasks.service.js +70 -14
- package/dist/schedules/schedules.controller.d.ts +5 -1
- package/dist/schedules/schedules.controller.js +15 -1
- package/dist/schedules/schedules.module.js +0 -4
- package/dist/schedules/schedules.schemas.d.ts +35 -0
- package/dist/schedules/schedules.schemas.js +27 -0
- package/dist/schedules/store.d.ts +5 -0
- package/dist/schedules/store.js +20 -0
- package/dist/skills/changes.controller.d.ts +1 -1
- package/dist/skills/changes.controller.js +1 -1
- package/dist/tools/approvals.service.d.ts +41 -11
- package/dist/tools/approvals.service.js +108 -14
- package/dist/tools/builtin-names.d.ts +5 -3
- package/dist/tools/builtin-names.js +12 -4
- package/dist/tools/builtins/builtin-tools.d.ts +1 -2
- package/dist/tools/builtins/builtin-tools.js +21 -17
- package/dist/tools/builtins/sandbox/bash.js +9 -3
- package/dist/tools/builtins/sandbox/common.d.ts +18 -1
- package/dist/tools/builtins/sandbox/common.js +14 -1
- package/dist/tools/builtins/sandbox/index.d.ts +2 -1
- package/dist/tools/builtins/sandbox/index.js +4 -1
- package/dist/tools/builtins/sandbox/task-list.d.ts +12 -0
- package/dist/tools/builtins/sandbox/task-list.js +130 -0
- package/dist/tools/builtins/sandbox/task-output.d.ts +8 -1
- package/dist/tools/builtins/sandbox/task-output.js +21 -3
- package/dist/tools/builtins/web/extract.d.ts +2 -4
- package/dist/tools/builtins/web/extract.js +2 -5
- package/dist/tools/pipeline.d.ts +12 -3
- package/dist/tools/pipeline.js +6 -1
- package/dist/tools/questions.service.d.ts +30 -10
- package/dist/tools/questions.service.js +41 -27
- package/dist/tools/registry.d.ts +12 -3
- package/dist/tools/registry.js +13 -3
- package/dist/tools/tools.module.js +4 -3
- package/package.json +3 -3
- package/src/migrations/0118_device_policy_ops.sql +6 -0
- package/src/migrations/0119_outbox.sql +22 -0
- package/src/migrations/0120_drop_usage_report_sweep.sql +7 -0
- package/src/migrations/0121_drop_platform_migration_runs.sql +12 -0
- package/src/migrations/0122_drop_scheduled_task_runs_rechecked_at.sql +5 -0
- package/src/migrations/0123_pgvector.sql +27 -0
- package/src/migrations/0124_conversation_source.sql +36 -0
- package/dist/catalog/platform-migration.d.ts +0 -115
- package/dist/catalog/platform-migration.js +0 -583
- package/dist/infra/queue/add-job-once.d.ts +0 -14
- package/dist/infra/queue/add-job-once.js +0 -16
|
@@ -5,21 +5,9 @@ import { ExtensionHost } from "../extensions/host.js";
|
|
|
5
5
|
import { KernelConfigService } from "../infra/config/kernel-config.service.js";
|
|
6
6
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
7
7
|
import { LockService } from "../infra/lock/lock.service.js";
|
|
8
|
-
import type
|
|
9
|
-
|
|
10
|
-
export {
|
|
11
|
-
/** The sweeper's cron job name (no `:`; every minute, once across replicas). */
|
|
12
|
-
export declare const USAGE_SWEEP_JOB = "usage-report-sweep";
|
|
13
|
-
/** The maintenance scheduler the sweep ran on before it became a `@LockedCron` (retired). */
|
|
14
|
-
export { USAGE_SWEEP_SCHEDULER_ID } from "./job-ids.js";
|
|
15
|
-
/** The sweep leaves the newest events to the fast path (and to transactions still committing). */
|
|
16
|
-
export declare const USAGE_SWEEP_LAG_MS = 30000;
|
|
17
|
-
/**
|
|
18
|
-
* Each sweep re-scans this far behind the watermark: events whose transaction committed after a
|
|
19
|
-
* previous sweep passed their `created_at` are still found. Far inside the dedupe window
|
|
20
|
-
* (`hooks.usageReportKeepHours` ≥ 1 h), so re-enqueued events dedupe on the kept job.
|
|
21
|
-
*/
|
|
22
|
-
export declare const USAGE_SWEEP_OVERLAP_MS: number;
|
|
8
|
+
import { type UsageReportJob } from "./usage-report-jobs.js";
|
|
9
|
+
export { queryUsageSubscribers } from "../extensions/usage-subscribers.js";
|
|
10
|
+
export { modelUsageJobId, USAGE_REPORT_JOB, USAGE_REPORT_QUEUE, type UsageReportJob, UsageReportJobs, UsageReportJobsModule, usageReportJobId, } from "./usage-report-jobs.js";
|
|
23
11
|
/**
|
|
24
12
|
* Of `plugins`, those that are **definitely** unservable right now (I-2, N-3): live replicas report
|
|
25
13
|
* them (a `extension_replica_status` row with a heartbeat newer than `staleSeconds`), no live row is
|
|
@@ -35,14 +23,16 @@ export declare function queryUnservable(db: KernelSchemaDb, plugins: readonly st
|
|
|
35
23
|
*/
|
|
36
24
|
export declare function narrowPending(stored: readonly string[] | undefined, failed: readonly string[]): string[];
|
|
37
25
|
/**
|
|
38
|
-
* Reliable `usage.report` delivery (F11, P-17,
|
|
39
|
-
* runs and full-run eval replays) is delivered at least once to every
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
* `
|
|
43
|
-
* -
|
|
44
|
-
*
|
|
45
|
-
*
|
|
26
|
+
* Reliable `usage.report` delivery (F11, P-17, transactional outbox spec §8.1). Every `usage` event
|
|
27
|
+
* written to the event log (chat runs and full-run eval replays) is delivered at least once to every
|
|
28
|
+
* subscribing plugin.
|
|
29
|
+
* - **Owed with the usage:** the transaction that writes a `usage` event (`StepLog`'s step writes, a
|
|
30
|
+
* title's writes) or a `model_usage` row also writes the outbox row of job
|
|
31
|
+
* `usage-<conversationId>-<seq>` / `usage-model-<rowId>` (`UsageReportJobs`), while the deployment
|
|
32
|
+
* has at least one subscriber (`UsageSubscribers`: from PG, whatever this replica loaded; cached
|
|
33
|
+
* 30 s per replica, cleared on `PLUGINS_CHANGED`). The outbox fast path adds the job after the
|
|
34
|
+
* commit and the relay catches up what it did not. Usage written while no plugin subscribed is
|
|
35
|
+
* never reported: a subscription starts when the plugin is enabled.
|
|
46
36
|
* - **Worker** (worker role, `hooks.usageReportConcurrency` per replica): reads the payload from the log
|
|
47
37
|
* at execution time (the `usage` event, the conversation with `kind`, the run). On the first attempt
|
|
48
38
|
* the targets are the deployment's subscribers the organization enabled (I-2: not this replica's
|
|
@@ -54,49 +44,45 @@ export declare function narrowPending(stored: readonly string[] | undefined, fai
|
|
|
54
44
|
* once (I-2). A drop (N-3) is never silent: after delivering to the other targets the job ends
|
|
55
45
|
* failed with an `UnrecoverableError` whose failedReason names the dropped plugins, plus a
|
|
56
46
|
* throttled warning. Plugins the organization disabled by delivery time are dropped silently (not
|
|
57
|
-
* targets any more). The final failure logs an error (no audit);
|
|
58
|
-
* most 100 000
|
|
59
|
-
*
|
|
60
|
-
* using up
|
|
47
|
+
* targets any more). The final failure logs an error (no audit); finished jobs are kept a day and
|
|
48
|
+
* failed jobs 7 days, at most 100 000 each (the outbox registry's `usage-report` options). Each
|
|
49
|
+
* call runs through the instance's `inflight.run`, limited by `hooks.timeoutMs`; when this
|
|
50
|
+
* replica stops, in-flight calls are aborted and the job goes back to the queue without using up
|
|
51
|
+
* an attempt.
|
|
61
52
|
* Handlers must be idempotent by `eventId` (`<conversationId>/<seq>`, or a `model_usage` row id).
|
|
62
53
|
* Nothing is written to the event log (never model-visible). Compaction calls log usage too (`purpose: "compaction"`) and are reported,
|
|
63
54
|
* as are conversation title calls (`purpose: "title"`, no run: `runId` is "") and `WebFetch`
|
|
64
55
|
* extraction calls (`purpose: "web_fetch"`).
|
|
65
56
|
* Single-step eval replays and judge requests log no usage: not reported (M7).
|
|
66
57
|
* - **Model usage outside conversations** (spec B §5.4): each `ctx.models.embed` call writes a
|
|
67
|
-
* `model_usage` row
|
|
68
|
-
* enqueues job `usage-model-<rowId>` (only while some plugin subscribes); the payload
|
|
58
|
+
* `model_usage` row and owes job `usage-model-<rowId>` with it; the payload
|
|
69
59
|
* (`purpose: "embedding"`, `eventId` = the row id, the calling `extension`, no conversation fields)
|
|
70
60
|
* is read from the row when the job runs, and the row's `reported_at` is set once nothing is left
|
|
71
|
-
* to deliver; a job that gives up after its last attempt sets `report_failed_at`.
|
|
72
|
-
* re-enqueues recent rows that are neither (see {@link sweep}).
|
|
61
|
+
* to deliver; a job that gives up after its last attempt sets `report_failed_at`.
|
|
73
62
|
*/
|
|
74
63
|
export declare class UsageReporter {
|
|
75
64
|
private readonly kdb;
|
|
76
65
|
private readonly queue;
|
|
77
66
|
/**
|
|
78
67
|
* The host's hook registry: handlers of the current local plugin generations (through
|
|
79
|
-
* inflight.run)
|
|
80
|
-
* I/O without handlers; and whether a plugin's current instance is active here.
|
|
68
|
+
* inflight.run); and whether a plugin's current instance is active here.
|
|
81
69
|
*/
|
|
82
70
|
private readonly host;
|
|
83
71
|
private readonly config;
|
|
84
72
|
/** The warning throttle's markers (`capk:usage-warn:<plugin>:<kind>`, SET NX PX). */
|
|
85
73
|
private readonly locks;
|
|
86
74
|
private readonly log;
|
|
87
|
-
private readonly retry;
|
|
88
75
|
private worker;
|
|
89
76
|
private closed;
|
|
90
77
|
private readonly stopping;
|
|
91
78
|
constructor(kdb: KernelDb, queue: Queue<UsageReportJob>,
|
|
92
79
|
/**
|
|
93
80
|
* The host's hook registry: handlers of the current local plugin generations (through
|
|
94
|
-
* inflight.run)
|
|
95
|
-
* I/O without handlers; and whether a plugin's current instance is active here.
|
|
81
|
+
* inflight.run); and whether a plugin's current instance is active here.
|
|
96
82
|
*/
|
|
97
83
|
host: Pick<ExtensionHost, "hooks" | "current">, config: Pick<KernelConfigService, "get">,
|
|
98
84
|
/** The warning throttle's markers (`capk:usage-warn:<plugin>:<kind>`, SET NX PX). */
|
|
99
|
-
locks: Pick<LockService, "lease">, log: Logger
|
|
85
|
+
locks: Pick<LockService, "lease">, log: Logger);
|
|
100
86
|
private get db();
|
|
101
87
|
private get hooks();
|
|
102
88
|
/**
|
|
@@ -116,30 +102,6 @@ export declare class UsageReporter {
|
|
|
116
102
|
* See {@link queryUsageSubscribers}.
|
|
117
103
|
*/
|
|
118
104
|
subscribers(orgId?: string): Promise<Set<string>>;
|
|
119
|
-
private jobOpts;
|
|
120
|
-
/**
|
|
121
|
-
* Enqueues the delivery of one `usage` event. Returns without a job when no plugin subscribes.
|
|
122
|
-
* Never throws: an enqueue failure (e.g. a Redis hiccup) only logs a warning; the sweeper catches up.
|
|
123
|
-
*/
|
|
124
|
-
enqueue(conversationId: string, seq: number): Promise<void>;
|
|
125
|
-
/**
|
|
126
|
-
* One sweep (R13): re-enqueues the `usage` events created in `[watermark - overlap, now - lag]` in
|
|
127
|
-
* pages, then advances the watermark to `now - lag`. With no subscriber it only advances. The first
|
|
128
|
-
* sweep starts at `now - lag - overlap`. Also re-enqueues the `model_usage` rows of the same
|
|
129
|
-
* window that are neither reported nor given up (spec B §5.4),
|
|
130
|
-
* or, with no subscriber, marks those rows reported (nobody to deliver to). Returns the number of
|
|
131
|
-
* events and rows offered to the queue.
|
|
132
|
-
*/
|
|
133
|
-
sweep(opts?: {
|
|
134
|
-
now?: number;
|
|
135
|
-
}): Promise<number>;
|
|
136
|
-
/**
|
|
137
|
-
* The `model_usage` half of {@link sweep}: rows neither reported nor given up that occurred in
|
|
138
|
-
* the same window as the events (`[watermark - overlap, to]`), oldest first. Bounded like the
|
|
139
|
-
* event sweep, so a row whose job failed for good is never offered again, even after its failed
|
|
140
|
-
* job left Redis (no repeated backlog, no second delivery to plugins already served).
|
|
141
|
-
*/
|
|
142
|
-
private sweepModelUsage;
|
|
143
105
|
/**
|
|
144
106
|
* Starts consuming (worker role only): `worker` is the `@Processor` worker of
|
|
145
107
|
* {@link USAGE_REPORT_QUEUE} (built with the run queue's job lock and stalled detection).
|
|
@@ -150,8 +112,8 @@ export declare class UsageReporter {
|
|
|
150
112
|
/** A job failed (the worker's `failed` event): logs retries and the final failure. */
|
|
151
113
|
onFailed(j: Job<UsageReportJob> | undefined, err: Error): void;
|
|
152
114
|
/**
|
|
153
|
-
* Stops:
|
|
154
|
-
*
|
|
115
|
+
* Stops: in-flight calls are aborted (their jobs requeued) and the worker closes. The queue
|
|
116
|
+
* itself closes with the application.
|
|
155
117
|
*/
|
|
156
118
|
stop(): Promise<void>;
|
|
157
119
|
/** The payload of a job: a `usage` event's or a `model_usage` row's; null when it is gone. */
|
|
@@ -163,7 +125,7 @@ export declare class UsageReporter {
|
|
|
163
125
|
private payloadOfUsage;
|
|
164
126
|
/** A `model_usage` job is done (delivered, or nothing left to deliver to): marks the row. */
|
|
165
127
|
private markReported;
|
|
166
|
-
/** A `model_usage` job gave up after its last attempt: the
|
|
128
|
+
/** A `model_usage` job gave up after its last attempt: the row records it. */
|
|
167
129
|
private markFailed;
|
|
168
130
|
/** The payload of one usage event, read from the log; null when the event no longer exists. */
|
|
169
131
|
private payloadOf;
|
|
@@ -183,19 +145,8 @@ export declare class UsageReporter {
|
|
|
183
145
|
private call;
|
|
184
146
|
}
|
|
185
147
|
/**
|
|
186
|
-
*
|
|
187
|
-
*
|
|
188
|
-
*/
|
|
189
|
-
export declare class UsageSweepJob {
|
|
190
|
-
private readonly reporter;
|
|
191
|
-
readonly lockService: LockService;
|
|
192
|
-
readonly log: KernelLogger;
|
|
193
|
-
constructor(reporter: Pick<UsageReporter, "sweep">, lockService: LockService, log: KernelLogger);
|
|
194
|
-
run(): Promise<void>;
|
|
195
|
-
}
|
|
196
|
-
/**
|
|
197
|
-
* `usage.report` delivery: the producer on every role; the consumer and the sweep ({@link UsageSweepJob})
|
|
198
|
-
* on the worker role (`WorkerModule`).
|
|
148
|
+
* `usage.report` delivery: the producer (`UsageReportJobs`) and the queue on every role; the
|
|
149
|
+
* consumer's `@Processor` on the worker role (`WorkerModule`).
|
|
199
150
|
*/
|
|
200
151
|
export declare class UsageReportModule {
|
|
201
152
|
}
|
|
@@ -8,36 +8,25 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
8
8
|
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
9
|
};
|
|
10
10
|
import { getQueueToken } from "@nestjs/bullmq";
|
|
11
|
-
import { Inject, Injectable, Module
|
|
11
|
+
import { Inject, Injectable, Module } from "@nestjs/common";
|
|
12
12
|
import { DelayedError, Job, UnrecoverableError } from "bullmq";
|
|
13
13
|
import { sql } from "kysely";
|
|
14
14
|
import { ExtensionsModule } from "../extensions/extensions.module.js";
|
|
15
15
|
import { deepFreeze, errMsg, HookNeedsConfigError, } from "../extensions/hooks.js";
|
|
16
16
|
import { ExtensionHost } from "../extensions/host.js";
|
|
17
17
|
import { STALE_RECONCILE_PERIODS } from "../extensions/lifecycle-topics.js";
|
|
18
|
+
import { queryUsageSubscribers } from "../extensions/usage-subscribers.js";
|
|
18
19
|
import { KernelConfigService } from "../infra/config/kernel-config.service.js";
|
|
19
20
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
20
21
|
import { LockService } from "../infra/lock/lock.service.js";
|
|
21
|
-
import {
|
|
22
|
+
import { QueueModule } from "../infra/queue/queue.module.js";
|
|
22
23
|
import { KERNEL_LOGGER } from "../infra/tokens.js";
|
|
23
24
|
import { deliveryRowOfChildRun } from "./subagents/child.js";
|
|
24
|
-
import {
|
|
25
|
-
export {
|
|
25
|
+
import { USAGE_REPORT_QUEUE, UsageReportJobsModule, } from "./usage-report-jobs.js";
|
|
26
|
+
export { queryUsageSubscribers } from "../extensions/usage-subscribers.js";
|
|
27
|
+
export { modelUsageJobId, USAGE_REPORT_JOB, USAGE_REPORT_QUEUE, UsageReportJobs, UsageReportJobsModule, usageReportJobId, } from "./usage-report-jobs.js";
|
|
26
28
|
/** Warnings about a plugin that cannot take deliveries are logged at most once per this window. */
|
|
27
29
|
const WARN_WINDOW_MS = 60_000;
|
|
28
|
-
/** The sweeper's cron job name (no `:`; every minute, once across replicas). */
|
|
29
|
-
export const USAGE_SWEEP_JOB = "usage-report-sweep";
|
|
30
|
-
/** The maintenance scheduler the sweep ran on before it became a `@LockedCron` (retired). */
|
|
31
|
-
export { USAGE_SWEEP_SCHEDULER_ID } from "./job-ids.js";
|
|
32
|
-
/** The sweep leaves the newest events to the fast path (and to transactions still committing). */
|
|
33
|
-
export const USAGE_SWEEP_LAG_MS = 30_000;
|
|
34
|
-
/**
|
|
35
|
-
* Each sweep re-scans this far behind the watermark: events whose transaction committed after a
|
|
36
|
-
* previous sweep passed their `created_at` are still found. Far inside the dedupe window
|
|
37
|
-
* (`hooks.usageReportKeepHours` ≥ 1 h), so re-enqueued events dedupe on the kept job.
|
|
38
|
-
*/
|
|
39
|
-
export const USAGE_SWEEP_OVERLAP_MS = 10 * 60_000;
|
|
40
|
-
const SWEEP_PAGE = 500;
|
|
41
30
|
/**
|
|
42
31
|
* Of `plugins`, those that are **definitely** unservable right now (I-2, N-3): live replicas report
|
|
43
32
|
* them (a `extension_replica_status` row with a heartbeat newer than `staleSeconds`), no live row is
|
|
@@ -89,14 +78,16 @@ export function narrowPending(stored, failed) {
|
|
|
89
78
|
return stored === undefined ? [...failed] : failed.filter((p) => stored.includes(p));
|
|
90
79
|
}
|
|
91
80
|
/**
|
|
92
|
-
* Reliable `usage.report` delivery (F11, P-17,
|
|
93
|
-
* runs and full-run eval replays) is delivered at least once to every
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
* `
|
|
97
|
-
* -
|
|
98
|
-
*
|
|
99
|
-
*
|
|
81
|
+
* Reliable `usage.report` delivery (F11, P-17, transactional outbox spec §8.1). Every `usage` event
|
|
82
|
+
* written to the event log (chat runs and full-run eval replays) is delivered at least once to every
|
|
83
|
+
* subscribing plugin.
|
|
84
|
+
* - **Owed with the usage:** the transaction that writes a `usage` event (`StepLog`'s step writes, a
|
|
85
|
+
* title's writes) or a `model_usage` row also writes the outbox row of job
|
|
86
|
+
* `usage-<conversationId>-<seq>` / `usage-model-<rowId>` (`UsageReportJobs`), while the deployment
|
|
87
|
+
* has at least one subscriber (`UsageSubscribers`: from PG, whatever this replica loaded; cached
|
|
88
|
+
* 30 s per replica, cleared on `PLUGINS_CHANGED`). The outbox fast path adds the job after the
|
|
89
|
+
* commit and the relay catches up what it did not. Usage written while no plugin subscribed is
|
|
90
|
+
* never reported: a subscription starts when the plugin is enabled.
|
|
100
91
|
* - **Worker** (worker role, `hooks.usageReportConcurrency` per replica): reads the payload from the log
|
|
101
92
|
* at execution time (the `usage` event, the conversation with `kind`, the run). On the first attempt
|
|
102
93
|
* the targets are the deployment's subscribers the organization enabled (I-2: not this replica's
|
|
@@ -108,22 +99,21 @@ export function narrowPending(stored, failed) {
|
|
|
108
99
|
* once (I-2). A drop (N-3) is never silent: after delivering to the other targets the job ends
|
|
109
100
|
* failed with an `UnrecoverableError` whose failedReason names the dropped plugins, plus a
|
|
110
101
|
* throttled warning. Plugins the organization disabled by delivery time are dropped silently (not
|
|
111
|
-
* targets any more). The final failure logs an error (no audit);
|
|
112
|
-
* most 100 000
|
|
113
|
-
*
|
|
114
|
-
* using up
|
|
102
|
+
* targets any more). The final failure logs an error (no audit); finished jobs are kept a day and
|
|
103
|
+
* failed jobs 7 days, at most 100 000 each (the outbox registry's `usage-report` options). Each
|
|
104
|
+
* call runs through the instance's `inflight.run`, limited by `hooks.timeoutMs`; when this
|
|
105
|
+
* replica stops, in-flight calls are aborted and the job goes back to the queue without using up
|
|
106
|
+
* an attempt.
|
|
115
107
|
* Handlers must be idempotent by `eventId` (`<conversationId>/<seq>`, or a `model_usage` row id).
|
|
116
108
|
* Nothing is written to the event log (never model-visible). Compaction calls log usage too (`purpose: "compaction"`) and are reported,
|
|
117
109
|
* as are conversation title calls (`purpose: "title"`, no run: `runId` is "") and `WebFetch`
|
|
118
110
|
* extraction calls (`purpose: "web_fetch"`).
|
|
119
111
|
* Single-step eval replays and judge requests log no usage: not reported (M7).
|
|
120
112
|
* - **Model usage outside conversations** (spec B §5.4): each `ctx.models.embed` call writes a
|
|
121
|
-
* `model_usage` row
|
|
122
|
-
* enqueues job `usage-model-<rowId>` (only while some plugin subscribes); the payload
|
|
113
|
+
* `model_usage` row and owes job `usage-model-<rowId>` with it; the payload
|
|
123
114
|
* (`purpose: "embedding"`, `eventId` = the row id, the calling `extension`, no conversation fields)
|
|
124
115
|
* is read from the row when the job runs, and the row's `reported_at` is set once nothing is left
|
|
125
|
-
* to deliver; a job that gives up after its last attempt sets `report_failed_at`.
|
|
126
|
-
* re-enqueues recent rows that are neither (see {@link sweep}).
|
|
116
|
+
* to deliver; a job that gives up after its last attempt sets `report_failed_at`.
|
|
127
117
|
*/
|
|
128
118
|
let UsageReporter = class UsageReporter {
|
|
129
119
|
kdb;
|
|
@@ -132,29 +122,23 @@ let UsageReporter = class UsageReporter {
|
|
|
132
122
|
config;
|
|
133
123
|
locks;
|
|
134
124
|
log;
|
|
135
|
-
retry;
|
|
136
125
|
worker;
|
|
137
126
|
closed = false;
|
|
138
127
|
stopping = new AbortController();
|
|
139
128
|
constructor(kdb, queue,
|
|
140
129
|
/**
|
|
141
130
|
* The host's hook registry: handlers of the current local plugin generations (through
|
|
142
|
-
* inflight.run)
|
|
143
|
-
* I/O without handlers; and whether a plugin's current instance is active here.
|
|
131
|
+
* inflight.run); and whether a plugin's current instance is active here.
|
|
144
132
|
*/
|
|
145
133
|
host, config,
|
|
146
134
|
/** The warning throttle's markers (`capk:usage-warn:<plugin>:<kind>`, SET NX PX). */
|
|
147
|
-
locks, log
|
|
148
|
-
attempts: USAGE_REPORT_ATTEMPTS,
|
|
149
|
-
backoffMs: USAGE_REPORT_BACKOFF_MS,
|
|
150
|
-
}) {
|
|
135
|
+
locks, log) {
|
|
151
136
|
this.kdb = kdb;
|
|
152
137
|
this.queue = queue;
|
|
153
138
|
this.host = host;
|
|
154
139
|
this.config = config;
|
|
155
140
|
this.locks = locks;
|
|
156
141
|
this.log = log;
|
|
157
|
-
this.retry = retry;
|
|
158
142
|
queue.on("error", (err) => this.log.error("usage report queue error", { err }));
|
|
159
143
|
}
|
|
160
144
|
get db() {
|
|
@@ -186,141 +170,6 @@ let UsageReporter = class UsageReporter {
|
|
|
186
170
|
subscribers(orgId) {
|
|
187
171
|
return queryUsageSubscribers(this.db, orgId);
|
|
188
172
|
}
|
|
189
|
-
jobOpts(jobId) {
|
|
190
|
-
return usageReportJobOpts(jobId, this.retry, this.config.get().hooks.usageReportKeepHours);
|
|
191
|
-
}
|
|
192
|
-
/**
|
|
193
|
-
* Enqueues the delivery of one `usage` event. Returns without a job when no plugin subscribes.
|
|
194
|
-
* Never throws: an enqueue failure (e.g. a Redis hiccup) only logs a warning; the sweeper catches up.
|
|
195
|
-
*/
|
|
196
|
-
async enqueue(conversationId, seq) {
|
|
197
|
-
if (this.closed)
|
|
198
|
-
return;
|
|
199
|
-
// M-2: no usage.report handler on this replica → no I/O at all. A plugin loaded only on other
|
|
200
|
-
// replicas still gets the event: the sweeper re-offers it within about a minute and a half.
|
|
201
|
-
if (!this.hooks.has("usage.report"))
|
|
202
|
-
return;
|
|
203
|
-
try {
|
|
204
|
-
if ((await this.subscribers()).size === 0)
|
|
205
|
-
return;
|
|
206
|
-
await this.queue.add(USAGE_REPORT_JOB, { conversationId, seq }, this.jobOpts(usageReportJobId(conversationId, seq)));
|
|
207
|
-
}
|
|
208
|
-
catch (err) {
|
|
209
|
-
this.log.warn("cannot enqueue usage report", { conversationId, seq, err });
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
/**
|
|
213
|
-
* One sweep (R13): re-enqueues the `usage` events created in `[watermark - overlap, now - lag]` in
|
|
214
|
-
* pages, then advances the watermark to `now - lag`. With no subscriber it only advances. The first
|
|
215
|
-
* sweep starts at `now - lag - overlap`. Also re-enqueues the `model_usage` rows of the same
|
|
216
|
-
* window that are neither reported nor given up (spec B §5.4),
|
|
217
|
-
* or, with no subscriber, marks those rows reported (nobody to deliver to). Returns the number of
|
|
218
|
-
* events and rows offered to the queue.
|
|
219
|
-
*/
|
|
220
|
-
async sweep(opts = {}) {
|
|
221
|
-
// bounds and keyset as full-precision timestamp text (microseconds), computed with the DB clock
|
|
222
|
-
// unless a test fixes "now" (m-a, m-c)
|
|
223
|
-
const to = opts.now !== undefined
|
|
224
|
-
? new Date(opts.now - USAGE_SWEEP_LAG_MS).toISOString()
|
|
225
|
-
: (await sql `SELECT (now() - make_interval(secs => ${USAGE_SWEEP_LAG_MS / 1000}))::text AS to`.execute(this.db)).rows[0]?.to;
|
|
226
|
-
if (to === undefined)
|
|
227
|
-
return 0;
|
|
228
|
-
const row = await this.db
|
|
229
|
-
.selectFrom("usage_report_sweep")
|
|
230
|
-
.select(sql `watermark::text`.as("watermark"))
|
|
231
|
-
.where("id", "=", 1)
|
|
232
|
-
.executeTakeFirst();
|
|
233
|
-
const watermark = row?.watermark ?? to;
|
|
234
|
-
let offered = 0;
|
|
235
|
-
const subscribed = (await this.subscribers()).size > 0;
|
|
236
|
-
if (subscribed) {
|
|
237
|
-
let after;
|
|
238
|
-
for (;;) {
|
|
239
|
-
let q = this.db
|
|
240
|
-
.selectFrom("conversation_events")
|
|
241
|
-
.select(["conversation_id", "seq", sql `created_at::text`.as("at")])
|
|
242
|
-
.where("type", "=", "usage")
|
|
243
|
-
.where(sql `created_at >= ${watermark}::timestamptz - make_interval(secs => ${USAGE_SWEEP_OVERLAP_MS / 1000})`)
|
|
244
|
-
.where(sql `created_at <= ${to}::timestamptz`);
|
|
245
|
-
if (after) {
|
|
246
|
-
q = q.where(sql `(created_at, conversation_id, seq) > (${after.at}::timestamptz, ${after.conv}, ${after.seq}::bigint)`);
|
|
247
|
-
}
|
|
248
|
-
const page = await q
|
|
249
|
-
.orderBy("created_at")
|
|
250
|
-
.orderBy("conversation_id")
|
|
251
|
-
.orderBy("seq")
|
|
252
|
-
.limit(SWEEP_PAGE)
|
|
253
|
-
.execute();
|
|
254
|
-
if (page.length > 0) {
|
|
255
|
-
await this.queue.addBulk(page.map((r) => ({
|
|
256
|
-
name: USAGE_REPORT_JOB,
|
|
257
|
-
data: { conversationId: r.conversation_id, seq: Number(r.seq) },
|
|
258
|
-
opts: this.jobOpts(usageReportJobId(r.conversation_id, Number(r.seq))),
|
|
259
|
-
})));
|
|
260
|
-
offered += page.length;
|
|
261
|
-
}
|
|
262
|
-
const last = page.at(-1);
|
|
263
|
-
if (!last || page.length < SWEEP_PAGE)
|
|
264
|
-
break;
|
|
265
|
-
after = { at: last.at, conv: last.conversation_id, seq: String(last.seq) };
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
offered += await this.sweepModelUsage(watermark, to, subscribed);
|
|
269
|
-
await this.db
|
|
270
|
-
.insertInto("usage_report_sweep")
|
|
271
|
-
.values({ id: 1, watermark: sql `${to}::timestamptz` })
|
|
272
|
-
.onConflict((oc) => oc.column("id").doUpdateSet({
|
|
273
|
-
watermark: sql `GREATEST(usage_report_sweep.watermark, excluded.watermark)`,
|
|
274
|
-
updated_at: sql `now()`,
|
|
275
|
-
}))
|
|
276
|
-
.execute();
|
|
277
|
-
return offered;
|
|
278
|
-
}
|
|
279
|
-
/**
|
|
280
|
-
* The `model_usage` half of {@link sweep}: rows neither reported nor given up that occurred in
|
|
281
|
-
* the same window as the events (`[watermark - overlap, to]`), oldest first. Bounded like the
|
|
282
|
-
* event sweep, so a row whose job failed for good is never offered again, even after its failed
|
|
283
|
-
* job left Redis (no repeated backlog, no second delivery to plugins already served).
|
|
284
|
-
*/
|
|
285
|
-
async sweepModelUsage(watermark, to, subscribed) {
|
|
286
|
-
if (!subscribed) {
|
|
287
|
-
await this.db
|
|
288
|
-
.updateTable("model_usage")
|
|
289
|
-
.set({ reported_at: sql `now()` })
|
|
290
|
-
.where("reported_at", "is", null)
|
|
291
|
-
.where(sql `occurred_at <= ${to}::timestamptz`)
|
|
292
|
-
.execute();
|
|
293
|
-
return 0;
|
|
294
|
-
}
|
|
295
|
-
let offered = 0;
|
|
296
|
-
let after;
|
|
297
|
-
for (;;) {
|
|
298
|
-
let q = this.db
|
|
299
|
-
.selectFrom("model_usage")
|
|
300
|
-
.select(["id", sql `occurred_at::text`.as("at")])
|
|
301
|
-
.where("reported_at", "is", null)
|
|
302
|
-
.where("report_failed_at", "is", null)
|
|
303
|
-
.where(sql `occurred_at >= ${watermark}::timestamptz - make_interval(secs => ${USAGE_SWEEP_OVERLAP_MS / 1000})`)
|
|
304
|
-
.where(sql `occurred_at <= ${to}::timestamptz`);
|
|
305
|
-
if (after) {
|
|
306
|
-
q = q.where(sql `(occurred_at, id) > (${after.at}::timestamptz, ${after.id}::uuid)`);
|
|
307
|
-
}
|
|
308
|
-
const page = await q.orderBy("occurred_at").orderBy("id").limit(SWEEP_PAGE).execute();
|
|
309
|
-
if (page.length > 0) {
|
|
310
|
-
await this.queue.addBulk(page.map((r) => ({
|
|
311
|
-
name: USAGE_REPORT_JOB,
|
|
312
|
-
data: { usageId: r.id },
|
|
313
|
-
opts: this.jobOpts(modelUsageJobId(r.id)),
|
|
314
|
-
})));
|
|
315
|
-
offered += page.length;
|
|
316
|
-
}
|
|
317
|
-
const last = page.at(-1);
|
|
318
|
-
if (!last || page.length < SWEEP_PAGE)
|
|
319
|
-
break;
|
|
320
|
-
after = { at: last.at, id: last.id };
|
|
321
|
-
}
|
|
322
|
-
return offered;
|
|
323
|
-
}
|
|
324
173
|
/**
|
|
325
174
|
* Starts consuming (worker role only): `worker` is the `@Processor` worker of
|
|
326
175
|
* {@link USAGE_REPORT_QUEUE} (built with the run queue's job lock and stalled detection).
|
|
@@ -360,8 +209,8 @@ let UsageReporter = class UsageReporter {
|
|
|
360
209
|
this.log.warn("usage report attempt failed; retrying", fields);
|
|
361
210
|
}
|
|
362
211
|
/**
|
|
363
|
-
* Stops:
|
|
364
|
-
*
|
|
212
|
+
* Stops: in-flight calls are aborted (their jobs requeued) and the worker closes. The queue
|
|
213
|
+
* itself closes with the application.
|
|
365
214
|
*/
|
|
366
215
|
async stop() {
|
|
367
216
|
this.closed = true;
|
|
@@ -415,7 +264,7 @@ let UsageReporter = class UsageReporter {
|
|
|
415
264
|
.where("reported_at", "is", null)
|
|
416
265
|
.execute();
|
|
417
266
|
}
|
|
418
|
-
/** A `model_usage` job gave up after its last attempt: the
|
|
267
|
+
/** A `model_usage` job gave up after its last attempt: the row records it. */
|
|
419
268
|
async markFailed(data) {
|
|
420
269
|
if (data.usageId === undefined)
|
|
421
270
|
return;
|
|
@@ -677,49 +526,22 @@ UsageReporter = __decorate([
|
|
|
677
526
|
__param(2, Inject(ExtensionHost)),
|
|
678
527
|
__param(3, Inject(KernelConfigService)),
|
|
679
528
|
__param(4, Inject(LockService)),
|
|
680
|
-
__param(5, Inject(KERNEL_LOGGER))
|
|
681
|
-
__param(6, Optional()),
|
|
682
|
-
__param(6, Inject(USAGE_REPORT_RETRY))
|
|
529
|
+
__param(5, Inject(KERNEL_LOGGER))
|
|
683
530
|
], UsageReporter);
|
|
684
531
|
export { UsageReporter };
|
|
532
|
+
/** The usage report queue (one instance per process). */
|
|
533
|
+
const USAGE_REPORT_QUEUE_MODULE = QueueModule.registerQueue(USAGE_REPORT_QUEUE);
|
|
685
534
|
/**
|
|
686
|
-
*
|
|
687
|
-
*
|
|
688
|
-
*/
|
|
689
|
-
let UsageSweepJob = class UsageSweepJob {
|
|
690
|
-
reporter;
|
|
691
|
-
lockService;
|
|
692
|
-
log;
|
|
693
|
-
constructor(reporter, lockService, log) {
|
|
694
|
-
this.reporter = reporter;
|
|
695
|
-
this.lockService = lockService;
|
|
696
|
-
this.log = log;
|
|
697
|
-
}
|
|
698
|
-
async run() {
|
|
699
|
-
await this.reporter.sweep();
|
|
700
|
-
}
|
|
701
|
-
};
|
|
702
|
-
__decorate([
|
|
703
|
-
LockedCron("* * * * *", { name: USAGE_SWEEP_JOB })
|
|
704
|
-
], UsageSweepJob.prototype, "run", null);
|
|
705
|
-
UsageSweepJob = __decorate([
|
|
706
|
-
Injectable(),
|
|
707
|
-
__param(0, Inject(UsageReporter)),
|
|
708
|
-
__param(1, Inject(LockService)),
|
|
709
|
-
__param(2, Inject(KERNEL_LOGGER))
|
|
710
|
-
], UsageSweepJob);
|
|
711
|
-
export { UsageSweepJob };
|
|
712
|
-
/**
|
|
713
|
-
* `usage.report` delivery: the producer on every role; the consumer and the sweep ({@link UsageSweepJob})
|
|
714
|
-
* on the worker role (`WorkerModule`).
|
|
535
|
+
* `usage.report` delivery: the producer (`UsageReportJobs`) and the queue on every role; the
|
|
536
|
+
* consumer's `@Processor` on the worker role (`WorkerModule`).
|
|
715
537
|
*/
|
|
716
538
|
let UsageReportModule = class UsageReportModule {
|
|
717
539
|
};
|
|
718
540
|
UsageReportModule = __decorate([
|
|
719
541
|
Module({
|
|
720
|
-
imports: [ExtensionsModule, UsageReportJobsModule],
|
|
542
|
+
imports: [ExtensionsModule, UsageReportJobsModule, USAGE_REPORT_QUEUE_MODULE],
|
|
721
543
|
providers: [UsageReporter],
|
|
722
|
-
exports: [UsageReporter, UsageReportJobsModule],
|
|
544
|
+
exports: [UsageReporter, UsageReportJobsModule, USAGE_REPORT_QUEUE_MODULE],
|
|
723
545
|
})
|
|
724
546
|
], UsageReportModule);
|
|
725
547
|
export { UsageReportModule };
|