@capybara-hq/kernel 0.1.0-rc.23 → 0.1.0-rc.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bootstrap.json +25 -25
- package/dist/admin/admin.module.d.ts +2 -1
- package/dist/admin/admin.module.js +7 -3
- package/dist/admin/outbox-dead.controller.d.ts +23 -0
- package/dist/admin/outbox-dead.controller.js +62 -0
- package/dist/admin/outbox-dead.schemas.d.ts +8 -0
- package/dist/admin/outbox-dead.schemas.js +18 -0
- package/dist/admin/outbox-dead.service.d.ts +66 -0
- package/dist/admin/outbox-dead.service.js +179 -0
- package/dist/admin/overview.service.d.ts +40 -5
- package/dist/admin/overview.service.js +41 -7
- package/dist/agents/from-template.js +1 -1
- package/dist/background/finish.d.ts +5 -2
- package/dist/background/finish.js +14 -3
- package/dist/background/launch.d.ts +2 -1
- package/dist/background/launch.js +5 -2
- package/dist/background/store.d.ts +8 -0
- package/dist/background/store.js +24 -0
- package/dist/catalog/admin-catalog.controller.d.ts +21 -2
- package/dist/catalog/admin-catalog.controller.js +45 -6
- package/dist/catalog/admin-catalog.schemas.d.ts +5 -0
- package/dist/catalog/admin-catalog.schemas.js +8 -0
- package/dist/catalog/catalog.module.d.ts +2 -1
- package/dist/catalog/catalog.module.js +5 -1
- package/dist/catalog/marketplaces.service.d.ts +1 -1
- package/dist/catalog/platform-import-update.d.ts +50 -0
- package/dist/catalog/platform-import-update.js +116 -0
- package/dist/catalog/plugin-import.d.ts +36 -1
- package/dist/catalog/plugin-import.js +23 -1
- package/dist/catalog/plugin-upgrade.d.ts +18 -0
- package/dist/catalog/plugin-upgrade.js +19 -0
- package/dist/channels/channels.module.d.ts +5 -2
- package/dist/channels/channels.module.js +9 -2
- package/dist/channels/deliver-job.d.ts +25 -0
- package/dist/channels/deliver-job.js +25 -0
- package/dist/channels/inbound.d.ts +1 -1
- package/dist/channels/inbound.js +2 -15
- package/dist/channels/jobs.d.ts +28 -18
- package/dist/channels/jobs.js +63 -26
- package/dist/config/schema.d.ts +0 -3
- package/dist/config/schema.js +11 -20
- package/dist/conversations/agent-tree.d.ts +16 -0
- package/dist/conversations/agent-tree.js +36 -0
- package/dist/conversations/conversations.service.d.ts +44 -14
- package/dist/conversations/conversations.service.js +52 -38
- package/dist/conversations/event-log.d.ts +7 -10
- package/dist/conversations/event-log.js +24 -15
- package/dist/conversations/events.js +2 -1
- package/dist/conversations/queue.service.d.ts +8 -3
- package/dist/conversations/queue.service.js +37 -13
- package/dist/conversations/titles.d.ts +21 -8
- package/dist/conversations/titles.js +42 -15
- package/dist/db/kernel-schema.d.ts +26 -8
- package/dist/devices/devices.service.d.ts +12 -3
- package/dist/devices/devices.service.js +18 -3
- package/dist/errors.d.ts +11 -0
- package/dist/errors.js +11 -0
- package/dist/evals/run-replay.js +14 -8
- package/dist/extensions/extensions.module.js +3 -0
- package/dist/extensions/lifecycle.d.ts +6 -1
- package/dist/extensions/lifecycle.js +9 -2
- package/dist/extensions/usage-subscribers.d.ts +43 -0
- package/dist/extensions/usage-subscribers.js +108 -0
- package/dist/http/api.module.js +2 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/infra/cli/kernel-cli.module.js +5 -0
- package/dist/infra/events/kernel-events.d.ts +9 -35
- package/dist/infra/outbox/outbox-relay.module.d.ts +6 -0
- package/dist/infra/outbox/outbox-relay.module.js +18 -0
- package/dist/infra/outbox/outbox.dispatcher.d.ts +103 -0
- package/dist/infra/outbox/outbox.dispatcher.js +310 -0
- package/dist/infra/outbox/outbox.module.d.ts +24 -0
- package/dist/infra/outbox/outbox.module.js +50 -0
- package/dist/infra/outbox/outbox.relay.d.ts +31 -0
- package/dist/infra/outbox/outbox.relay.js +76 -0
- package/dist/infra/outbox/outbox.service.d.ts +51 -0
- package/dist/infra/outbox/outbox.service.js +131 -0
- package/dist/infra/outbox/registry.d.ts +120 -0
- package/dist/infra/outbox/registry.js +108 -0
- package/dist/infra/queue/queue-names.d.ts +8 -0
- package/dist/infra/queue/queue-names.js +16 -0
- package/dist/kernel.module.js +3 -0
- package/dist/local-devices/device-policy.service.d.ts +4 -0
- package/dist/local-devices/device-policy.service.js +61 -3
- package/dist/local-devices/machines.d.ts +13 -5
- package/dist/local-devices/machines.js +45 -7
- package/dist/local-devices/policy-summary.d.ts +44 -1
- package/dist/local-devices/policy-summary.js +69 -0
- package/dist/models/embeddings.d.ts +9 -9
- package/dist/models/embeddings.js +11 -18
- package/dist/models/models.module.js +3 -0
- package/dist/orgs/orgs.service.js +7 -0
- package/dist/overrides.d.ts +20 -3
- package/dist/overrides.js +10 -2
- package/dist/platform/exclusions.js +5 -0
- package/dist/platform/ops/admin.ops.js +18 -2
- package/dist/platform/ops/catalog-admin.ops.d.ts +3 -1
- package/dist/platform/ops/catalog-admin.ops.js +32 -4
- package/dist/platform/ops/outbox.ops.d.ts +21 -0
- package/dist/platform/ops/outbox.ops.js +125 -0
- package/dist/platform/ops/schedules.ops.js +40 -1
- package/dist/platform/platform-ops.module.d.ts +2 -1
- package/dist/platform/platform-ops.module.js +3 -0
- package/dist/platform/route-schemas.js +3 -0
- package/dist/roles/worker.module.js +9 -4
- package/dist/runtime/langgraph-loop.js +13 -5
- package/dist/runtime/limits.d.ts +15 -0
- package/dist/runtime/limits.js +25 -0
- package/dist/runtime/loop.d.ts +8 -3
- package/dist/runtime/loop.js +2 -1
- package/dist/runtime/maintenance.d.ts +4 -2
- package/dist/runtime/maintenance.js +17 -2
- package/dist/runtime/model-call.d.ts +14 -16
- package/dist/runtime/model-call.js +14 -24
- package/dist/runtime/queue-lost-jobs.d.ts +24 -0
- package/dist/runtime/queue-lost-jobs.js +51 -0
- package/dist/runtime/run-control.d.ts +42 -5
- package/dist/runtime/run-control.js +66 -12
- package/dist/runtime/run-finish.d.ts +39 -12
- package/dist/runtime/run-finish.js +97 -14
- package/dist/runtime/run-jobs.d.ts +95 -13
- package/dist/runtime/run-jobs.js +199 -28
- package/dist/runtime/run-orphan-check.job.d.ts +142 -0
- package/dist/runtime/run-orphan-check.job.js +339 -0
- package/dist/runtime/run-processors.d.ts +16 -9
- package/dist/runtime/run-processors.js +1 -1
- package/dist/runtime/run-queue.d.ts +64 -55
- package/dist/runtime/run-queue.js +147 -284
- package/dist/runtime/run-steps.d.ts +1 -2
- package/dist/runtime/run-steps.js +8 -11
- package/dist/runtime/run-worker.d.ts +6 -17
- package/dist/runtime/run-worker.js +32 -34
- package/dist/runtime/runtime.module.d.ts +2 -3
- package/dist/runtime/runtime.module.js +5 -6
- package/dist/runtime/step-log.d.ts +16 -4
- package/dist/runtime/step-log.js +26 -6
- package/dist/runtime/stream.controller.d.ts +2 -4
- package/dist/runtime/stream.controller.js +5 -11
- package/dist/runtime/subagents/child.js +1 -1
- package/dist/runtime/subagents/definition.d.ts +2 -1
- package/dist/runtime/subagents/definition.js +4 -2
- package/dist/runtime/subagents/deliveries.d.ts +6 -3
- package/dist/runtime/subagents/deliveries.js +16 -7
- package/dist/runtime/subagents/dispatch.d.ts +12 -12
- package/dist/runtime/subagents/dispatch.js +13 -38
- package/dist/runtime/subagents/messages.d.ts +5 -1
- package/dist/runtime/subagents/messages.js +11 -3
- package/dist/runtime/subagents/skill-fork.d.ts +3 -3
- package/dist/runtime/subagents/skill-fork.js +5 -5
- package/dist/runtime/subagents/subagents.service.js +1 -1
- package/dist/runtime/subagents/tree-jobs.d.ts +61 -0
- package/dist/runtime/subagents/tree-jobs.js +86 -0
- package/dist/runtime/usage-report-jobs.d.ts +23 -48
- package/dist/runtime/usage-report-jobs.js +67 -119
- package/dist/runtime/usage-report.d.ts +28 -77
- package/dist/runtime/usage-report.js +35 -213
- package/dist/runtime/waits.d.ts +48 -198
- package/dist/runtime/waits.js +103 -805
- package/dist/sandbox/sandbox.service.d.ts +17 -4
- package/dist/sandbox/sandbox.service.js +55 -14
- package/dist/schedules/calendar.d.ts +134 -0
- package/dist/schedules/calendar.js +410 -0
- package/dist/schedules/fire.d.ts +1 -3
- package/dist/schedules/fire.js +5 -10
- package/dist/schedules/run-tracker.d.ts +0 -5
- package/dist/schedules/run-tracker.js +0 -13
- package/dist/schedules/scan.job.js +15 -20
- package/dist/schedules/schedule-math.d.ts +42 -0
- package/dist/schedules/schedule-math.js +52 -0
- package/dist/schedules/scheduled-tasks.service.d.ts +16 -0
- package/dist/schedules/scheduled-tasks.service.js +53 -0
- package/dist/schedules/schedules.controller.d.ts +5 -1
- package/dist/schedules/schedules.controller.js +15 -1
- package/dist/schedules/schedules.schemas.d.ts +35 -0
- package/dist/schedules/schedules.schemas.js +27 -0
- package/dist/schedules/store.d.ts +5 -0
- package/dist/schedules/store.js +20 -0
- package/dist/tools/approvals.service.d.ts +26 -7
- package/dist/tools/approvals.service.js +88 -10
- package/dist/tools/builtin-names.d.ts +5 -3
- package/dist/tools/builtin-names.js +12 -4
- package/dist/tools/builtins/builtin-tools.d.ts +1 -2
- package/dist/tools/builtins/builtin-tools.js +21 -17
- package/dist/tools/builtins/sandbox/bash.js +9 -3
- package/dist/tools/builtins/sandbox/common.d.ts +18 -1
- package/dist/tools/builtins/sandbox/common.js +14 -1
- package/dist/tools/builtins/sandbox/index.d.ts +2 -1
- package/dist/tools/builtins/sandbox/index.js +4 -1
- package/dist/tools/builtins/sandbox/task-list.d.ts +12 -0
- package/dist/tools/builtins/sandbox/task-list.js +130 -0
- package/dist/tools/builtins/sandbox/task-output.d.ts +8 -1
- package/dist/tools/builtins/sandbox/task-output.js +21 -3
- package/dist/tools/builtins/web/extract.d.ts +2 -4
- package/dist/tools/builtins/web/extract.js +2 -5
- package/dist/tools/pipeline.d.ts +1 -1
- package/dist/tools/pipeline.js +1 -1
- package/dist/tools/questions.service.d.ts +15 -6
- package/dist/tools/questions.service.js +20 -8
- package/dist/tools/registry.d.ts +4 -2
- package/dist/tools/registry.js +4 -3
- package/dist/tools/tools.module.js +4 -3
- package/package.json +3 -3
- package/src/migrations/0118_device_policy_ops.sql +6 -0
- package/src/migrations/0119_outbox.sql +22 -0
- package/src/migrations/0120_drop_usage_report_sweep.sql +7 -0
|
@@ -1,20 +1,17 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import {
|
|
3
|
-
import type { JobsOptions, Queue } from "bullmq";
|
|
4
|
-
import type { KernelSchemaDb } from "../db/kernel-schema.js";
|
|
5
|
-
import { KernelConfigService } from "../infra/config/kernel-config.service.js";
|
|
1
|
+
import type { ConversationEvent } from "../conversations/events.js";
|
|
2
|
+
import { UsageSubscribers } from "../extensions/usage-subscribers.js";
|
|
6
3
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
7
|
-
import {
|
|
4
|
+
import { TransactionHost } from "../infra/database/transaction-host.js";
|
|
5
|
+
import { OutboxService } from "../infra/outbox/outbox.service.js";
|
|
8
6
|
/**
|
|
9
|
-
* The producer half of `usage.report` delivery
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* {@link ModelUsageJobs} enqueues it here.
|
|
7
|
+
* The producer half of `usage.report` delivery, and the queue's ids shared with the reporter
|
|
8
|
+
* (`usage-report.ts`): a delivery is owed in the transaction that writes the usage it reports
|
|
9
|
+
* (transactional outbox spec §8.1). It registers no queue (the outbox dispatcher adds the job), so
|
|
10
|
+
* the `ctx.models` broker below the extension host and the step writes can use it.
|
|
14
11
|
*/
|
|
15
12
|
/** Queue of `usage.report` deliveries (no `:` in the name). */
|
|
16
13
|
export declare const USAGE_REPORT_QUEUE = "cap-kernel-usage-report";
|
|
17
|
-
/** The job name of every delivery. */
|
|
14
|
+
/** The job name of every delivery (its outbox registry entry holds the job options). */
|
|
18
15
|
export declare const USAGE_REPORT_JOB = "usage-report";
|
|
19
16
|
/** One job per `usage` event (no `:`: conversation ids are UUIDs, seq is a number). */
|
|
20
17
|
export declare const usageReportJobId: (conversationId: string, seq: number) => string;
|
|
@@ -42,47 +39,25 @@ export type UsageReportJob = ({
|
|
|
42
39
|
*/
|
|
43
40
|
dropped?: Record<string, string>;
|
|
44
41
|
};
|
|
45
|
-
/** First attempt plus 5 retries (F11). */
|
|
46
|
-
export declare const USAGE_REPORT_ATTEMPTS = 6;
|
|
47
|
-
/** Retry backoff: 2 s, doubling (2, 4, 8, 16, 32 s). */
|
|
48
|
-
export declare const USAGE_REPORT_BACKOFF_MS = 2000;
|
|
49
|
-
/** The delivery retry policy: attempts and the first exponential backoff. */
|
|
50
|
-
export interface UsageReportRetry {
|
|
51
|
-
attempts: number;
|
|
52
|
-
backoffMs: number;
|
|
53
|
-
}
|
|
54
|
-
/** Optional provider: the {@link UsageReportRetry} instead of the defaults above (tests). */
|
|
55
|
-
export declare const USAGE_REPORT_RETRY: unique symbol;
|
|
56
|
-
/** The options of one delivery job: its id, the retry policy and the retention of finished jobs. */
|
|
57
|
-
export declare function usageReportJobOpts(jobId: string, retry: UsageReportRetry, keepHours: number): JobsOptions;
|
|
58
|
-
/**
|
|
59
|
-
* Plugins that subscribe to `usage.report` according to deployment state: an approved permission
|
|
60
|
-
* snapshot that declares the `usage.report` hook, for a plugin the deployment still knows: an enabled
|
|
61
|
-
* install, or a config-declared plugin with a replica status row that is not `unloaded`. With `orgId`, only plugins
|
|
62
|
-
* the organization enabled (org config rows and required bootstrap plugins).
|
|
63
|
-
*/
|
|
64
|
-
export declare function queryUsageSubscribers(db: KernelSchemaDb, orgId?: string): Promise<Set<string>>;
|
|
65
42
|
/**
|
|
66
|
-
*
|
|
67
|
-
*
|
|
43
|
+
* Owes `usage.report` deliveries in the caller's transaction, only while the deployment has a
|
|
44
|
+
* subscriber ({@link UsageSubscribers}): usage written while no plugin subscribes is never
|
|
45
|
+
* reported.
|
|
68
46
|
*/
|
|
69
|
-
export declare class
|
|
47
|
+
export declare class UsageReportJobs {
|
|
70
48
|
private readonly kdb;
|
|
71
|
-
private readonly
|
|
72
|
-
private readonly
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
private readonly unsubscribe;
|
|
76
|
-
/** Set at shutdown: a late event enqueues nothing (the sweep offers the row later). */
|
|
77
|
-
private closed;
|
|
78
|
-
constructor(kdb: KernelDb, queue: Queue<UsageReportJob>, config: Pick<KernelConfigService, "get">, log: Logger, retry: UsageReportRetry | undefined, events: KernelEvents);
|
|
79
|
-
onApplicationShutdown(): void;
|
|
49
|
+
private readonly outbox;
|
|
50
|
+
private readonly subscribers;
|
|
51
|
+
readonly txHost: TransactionHost;
|
|
52
|
+
constructor(kdb: KernelDb, outbox: Pick<OutboxService, "add">, subscribers: Pick<UsageSubscribers, "any">, txHost: TransactionHost);
|
|
80
53
|
/**
|
|
81
|
-
*
|
|
82
|
-
*
|
|
54
|
+
* Owes the delivery of every `usage` event among `events` (the events this transaction wrote
|
|
55
|
+
* to the conversation's log), job `usage-<conversationId>-<seq>`.
|
|
83
56
|
*/
|
|
84
|
-
|
|
57
|
+
oweEvents(conversationId: string, events: readonly ConversationEvent[]): Promise<void>;
|
|
58
|
+
/** Owes the delivery of one `model_usage` row (job `usage-model-<rowId>`). */
|
|
59
|
+
oweModelUsage(usageId: string, orgId: string): Promise<void>;
|
|
85
60
|
}
|
|
86
|
-
/** The usage report
|
|
61
|
+
/** The usage report producer (no queue, no dependency on the extension host). */
|
|
87
62
|
export declare class UsageReportJobsModule {
|
|
88
63
|
}
|
|
@@ -7,152 +7,100 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
7
7
|
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
8
8
|
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
9
|
};
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import { sql } from "kysely";
|
|
13
|
-
import { enabledExtensionsQuery } from "../extensions/enabled.js";
|
|
14
|
-
import { KernelConfigService } from "../infra/config/kernel-config.service.js";
|
|
10
|
+
import { Inject, Injectable, Module } from "@nestjs/common";
|
|
11
|
+
import { UsageSubscribers, UsageSubscribersModule } from "../extensions/usage-subscribers.js";
|
|
15
12
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
13
|
+
import { TransactionHost } from "../infra/database/transaction-host.js";
|
|
14
|
+
import { Transactional } from "../infra/database/transactional.decorator.js";
|
|
15
|
+
import { OutboxService } from "../infra/outbox/outbox.service.js";
|
|
19
16
|
/**
|
|
20
|
-
* The producer half of `usage.report` delivery
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
* {@link ModelUsageJobs} enqueues it here.
|
|
17
|
+
* The producer half of `usage.report` delivery, and the queue's ids shared with the reporter
|
|
18
|
+
* (`usage-report.ts`): a delivery is owed in the transaction that writes the usage it reports
|
|
19
|
+
* (transactional outbox spec §8.1). It registers no queue (the outbox dispatcher adds the job), so
|
|
20
|
+
* the `ctx.models` broker below the extension host and the step writes can use it.
|
|
25
21
|
*/
|
|
26
22
|
/** Queue of `usage.report` deliveries (no `:` in the name). */
|
|
27
23
|
export const USAGE_REPORT_QUEUE = "cap-kernel-usage-report";
|
|
28
|
-
/** The job name of every delivery. */
|
|
24
|
+
/** The job name of every delivery (its outbox registry entry holds the job options). */
|
|
29
25
|
export const USAGE_REPORT_JOB = "usage-report";
|
|
30
26
|
/** One job per `usage` event (no `:`: conversation ids are UUIDs, seq is a number). */
|
|
31
27
|
export const usageReportJobId = (conversationId, seq) => `usage-${conversationId}-${seq}`;
|
|
32
28
|
/** One job per `model_usage` row (spec B §5.4; row ids are UUIDs, so no clash with the above). */
|
|
33
29
|
export const modelUsageJobId = (usageId) => `usage-model-${usageId}`;
|
|
34
|
-
/** First attempt plus 5 retries (F11). */
|
|
35
|
-
export const USAGE_REPORT_ATTEMPTS = 6;
|
|
36
|
-
/** Retry backoff: 2 s, doubling (2, 4, 8, 16, 32 s). */
|
|
37
|
-
export const USAGE_REPORT_BACKOFF_MS = 2000;
|
|
38
|
-
/** Optional provider: the {@link UsageReportRetry} instead of the defaults above (tests). */
|
|
39
|
-
export const USAGE_REPORT_RETRY = Symbol("USAGE_REPORT_RETRY");
|
|
40
|
-
/** Failed jobs (given up) are kept for inspection, at most {@link KEEP_FAILED_MAX} of them. */
|
|
41
|
-
const KEEP_FAILED_S = 7 * 86_400;
|
|
42
|
-
/** Upper bound on kept failed jobs (I-2): a long outage of a billing endpoint cannot grow Redis. */
|
|
43
|
-
const KEEP_FAILED_MAX = 100_000;
|
|
44
30
|
/**
|
|
45
|
-
*
|
|
46
|
-
*
|
|
31
|
+
* Owes `usage.report` deliveries in the caller's transaction, only while the deployment has a
|
|
32
|
+
* subscriber ({@link UsageSubscribers}): usage written while no plugin subscribes is never
|
|
33
|
+
* reported.
|
|
47
34
|
*/
|
|
48
|
-
|
|
49
|
-
/** The options of one delivery job: its id, the retry policy and the retention of finished jobs. */
|
|
50
|
-
export function usageReportJobOpts(jobId, retry, keepHours) {
|
|
51
|
-
return {
|
|
52
|
-
jobId,
|
|
53
|
-
attempts: retry.attempts,
|
|
54
|
-
backoff: { type: "exponential", delay: retry.backoffMs },
|
|
55
|
-
removeOnComplete: { age: keepHours * 3600, count: KEEP_COMPLETED_MAX },
|
|
56
|
-
removeOnFail: { age: KEEP_FAILED_S, count: KEEP_FAILED_MAX },
|
|
57
|
-
};
|
|
58
|
-
}
|
|
59
|
-
/**
|
|
60
|
-
* Plugins that subscribe to `usage.report` according to deployment state: an approved permission
|
|
61
|
-
* snapshot that declares the `usage.report` hook, for a plugin the deployment still knows: an enabled
|
|
62
|
-
* install, or a config-declared plugin with a replica status row that is not `unloaded`. With `orgId`, only plugins
|
|
63
|
-
* the organization enabled (org config rows and required bootstrap plugins).
|
|
64
|
-
*/
|
|
65
|
-
export async function queryUsageSubscribers(db, orgId) {
|
|
66
|
-
let q = db
|
|
67
|
-
.selectFrom("extension_permission_snapshots as s")
|
|
68
|
-
.select("s.plugin")
|
|
69
|
-
.where(sql `s.permissions -> 'permissions' -> 'hooks' ? 'usage.report'`)
|
|
70
|
-
.where((eb) => eb.or([
|
|
71
|
-
eb.exists(eb
|
|
72
|
-
.selectFrom("extension_installs as i")
|
|
73
|
-
.select(sql `1`.as("one"))
|
|
74
|
-
.whereRef("i.name", "=", "s.plugin")
|
|
75
|
-
.where("i.desired_state", "=", "enabled")),
|
|
76
|
-
// config-declared plugins still declared (and not unloaded) on some replica; installs are
|
|
77
|
-
// covered by desired_state above (their replica rows linger as `unloaded` when disabled)
|
|
78
|
-
eb.exists(eb
|
|
79
|
-
.selectFrom("extension_replica_status as r")
|
|
80
|
-
.select(sql `1`.as("one"))
|
|
81
|
-
.whereRef("r.plugin", "=", "s.plugin")
|
|
82
|
-
.where("r.source", "=", "config")
|
|
83
|
-
.where("r.state", "<>", "unloaded")),
|
|
84
|
-
]));
|
|
85
|
-
if (orgId !== undefined) {
|
|
86
|
-
q = q.where("s.plugin", "in", enabledExtensionsQuery(db, orgId));
|
|
87
|
-
}
|
|
88
|
-
return new Set((await q.execute()).map((r) => r.plugin));
|
|
89
|
-
}
|
|
90
|
-
/**
|
|
91
|
-
* Enqueues the `usage.report` delivery of `model_usage` rows on `model_usage.recorded` (fast path;
|
|
92
|
-
* the reporter's sweep re-enqueues rows still unreported, so a lost event or enqueue is caught up).
|
|
93
|
-
*/
|
|
94
|
-
let ModelUsageJobs = class ModelUsageJobs {
|
|
35
|
+
let UsageReportJobs = class UsageReportJobs {
|
|
95
36
|
kdb;
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
unsubscribe;
|
|
101
|
-
/** Set at shutdown: a late event enqueues nothing (the sweep offers the row later). */
|
|
102
|
-
closed = false;
|
|
103
|
-
constructor(kdb, queue, config, log, retry = {
|
|
104
|
-
attempts: USAGE_REPORT_ATTEMPTS,
|
|
105
|
-
backoffMs: USAGE_REPORT_BACKOFF_MS,
|
|
106
|
-
}, events) {
|
|
37
|
+
outbox;
|
|
38
|
+
subscribers;
|
|
39
|
+
txHost;
|
|
40
|
+
constructor(kdb, outbox, subscribers, txHost) {
|
|
107
41
|
this.kdb = kdb;
|
|
108
|
-
this.
|
|
109
|
-
this.
|
|
110
|
-
this.
|
|
111
|
-
this.retry = retry;
|
|
112
|
-
this.unsubscribe = events.on("model_usage.recorded", ({ id }) => this.enqueue(id));
|
|
113
|
-
}
|
|
114
|
-
onApplicationShutdown() {
|
|
115
|
-
this.closed = true;
|
|
116
|
-
this.unsubscribe();
|
|
42
|
+
this.outbox = outbox;
|
|
43
|
+
this.subscribers = subscribers;
|
|
44
|
+
this.txHost = txHost;
|
|
117
45
|
}
|
|
118
46
|
/**
|
|
119
|
-
*
|
|
120
|
-
*
|
|
47
|
+
* Owes the delivery of every `usage` event among `events` (the events this transaction wrote
|
|
48
|
+
* to the conversation's log), job `usage-<conversationId>-<seq>`.
|
|
121
49
|
*/
|
|
122
|
-
async
|
|
123
|
-
|
|
50
|
+
async oweEvents(conversationId, events) {
|
|
51
|
+
const usage = events.filter((e) => e.type === "usage");
|
|
52
|
+
if (usage.length === 0 || !(await this.subscribers.any()))
|
|
124
53
|
return;
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
this.
|
|
54
|
+
const conv = await this.kdb.db
|
|
55
|
+
.selectFrom("conversations")
|
|
56
|
+
.select("org_id")
|
|
57
|
+
.where("id", "=", conversationId)
|
|
58
|
+
.executeTakeFirst();
|
|
59
|
+
for (const e of usage) {
|
|
60
|
+
await this.outbox.add({
|
|
61
|
+
queue: USAGE_REPORT_QUEUE,
|
|
62
|
+
name: USAGE_REPORT_JOB,
|
|
63
|
+
jobId: usageReportJobId(conversationId, e.seq),
|
|
64
|
+
data: { conversationId, seq: e.seq },
|
|
65
|
+
...(conv !== undefined && { orgId: conv.org_id }),
|
|
66
|
+
});
|
|
132
67
|
}
|
|
133
68
|
}
|
|
69
|
+
/** Owes the delivery of one `model_usage` row (job `usage-model-<rowId>`). */
|
|
70
|
+
async oweModelUsage(usageId, orgId) {
|
|
71
|
+
if (!(await this.subscribers.any()))
|
|
72
|
+
return;
|
|
73
|
+
await this.outbox.add({
|
|
74
|
+
queue: USAGE_REPORT_QUEUE,
|
|
75
|
+
name: USAGE_REPORT_JOB,
|
|
76
|
+
jobId: modelUsageJobId(usageId),
|
|
77
|
+
data: { usageId },
|
|
78
|
+
orgId,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
134
81
|
};
|
|
135
|
-
|
|
82
|
+
__decorate([
|
|
83
|
+
Transactional({ propagation: "mandatory" })
|
|
84
|
+
], UsageReportJobs.prototype, "oweEvents", null);
|
|
85
|
+
__decorate([
|
|
86
|
+
Transactional({ propagation: "mandatory" })
|
|
87
|
+
], UsageReportJobs.prototype, "oweModelUsage", null);
|
|
88
|
+
UsageReportJobs = __decorate([
|
|
136
89
|
Injectable(),
|
|
137
90
|
__param(0, Inject(KernelDb)),
|
|
138
|
-
__param(1, Inject(
|
|
139
|
-
__param(2, Inject(
|
|
140
|
-
__param(3, Inject(
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
], ModelUsageJobs);
|
|
145
|
-
export { ModelUsageJobs };
|
|
146
|
-
/** The usage report queue's registration, shared by the producer and the reporter's module. */
|
|
147
|
-
const USAGE_REPORT_QUEUE_MODULE = QueueModule.registerQueue(USAGE_REPORT_QUEUE);
|
|
148
|
-
/** The usage report queue and its `model_usage` producer (no dependency on the extension host). */
|
|
91
|
+
__param(1, Inject(OutboxService)),
|
|
92
|
+
__param(2, Inject(UsageSubscribers)),
|
|
93
|
+
__param(3, Inject(TransactionHost))
|
|
94
|
+
], UsageReportJobs);
|
|
95
|
+
export { UsageReportJobs };
|
|
96
|
+
/** The usage report producer (no queue, no dependency on the extension host). */
|
|
149
97
|
let UsageReportJobsModule = class UsageReportJobsModule {
|
|
150
98
|
};
|
|
151
99
|
UsageReportJobsModule = __decorate([
|
|
152
100
|
Module({
|
|
153
|
-
imports: [
|
|
154
|
-
providers: [
|
|
155
|
-
exports: [
|
|
101
|
+
imports: [UsageSubscribersModule],
|
|
102
|
+
providers: [UsageReportJobs],
|
|
103
|
+
exports: [UsageReportJobs],
|
|
156
104
|
})
|
|
157
105
|
], UsageReportJobsModule);
|
|
158
106
|
export { UsageReportJobsModule };
|
|
@@ -5,21 +5,9 @@ import { ExtensionHost } from "../extensions/host.js";
|
|
|
5
5
|
import { KernelConfigService } from "../infra/config/kernel-config.service.js";
|
|
6
6
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
7
7
|
import { LockService } from "../infra/lock/lock.service.js";
|
|
8
|
-
import type
|
|
9
|
-
|
|
10
|
-
export {
|
|
11
|
-
/** The sweeper's cron job name (no `:`; every minute, once across replicas). */
|
|
12
|
-
export declare const USAGE_SWEEP_JOB = "usage-report-sweep";
|
|
13
|
-
/** The maintenance scheduler the sweep ran on before it became a `@LockedCron` (retired). */
|
|
14
|
-
export { USAGE_SWEEP_SCHEDULER_ID } from "./job-ids.js";
|
|
15
|
-
/** The sweep leaves the newest events to the fast path (and to transactions still committing). */
|
|
16
|
-
export declare const USAGE_SWEEP_LAG_MS = 30000;
|
|
17
|
-
/**
|
|
18
|
-
* Each sweep re-scans this far behind the watermark: events whose transaction committed after a
|
|
19
|
-
* previous sweep passed their `created_at` are still found. Far inside the dedupe window
|
|
20
|
-
* (`hooks.usageReportKeepHours` ≥ 1 h), so re-enqueued events dedupe on the kept job.
|
|
21
|
-
*/
|
|
22
|
-
export declare const USAGE_SWEEP_OVERLAP_MS: number;
|
|
8
|
+
import { type UsageReportJob } from "./usage-report-jobs.js";
|
|
9
|
+
export { queryUsageSubscribers } from "../extensions/usage-subscribers.js";
|
|
10
|
+
export { modelUsageJobId, USAGE_REPORT_JOB, USAGE_REPORT_QUEUE, type UsageReportJob, UsageReportJobs, UsageReportJobsModule, usageReportJobId, } from "./usage-report-jobs.js";
|
|
23
11
|
/**
|
|
24
12
|
* Of `plugins`, those that are **definitely** unservable right now (I-2, N-3): live replicas report
|
|
25
13
|
* them (a `extension_replica_status` row with a heartbeat newer than `staleSeconds`), no live row is
|
|
@@ -35,14 +23,16 @@ export declare function queryUnservable(db: KernelSchemaDb, plugins: readonly st
|
|
|
35
23
|
*/
|
|
36
24
|
export declare function narrowPending(stored: readonly string[] | undefined, failed: readonly string[]): string[];
|
|
37
25
|
/**
|
|
38
|
-
* Reliable `usage.report` delivery (F11, P-17,
|
|
39
|
-
* runs and full-run eval replays) is delivered at least once to every
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
* `
|
|
43
|
-
* -
|
|
44
|
-
*
|
|
45
|
-
*
|
|
26
|
+
* Reliable `usage.report` delivery (F11, P-17, transactional outbox spec §8.1). Every `usage` event
|
|
27
|
+
* written to the event log (chat runs and full-run eval replays) is delivered at least once to every
|
|
28
|
+
* subscribing plugin.
|
|
29
|
+
* - **Owed with the usage:** the transaction that writes a `usage` event (`StepLog`'s step writes, a
|
|
30
|
+
* title's writes) or a `model_usage` row also writes the outbox row of job
|
|
31
|
+
* `usage-<conversationId>-<seq>` / `usage-model-<rowId>` (`UsageReportJobs`), while the deployment
|
|
32
|
+
* has at least one subscriber (`UsageSubscribers`: from PG, whatever this replica loaded; cached
|
|
33
|
+
* 30 s per replica, cleared on `PLUGINS_CHANGED`). The outbox fast path adds the job after the
|
|
34
|
+
* commit and the relay catches up what it did not. Usage written while no plugin subscribed is
|
|
35
|
+
* never reported: a subscription starts when the plugin is enabled.
|
|
46
36
|
* - **Worker** (worker role, `hooks.usageReportConcurrency` per replica): reads the payload from the log
|
|
47
37
|
* at execution time (the `usage` event, the conversation with `kind`, the run). On the first attempt
|
|
48
38
|
* the targets are the deployment's subscribers the organization enabled (I-2: not this replica's
|
|
@@ -54,49 +44,45 @@ export declare function narrowPending(stored: readonly string[] | undefined, fai
|
|
|
54
44
|
* once (I-2). A drop (N-3) is never silent: after delivering to the other targets the job ends
|
|
55
45
|
* failed with an `UnrecoverableError` whose failedReason names the dropped plugins, plus a
|
|
56
46
|
* throttled warning. Plugins the organization disabled by delivery time are dropped silently (not
|
|
57
|
-
* targets any more). The final failure logs an error (no audit);
|
|
58
|
-
* most 100 000
|
|
59
|
-
*
|
|
60
|
-
* using up
|
|
47
|
+
* targets any more). The final failure logs an error (no audit); finished jobs are kept a day and
|
|
48
|
+
* failed jobs 7 days, at most 100 000 each (the outbox registry's `usage-report` options). Each
|
|
49
|
+
* call runs through the instance's `inflight.run`, limited by `hooks.timeoutMs`; when this
|
|
50
|
+
* replica stops, in-flight calls are aborted and the job goes back to the queue without using up
|
|
51
|
+
* an attempt.
|
|
61
52
|
* Handlers must be idempotent by `eventId` (`<conversationId>/<seq>`, or a `model_usage` row id).
|
|
62
53
|
* Nothing is written to the event log (never model-visible). Compaction calls log usage too (`purpose: "compaction"`) and are reported,
|
|
63
54
|
* as are conversation title calls (`purpose: "title"`, no run: `runId` is "") and `WebFetch`
|
|
64
55
|
* extraction calls (`purpose: "web_fetch"`).
|
|
65
56
|
* Single-step eval replays and judge requests log no usage: not reported (M7).
|
|
66
57
|
* - **Model usage outside conversations** (spec B §5.4): each `ctx.models.embed` call writes a
|
|
67
|
-
* `model_usage` row
|
|
68
|
-
* enqueues job `usage-model-<rowId>` (only while some plugin subscribes); the payload
|
|
58
|
+
* `model_usage` row and owes job `usage-model-<rowId>` with it; the payload
|
|
69
59
|
* (`purpose: "embedding"`, `eventId` = the row id, the calling `extension`, no conversation fields)
|
|
70
60
|
* is read from the row when the job runs, and the row's `reported_at` is set once nothing is left
|
|
71
|
-
* to deliver; a job that gives up after its last attempt sets `report_failed_at`.
|
|
72
|
-
* re-enqueues recent rows that are neither (see {@link sweep}).
|
|
61
|
+
* to deliver; a job that gives up after its last attempt sets `report_failed_at`.
|
|
73
62
|
*/
|
|
74
63
|
export declare class UsageReporter {
|
|
75
64
|
private readonly kdb;
|
|
76
65
|
private readonly queue;
|
|
77
66
|
/**
|
|
78
67
|
* The host's hook registry: handlers of the current local plugin generations (through
|
|
79
|
-
* inflight.run)
|
|
80
|
-
* I/O without handlers; and whether a plugin's current instance is active here.
|
|
68
|
+
* inflight.run); and whether a plugin's current instance is active here.
|
|
81
69
|
*/
|
|
82
70
|
private readonly host;
|
|
83
71
|
private readonly config;
|
|
84
72
|
/** The warning throttle's markers (`capk:usage-warn:<plugin>:<kind>`, SET NX PX). */
|
|
85
73
|
private readonly locks;
|
|
86
74
|
private readonly log;
|
|
87
|
-
private readonly retry;
|
|
88
75
|
private worker;
|
|
89
76
|
private closed;
|
|
90
77
|
private readonly stopping;
|
|
91
78
|
constructor(kdb: KernelDb, queue: Queue<UsageReportJob>,
|
|
92
79
|
/**
|
|
93
80
|
* The host's hook registry: handlers of the current local plugin generations (through
|
|
94
|
-
* inflight.run)
|
|
95
|
-
* I/O without handlers; and whether a plugin's current instance is active here.
|
|
81
|
+
* inflight.run); and whether a plugin's current instance is active here.
|
|
96
82
|
*/
|
|
97
83
|
host: Pick<ExtensionHost, "hooks" | "current">, config: Pick<KernelConfigService, "get">,
|
|
98
84
|
/** The warning throttle's markers (`capk:usage-warn:<plugin>:<kind>`, SET NX PX). */
|
|
99
|
-
locks: Pick<LockService, "lease">, log: Logger
|
|
85
|
+
locks: Pick<LockService, "lease">, log: Logger);
|
|
100
86
|
private get db();
|
|
101
87
|
private get hooks();
|
|
102
88
|
/**
|
|
@@ -116,30 +102,6 @@ export declare class UsageReporter {
|
|
|
116
102
|
* See {@link queryUsageSubscribers}.
|
|
117
103
|
*/
|
|
118
104
|
subscribers(orgId?: string): Promise<Set<string>>;
|
|
119
|
-
private jobOpts;
|
|
120
|
-
/**
|
|
121
|
-
* Enqueues the delivery of one `usage` event. Returns without a job when no plugin subscribes.
|
|
122
|
-
* Never throws: an enqueue failure (e.g. a Redis hiccup) only logs a warning; the sweeper catches up.
|
|
123
|
-
*/
|
|
124
|
-
enqueue(conversationId: string, seq: number): Promise<void>;
|
|
125
|
-
/**
|
|
126
|
-
* One sweep (R13): re-enqueues the `usage` events created in `[watermark - overlap, now - lag]` in
|
|
127
|
-
* pages, then advances the watermark to `now - lag`. With no subscriber it only advances. The first
|
|
128
|
-
* sweep starts at `now - lag - overlap`. Also re-enqueues the `model_usage` rows of the same
|
|
129
|
-
* window that are neither reported nor given up (spec B §5.4),
|
|
130
|
-
* or, with no subscriber, marks those rows reported (nobody to deliver to). Returns the number of
|
|
131
|
-
* events and rows offered to the queue.
|
|
132
|
-
*/
|
|
133
|
-
sweep(opts?: {
|
|
134
|
-
now?: number;
|
|
135
|
-
}): Promise<number>;
|
|
136
|
-
/**
|
|
137
|
-
* The `model_usage` half of {@link sweep}: rows neither reported nor given up that occurred in
|
|
138
|
-
* the same window as the events (`[watermark - overlap, to]`), oldest first. Bounded like the
|
|
139
|
-
* event sweep, so a row whose job failed for good is never offered again, even after its failed
|
|
140
|
-
* job left Redis (no repeated backlog, no second delivery to plugins already served).
|
|
141
|
-
*/
|
|
142
|
-
private sweepModelUsage;
|
|
143
105
|
/**
|
|
144
106
|
* Starts consuming (worker role only): `worker` is the `@Processor` worker of
|
|
145
107
|
* {@link USAGE_REPORT_QUEUE} (built with the run queue's job lock and stalled detection).
|
|
@@ -150,8 +112,8 @@ export declare class UsageReporter {
|
|
|
150
112
|
/** A job failed (the worker's `failed` event): logs retries and the final failure. */
|
|
151
113
|
onFailed(j: Job<UsageReportJob> | undefined, err: Error): void;
|
|
152
114
|
/**
|
|
153
|
-
* Stops:
|
|
154
|
-
*
|
|
115
|
+
* Stops: in-flight calls are aborted (their jobs requeued) and the worker closes. The queue
|
|
116
|
+
* itself closes with the application.
|
|
155
117
|
*/
|
|
156
118
|
stop(): Promise<void>;
|
|
157
119
|
/** The payload of a job: a `usage` event's or a `model_usage` row's; null when it is gone. */
|
|
@@ -163,7 +125,7 @@ export declare class UsageReporter {
|
|
|
163
125
|
private payloadOfUsage;
|
|
164
126
|
/** A `model_usage` job is done (delivered, or nothing left to deliver to): marks the row. */
|
|
165
127
|
private markReported;
|
|
166
|
-
/** A `model_usage` job gave up after its last attempt: the
|
|
128
|
+
/** A `model_usage` job gave up after its last attempt: the row records it. */
|
|
167
129
|
private markFailed;
|
|
168
130
|
/** The payload of one usage event, read from the log; null when the event no longer exists. */
|
|
169
131
|
private payloadOf;
|
|
@@ -183,19 +145,8 @@ export declare class UsageReporter {
|
|
|
183
145
|
private call;
|
|
184
146
|
}
|
|
185
147
|
/**
|
|
186
|
-
*
|
|
187
|
-
*
|
|
188
|
-
*/
|
|
189
|
-
export declare class UsageSweepJob {
|
|
190
|
-
private readonly reporter;
|
|
191
|
-
readonly lockService: LockService;
|
|
192
|
-
readonly log: KernelLogger;
|
|
193
|
-
constructor(reporter: Pick<UsageReporter, "sweep">, lockService: LockService, log: KernelLogger);
|
|
194
|
-
run(): Promise<void>;
|
|
195
|
-
}
|
|
196
|
-
/**
|
|
197
|
-
* `usage.report` delivery: the producer on every role; the consumer and the sweep ({@link UsageSweepJob})
|
|
198
|
-
* on the worker role (`WorkerModule`).
|
|
148
|
+
* `usage.report` delivery: the producer (`UsageReportJobs`) and the queue on every role; the
|
|
149
|
+
* consumer's `@Processor` on the worker role (`WorkerModule`).
|
|
199
150
|
*/
|
|
200
151
|
export declare class UsageReportModule {
|
|
201
152
|
}
|