@capybara-hq/kernel 0.1.0-rc.23 → 0.1.0-rc.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bootstrap.json +25 -25
- package/dist/admin/admin.module.d.ts +2 -1
- package/dist/admin/admin.module.js +9 -3
- package/dist/admin/health.service.d.ts +2 -2
- package/dist/admin/health.service.js +6 -2
- package/dist/admin/outbox-dead.controller.d.ts +23 -0
- package/dist/admin/outbox-dead.controller.js +62 -0
- package/dist/admin/outbox-dead.schemas.d.ts +8 -0
- package/dist/admin/outbox-dead.schemas.js +18 -0
- package/dist/admin/outbox-dead.service.d.ts +66 -0
- package/dist/admin/outbox-dead.service.js +179 -0
- package/dist/admin/overview.service.d.ts +40 -5
- package/dist/admin/overview.service.js +41 -7
- package/dist/agents/agents.module.js +3 -0
- package/dist/agents/agents.service.d.ts +6 -1
- package/dist/agents/agents.service.js +11 -2
- package/dist/agents/from-template.js +1 -1
- package/dist/background/finish.d.ts +5 -2
- package/dist/background/finish.js +14 -3
- package/dist/background/launch.d.ts +2 -1
- package/dist/background/launch.js +5 -2
- package/dist/background/store.d.ts +8 -0
- package/dist/background/store.js +24 -0
- package/dist/catalog/admin-catalog.controller.d.ts +21 -2
- package/dist/catalog/admin-catalog.controller.js +45 -6
- package/dist/catalog/admin-catalog.schemas.d.ts +5 -0
- package/dist/catalog/admin-catalog.schemas.js +8 -0
- package/dist/catalog/catalog-import.module.js +0 -4
- package/dist/catalog/catalog-scope.service.d.ts +63 -26
- package/dist/catalog/catalog-scope.service.js +146 -71
- package/dist/catalog/catalog.module.d.ts +2 -1
- package/dist/catalog/catalog.module.js +9 -4
- package/dist/catalog/catalog.service.d.ts +11 -4
- package/dist/catalog/catalog.service.js +21 -5
- package/dist/catalog/cli.d.ts +4 -3
- package/dist/catalog/cli.js +8 -13
- package/dist/catalog/job-ids.d.ts +8 -20
- package/dist/catalog/job-ids.js +11 -16
- package/dist/catalog/marketplaces.service.d.ts +1 -1
- package/dist/catalog/org-plugin-uploads.d.ts +11 -4
- package/dist/catalog/org-plugin-uploads.js +21 -6
- package/dist/catalog/owned-items.d.ts +6 -13
- package/dist/catalog/owned-items.js +11 -29
- package/dist/catalog/owned-plugin-cleanup.d.ts +8 -17
- package/dist/catalog/owned-plugin-cleanup.js +12 -66
- package/dist/catalog/platform-components.d.ts +56 -34
- package/dist/catalog/platform-components.js +68 -78
- package/dist/catalog/platform-import-update.d.ts +50 -0
- package/dist/catalog/platform-import-update.js +116 -0
- package/dist/catalog/plugin-import.d.ts +64 -26
- package/dist/catalog/plugin-import.js +113 -77
- package/dist/catalog/plugin-upgrade.d.ts +22 -4
- package/dist/catalog/plugin-upgrade.js +26 -19
- package/dist/catalog/scope-jobs.d.ts +59 -0
- package/dist/catalog/scope-jobs.js +65 -0
- package/dist/catalog/scope-sync-workers.d.ts +22 -0
- package/dist/catalog/scope-sync-workers.js +58 -2
- package/dist/catalog/scope-sync.d.ts +40 -65
- package/dist/catalog/scope-sync.js +114 -122
- package/dist/channels/channels.module.d.ts +5 -2
- package/dist/channels/channels.module.js +9 -2
- package/dist/channels/deliver-job.d.ts +25 -0
- package/dist/channels/deliver-job.js +25 -0
- package/dist/channels/inbound.d.ts +1 -1
- package/dist/channels/inbound.js +4 -17
- package/dist/channels/jobs.d.ts +28 -18
- package/dist/channels/jobs.js +63 -26
- package/dist/channels/threads.d.ts +4 -3
- package/dist/channels/threads.js +4 -3
- package/dist/config/schema.d.ts +0 -3
- package/dist/config/schema.js +11 -20
- package/dist/conversations/agent-tree.d.ts +16 -0
- package/dist/conversations/agent-tree.js +36 -0
- package/dist/conversations/conversations.controller.js +4 -2
- package/dist/conversations/conversations.service.d.ts +61 -19
- package/dist/conversations/conversations.service.js +69 -43
- package/dist/conversations/event-log.d.ts +7 -10
- package/dist/conversations/event-log.js +24 -15
- package/dist/conversations/events.d.ts +21 -0
- package/dist/conversations/events.js +26 -2
- package/dist/conversations/queue.service.d.ts +8 -3
- package/dist/conversations/queue.service.js +37 -13
- package/dist/conversations/source.d.ts +10 -0
- package/dist/conversations/source.js +17 -0
- package/dist/conversations/titles.d.ts +21 -8
- package/dist/conversations/titles.js +42 -15
- package/dist/conversations/tree.d.ts +10 -0
- package/dist/conversations/tree.js +35 -0
- package/dist/db/kernel-schema.d.ts +35 -22
- package/dist/devices/devices.service.d.ts +23 -11
- package/dist/devices/devices.service.js +34 -26
- package/dist/errors.d.ts +11 -0
- package/dist/errors.js +11 -0
- package/dist/evals/run-replay.d.ts +13 -2
- package/dist/evals/run-replay.js +42 -9
- package/dist/evals/snapshot.d.ts +7 -1
- package/dist/evals/snapshot.js +2 -1
- package/dist/extensions/extensions.module.js +3 -0
- package/dist/extensions/hooks.d.ts +22 -3
- package/dist/extensions/hooks.js +15 -1
- package/dist/extensions/lifecycle-hooks.d.ts +99 -0
- package/dist/extensions/lifecycle-hooks.js +273 -0
- package/dist/extensions/lifecycle.d.ts +6 -1
- package/dist/extensions/lifecycle.js +9 -2
- package/dist/extensions/org-config.d.ts +13 -1
- package/dist/extensions/org-config.js +19 -2
- package/dist/extensions/permission-view.d.ts +9 -5
- package/dist/extensions/permission-view.js +9 -4
- package/dist/extensions/point-defaults.d.ts +28 -1
- package/dist/extensions/point-defaults.js +44 -13
- package/dist/extensions/usage-subscribers.d.ts +43 -0
- package/dist/extensions/usage-subscribers.js +108 -0
- package/dist/http/api.module.js +2 -0
- package/dist/inbox/inbox.service.d.ts +6 -20
- package/dist/inbox/inbox.service.js +0 -3
- package/dist/inbox/org-notices.d.ts +1 -1
- package/dist/inbox/platform-inbox.service.d.ts +3 -32
- package/dist/inbox/platform-inbox.service.js +5 -54
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/infra/cli/kernel-cli.module.js +5 -0
- package/dist/infra/events/kernel-events.d.ts +9 -35
- package/dist/infra/outbox/outbox-relay.module.d.ts +6 -0
- package/dist/infra/outbox/outbox-relay.module.js +18 -0
- package/dist/infra/outbox/outbox.dispatcher.d.ts +103 -0
- package/dist/infra/outbox/outbox.dispatcher.js +310 -0
- package/dist/infra/outbox/outbox.module.d.ts +24 -0
- package/dist/infra/outbox/outbox.module.js +50 -0
- package/dist/infra/outbox/outbox.relay.d.ts +31 -0
- package/dist/infra/outbox/outbox.relay.js +76 -0
- package/dist/infra/outbox/outbox.service.d.ts +51 -0
- package/dist/infra/outbox/outbox.service.js +131 -0
- package/dist/infra/outbox/registry.d.ts +244 -0
- package/dist/infra/outbox/registry.js +234 -0
- package/dist/infra/queue/queue-names.d.ts +8 -0
- package/dist/infra/queue/queue-names.js +17 -0
- package/dist/kernel.module.js +3 -0
- package/dist/local-devices/device-policy.service.d.ts +4 -0
- package/dist/local-devices/device-policy.service.js +61 -3
- package/dist/local-devices/machines.d.ts +13 -5
- package/dist/local-devices/machines.js +45 -7
- package/dist/local-devices/policy-summary.d.ts +44 -1
- package/dist/local-devices/policy-summary.js +69 -0
- package/dist/models/embeddings.d.ts +9 -9
- package/dist/models/embeddings.js +11 -18
- package/dist/models/models.module.js +3 -0
- package/dist/orgs/directory-sync.js +1 -1
- package/dist/orgs/members.service.js +1 -1
- package/dist/orgs/offboarding.d.ts +50 -57
- package/dist/orgs/offboarding.js +117 -162
- package/dist/orgs/orgs.module.js +3 -0
- package/dist/orgs/orgs.service.d.ts +11 -5
- package/dist/orgs/orgs.service.js +22 -6
- package/dist/overrides.d.ts +23 -3
- package/dist/overrides.js +12 -2
- package/dist/platform/exclusions.js +5 -0
- package/dist/platform/ops/admin.ops.js +18 -2
- package/dist/platform/ops/catalog-admin.ops.d.ts +3 -1
- package/dist/platform/ops/catalog-admin.ops.js +32 -4
- package/dist/platform/ops/inbox.ops.js +4 -20
- package/dist/platform/ops/outbox.ops.d.ts +21 -0
- package/dist/platform/ops/outbox.ops.js +125 -0
- package/dist/platform/ops/platform.ops.d.ts +1 -0
- package/dist/platform/ops/platform.ops.js +6 -3
- package/dist/platform/ops/schedules.ops.js +40 -1
- package/dist/platform/platform-ops.module.d.ts +2 -1
- package/dist/platform/platform-ops.module.js +3 -0
- package/dist/platform/route-schemas.js +3 -0
- package/dist/roles/worker-role.d.ts +10 -5
- package/dist/roles/worker-role.js +19 -9
- package/dist/roles/worker.module.js +11 -8
- package/dist/runtime/context/build.d.ts +3 -0
- package/dist/runtime/context/build.js +15 -0
- package/dist/runtime/context/plugin-context.d.ts +20 -0
- package/dist/runtime/context/plugin-context.js +30 -0
- package/dist/runtime/langgraph-loop.js +17 -5
- package/dist/runtime/limits.d.ts +15 -0
- package/dist/runtime/limits.js +25 -0
- package/dist/runtime/loop.d.ts +16 -3
- package/dist/runtime/loop.js +2 -1
- package/dist/runtime/maintenance.d.ts +7 -3
- package/dist/runtime/maintenance.js +52 -12
- package/dist/runtime/model-call.d.ts +20 -17
- package/dist/runtime/model-call.js +15 -24
- package/dist/runtime/plugin-context.service.d.ts +69 -0
- package/dist/runtime/plugin-context.service.js +208 -0
- package/dist/runtime/queue-lost-jobs.d.ts +24 -0
- package/dist/runtime/queue-lost-jobs.js +51 -0
- package/dist/runtime/run-control.d.ts +47 -5
- package/dist/runtime/run-control.js +79 -12
- package/dist/runtime/run-finish.d.ts +39 -12
- package/dist/runtime/run-finish.js +97 -14
- package/dist/runtime/run-jobs.d.ts +95 -13
- package/dist/runtime/run-jobs.js +199 -28
- package/dist/runtime/run-orphan-check.job.d.ts +142 -0
- package/dist/runtime/run-orphan-check.job.js +339 -0
- package/dist/runtime/run-processors.d.ts +16 -9
- package/dist/runtime/run-processors.js +1 -1
- package/dist/runtime/run-queue.d.ts +75 -53
- package/dist/runtime/run-queue.js +189 -263
- package/dist/runtime/run-steps.d.ts +1 -2
- package/dist/runtime/run-steps.js +8 -11
- package/dist/runtime/run-worker.d.ts +26 -16
- package/dist/runtime/run-worker.js +132 -37
- package/dist/runtime/runtime.module.d.ts +2 -3
- package/dist/runtime/runtime.module.js +7 -6
- package/dist/runtime/step-log.d.ts +16 -4
- package/dist/runtime/step-log.js +26 -6
- package/dist/runtime/stream.controller.d.ts +2 -4
- package/dist/runtime/stream.controller.js +5 -11
- package/dist/runtime/subagents/child.js +1 -1
- package/dist/runtime/subagents/definition.d.ts +2 -1
- package/dist/runtime/subagents/definition.js +4 -2
- package/dist/runtime/subagents/deliveries.d.ts +6 -3
- package/dist/runtime/subagents/deliveries.js +16 -7
- package/dist/runtime/subagents/dispatch.d.ts +12 -12
- package/dist/runtime/subagents/dispatch.js +14 -38
- package/dist/runtime/subagents/messages.d.ts +5 -1
- package/dist/runtime/subagents/messages.js +11 -3
- package/dist/runtime/subagents/skill-fork.d.ts +3 -3
- package/dist/runtime/subagents/skill-fork.js +8 -5
- package/dist/runtime/subagents/subagents.service.js +1 -1
- package/dist/runtime/subagents/tree-jobs.d.ts +61 -0
- package/dist/runtime/subagents/tree-jobs.js +86 -0
- package/dist/runtime/usage-report-jobs.d.ts +23 -48
- package/dist/runtime/usage-report-jobs.js +67 -119
- package/dist/runtime/usage-report.d.ts +28 -77
- package/dist/runtime/usage-report.js +35 -213
- package/dist/runtime/waits.d.ts +48 -198
- package/dist/runtime/waits.js +103 -805
- package/dist/sandbox/jobs.d.ts +20 -10
- package/dist/sandbox/jobs.js +23 -17
- package/dist/sandbox/sandbox.service.d.ts +17 -4
- package/dist/sandbox/sandbox.service.js +55 -14
- package/dist/schedules/calendar.d.ts +134 -0
- package/dist/schedules/calendar.js +410 -0
- package/dist/schedules/fire.d.ts +26 -7
- package/dist/schedules/fire.js +91 -22
- package/dist/schedules/job-ids.d.ts +66 -17
- package/dist/schedules/job-ids.js +81 -13
- package/dist/schedules/run-tracker.d.ts +0 -5
- package/dist/schedules/run-tracker.js +2 -14
- package/dist/schedules/scan.job.d.ts +13 -32
- package/dist/schedules/scan.job.js +50 -164
- package/dist/schedules/schedule-math.d.ts +42 -0
- package/dist/schedules/schedule-math.js +52 -0
- package/dist/schedules/scheduled-tasks.service.d.ts +22 -6
- package/dist/schedules/scheduled-tasks.service.js +70 -14
- package/dist/schedules/schedules.controller.d.ts +5 -1
- package/dist/schedules/schedules.controller.js +15 -1
- package/dist/schedules/schedules.module.js +0 -4
- package/dist/schedules/schedules.schemas.d.ts +35 -0
- package/dist/schedules/schedules.schemas.js +27 -0
- package/dist/schedules/store.d.ts +5 -0
- package/dist/schedules/store.js +20 -0
- package/dist/skills/changes.controller.d.ts +1 -1
- package/dist/skills/changes.controller.js +1 -1
- package/dist/tools/approvals.service.d.ts +41 -11
- package/dist/tools/approvals.service.js +108 -14
- package/dist/tools/builtin-names.d.ts +5 -3
- package/dist/tools/builtin-names.js +12 -4
- package/dist/tools/builtins/builtin-tools.d.ts +1 -2
- package/dist/tools/builtins/builtin-tools.js +21 -17
- package/dist/tools/builtins/sandbox/bash.js +9 -3
- package/dist/tools/builtins/sandbox/common.d.ts +18 -1
- package/dist/tools/builtins/sandbox/common.js +14 -1
- package/dist/tools/builtins/sandbox/index.d.ts +2 -1
- package/dist/tools/builtins/sandbox/index.js +4 -1
- package/dist/tools/builtins/sandbox/task-list.d.ts +12 -0
- package/dist/tools/builtins/sandbox/task-list.js +130 -0
- package/dist/tools/builtins/sandbox/task-output.d.ts +8 -1
- package/dist/tools/builtins/sandbox/task-output.js +21 -3
- package/dist/tools/builtins/web/extract.d.ts +2 -4
- package/dist/tools/builtins/web/extract.js +2 -5
- package/dist/tools/pipeline.d.ts +12 -3
- package/dist/tools/pipeline.js +6 -1
- package/dist/tools/questions.service.d.ts +30 -10
- package/dist/tools/questions.service.js +41 -27
- package/dist/tools/registry.d.ts +12 -3
- package/dist/tools/registry.js +13 -3
- package/dist/tools/tools.module.js +4 -3
- package/package.json +3 -3
- package/src/migrations/0118_device_policy_ops.sql +6 -0
- package/src/migrations/0119_outbox.sql +22 -0
- package/src/migrations/0120_drop_usage_report_sweep.sql +7 -0
- package/src/migrations/0121_drop_platform_migration_runs.sql +12 -0
- package/src/migrations/0122_drop_scheduled_task_runs_rechecked_at.sql +5 -0
- package/src/migrations/0123_pgvector.sql +27 -0
- package/src/migrations/0124_conversation_source.sql +36 -0
- package/dist/catalog/platform-migration.d.ts +0 -115
- package/dist/catalog/platform-migration.js +0 -583
- package/dist/infra/queue/add-job-once.d.ts +0 -14
- package/dist/infra/queue/add-job-once.js +0 -16
package/dist/schedules/fire.js
CHANGED
|
@@ -16,20 +16,20 @@ import { ConversationAccess, ConversationAccessModule } from "../conversations/a
|
|
|
16
16
|
import { ConversationsModule } from "../conversations/conversations.module.js";
|
|
17
17
|
import { ConversationsService } from "../conversations/conversations.service.js";
|
|
18
18
|
import { ConversationQueue } from "../conversations/queue.service.js";
|
|
19
|
+
import { channelSource } from "../conversations/source.js";
|
|
19
20
|
import { ApiError } from "../errors.js";
|
|
20
21
|
import { KernelConfigService } from "../infra/config/kernel-config.service.js";
|
|
21
22
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
22
23
|
import { TransactionHost } from "../infra/database/transaction-host.js";
|
|
23
24
|
import { Transactional } from "../infra/database/transactional.decorator.js";
|
|
24
25
|
import { LockService } from "../infra/lock/lock.service.js";
|
|
25
|
-
import {
|
|
26
|
+
import { OutboxService } from "../infra/outbox/outbox.service.js";
|
|
26
27
|
import { KERNEL_LOGGER } from "../infra/tokens.js";
|
|
27
28
|
import { deviceLeaseKey } from "../local-devices/keys.js";
|
|
28
29
|
import { OrgLock, OrgLockModule } from "../orgs/org-lock.js";
|
|
29
|
-
import { MAINTENANCE_QUEUE_NAME } from "../runtime/maintenance-queue.js";
|
|
30
|
-
import { RunControl } from "../runtime/run-control.js";
|
|
31
30
|
import { RuntimeModule } from "../runtime/runtime.module.js";
|
|
32
31
|
import { readScheduleSettings } from "./effective-settings.js";
|
|
32
|
+
import { oweSchedSettle, SCHED_LATE_FIRE_SLACK_MS, SCHED_SETTLE_AFTER_MS, SCHED_SETTLE_LONG_CHECKS, SCHED_SETTLE_RECHECK_MS, } from "./job-ids.js";
|
|
33
33
|
import { pauseTaskIn } from "./pause.js";
|
|
34
34
|
import { lockTaskRow, ScheduledTaskRunTracker, ScheduledTaskTrackerModule } from "./run-tracker.js";
|
|
35
35
|
import { ScheduledTaskScanJob } from "./scan.job.js";
|
|
@@ -70,10 +70,12 @@ const DEVICE_GONE = new Set(["device.not_found", "device.unavailable", "sandbox.
|
|
|
70
70
|
*
|
|
71
71
|
* 1. The claim, one transaction under the org row lock (which serializes the org's rate limit
|
|
72
72
|
* count), then the task run and the task rows: the checks in order (task paused since the
|
|
73
|
-
* scan, unless a person's "run now"; org
|
|
74
|
-
*
|
|
73
|
+
* scan, unless a person's "run now"; a tick fired more than 10 minutes past the org's catch-up
|
|
74
|
+
* window, `missed`, unless a "run now"; org disabled, creator inactive or not a member, agent
|
|
75
|
+
* gone or not visible, conversation gone, connection inactive,
|
|
75
76
|
* overlap, rate limit, device offline, queue paused), each a `skipped` run with its reason, the
|
|
76
|
-
* lasting ones also pausing the task; then `fired_at` is written, after the rate limit count
|
|
77
|
+
* lasting ones also pausing the task; then `fired_at` is written, after the rate limit count,
|
|
78
|
+
* with the run's delayed settle check owed (`sched-settle`, {@link settle}).
|
|
77
79
|
* A run whose claim committed before (a retried job) goes straight to the start.
|
|
78
80
|
* 2. The start, idempotent: `new` creates the conversation with its first message keyed by the
|
|
79
81
|
* task run id (`create_request_id`); `conversation` queues the prompt in the target
|
|
@@ -97,12 +99,14 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
|
|
|
97
99
|
conversations;
|
|
98
100
|
queue;
|
|
99
101
|
connections;
|
|
100
|
-
runs;
|
|
101
102
|
locks;
|
|
102
103
|
log;
|
|
103
|
-
|
|
104
|
+
outbox;
|
|
105
|
+
constructor(kdb, txHost, config, store, tracker, orgLock, agents, access, conversations, queue, connections,
|
|
104
106
|
/** Reads a local device's connection lease (online or not). */
|
|
105
|
-
locks, log
|
|
107
|
+
locks, log,
|
|
108
|
+
/** The claim owes the run's settle check (ruling R26). */
|
|
109
|
+
outbox) {
|
|
106
110
|
this.kdb = kdb;
|
|
107
111
|
this.txHost = txHost;
|
|
108
112
|
this.config = config;
|
|
@@ -114,9 +118,9 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
|
|
|
114
118
|
this.conversations = conversations;
|
|
115
119
|
this.queue = queue;
|
|
116
120
|
this.connections = connections;
|
|
117
|
-
this.runs = runs;
|
|
118
121
|
this.locks = locks;
|
|
119
122
|
this.log = log;
|
|
123
|
+
this.outbox = outbox;
|
|
120
124
|
}
|
|
121
125
|
get db() {
|
|
122
126
|
return this.kdb.db;
|
|
@@ -140,6 +144,61 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
|
|
|
140
144
|
return;
|
|
141
145
|
await this.start(claimed.plan);
|
|
142
146
|
}
|
|
147
|
+
/**
|
|
148
|
+
* The `sched-settle` job (ruling R26; `job-ids.ts`). The claimed check: a task run claimed
|
|
149
|
+
* (`fired_at`) and still `pending` goes through the run tracker's check, which links it to its
|
|
150
|
+
* run, settles it, or releases a run that left no trace (`skipped`, `removed`), so a `once`
|
|
151
|
+
* task's last run never stays pending; one still in flight (its item queued, its message posted)
|
|
152
|
+
* owes the next check an hour on. The unfired check: a run still `pending` that no claim took
|
|
153
|
+
* (its fire job failed for good before the claim) is skipped, `missed` for a tick, `removed` for
|
|
154
|
+
* a "run now", so the overlap check never waits on it. A run claimed, started, ended or skipped
|
|
155
|
+
* since is left as it is (the claimed check follows a claimed run).
|
|
156
|
+
*/
|
|
157
|
+
async settle(data) {
|
|
158
|
+
const { taskRunId } = data;
|
|
159
|
+
if ("unfired" in data) {
|
|
160
|
+
await this.settleUnfired(taskRunId);
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
const run = await this.store.updateRun(this.db, taskRunId, {});
|
|
164
|
+
if (run?.status !== "pending" || run.firedAt === null)
|
|
165
|
+
return;
|
|
166
|
+
// inFlightIn takes the task row before the run (the one lock order)
|
|
167
|
+
if (!(await this.tracker.inFlightIn(run)))
|
|
168
|
+
return;
|
|
169
|
+
const now = await this.store.updateRun(this.db, taskRunId, {});
|
|
170
|
+
if (now?.status !== "pending" || now.firedAt === null)
|
|
171
|
+
return;
|
|
172
|
+
const next = data.check + 1;
|
|
173
|
+
if (next === SCHED_SETTLE_LONG_CHECKS) {
|
|
174
|
+
this.log.warn("scheduled task run still in flight a week after its claim", {
|
|
175
|
+
taskId: run.taskId,
|
|
176
|
+
taskRunId,
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
await oweSchedSettle(this.outbox, taskRunId, run.orgId, next, SCHED_SETTLE_RECHECK_MS);
|
|
180
|
+
}
|
|
181
|
+
/** The unfired check (see {@link settle}), task row first (the one lock order). */
|
|
182
|
+
async settleUnfired(taskRunId) {
|
|
183
|
+
const t = this.db;
|
|
184
|
+
const owner = await t
|
|
185
|
+
.selectFrom("scheduled_task_runs")
|
|
186
|
+
.select("task_id")
|
|
187
|
+
.where("id", "=", taskRunId)
|
|
188
|
+
.executeTakeFirst();
|
|
189
|
+
if (!owner)
|
|
190
|
+
return;
|
|
191
|
+
await lockTaskRow(t, owner.task_id);
|
|
192
|
+
const run = await t
|
|
193
|
+
.selectFrom("scheduled_task_runs")
|
|
194
|
+
.select(["status", "fired_at", "manual"])
|
|
195
|
+
.where("id", "=", taskRunId)
|
|
196
|
+
.forUpdate()
|
|
197
|
+
.executeTakeFirst();
|
|
198
|
+
if (run?.status !== "pending" || run.fired_at !== null)
|
|
199
|
+
return;
|
|
200
|
+
await this.skipIn(t, taskRunId, owner.task_id, run.manual ? "removed" : "missed");
|
|
201
|
+
}
|
|
143
202
|
/** Step 1 (see the class): a skip, the plan, or null when there is nothing to fire. */
|
|
144
203
|
async claim(taskRunId, orgId, deviceOffline) {
|
|
145
204
|
const t = this.db;
|
|
@@ -182,6 +241,12 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
|
|
|
182
241
|
if (!resumed && !run.manual && task.status === "paused")
|
|
183
242
|
return skip("task_paused");
|
|
184
243
|
const settings = await readScheduleSettings(t, this.config, orgId);
|
|
244
|
+
// a tick whose fire comes this late (workers down for long) is missed, as the scan records a
|
|
245
|
+
// tick it finds outside the window; a person's "run now" still runs
|
|
246
|
+
const lateAfter = settings.catchUpWindowMinutes * 60_000 + SCHED_LATE_FIRE_SLACK_MS;
|
|
247
|
+
if (!resumed && !run.manual && Date.now() - run.scheduledFor.getTime() > lateAfter) {
|
|
248
|
+
return skip("missed");
|
|
249
|
+
}
|
|
185
250
|
if (!resumed && !settings.enabled)
|
|
186
251
|
return skip("org_disabled");
|
|
187
252
|
if (!resumed && !(await this.creatorActive(orgId, task.accountId))) {
|
|
@@ -263,6 +328,8 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
|
|
|
263
328
|
firedAt: new Date(),
|
|
264
329
|
...(plan.mode === "conversation" && { conversationId: plan.conversationId }),
|
|
265
330
|
});
|
|
331
|
+
// a start that fails for good leaves the run claimed: its settle check releases or ends it
|
|
332
|
+
await oweSchedSettle(this.outbox, run.id, orgId, 0, SCHED_SETTLE_AFTER_MS);
|
|
266
333
|
return { plan: { ...plan, run: claimed ?? run } };
|
|
267
334
|
}
|
|
268
335
|
/** Step 2 (see the class). */
|
|
@@ -285,11 +352,10 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
|
|
|
285
352
|
...(task.model !== null && { model: task.model }),
|
|
286
353
|
...(task.effort !== null && { effort: task.effort }),
|
|
287
354
|
};
|
|
288
|
-
const out = await this.conversations.createWithMessage(task.orgId, task.accountId, task.agentId, { text: task.prompt, clientMessageId: run.id, ...pick }, task.deviceId === null ? {} : { deviceId: task.deviceId }, internal);
|
|
355
|
+
const out = await this.conversations.createWithMessage(task.orgId, task.accountId, task.agentId, { text: task.prompt, clientMessageId: run.id, ...pick }, task.deviceId === null ? {} : { deviceId: task.deviceId }, { ...internal, source: "web" });
|
|
356
|
+
// a retried start found the conversation made before: `createWithMessage` already asked
|
|
357
|
+
// for its run (and the first start's message owes the ensure-run backstop)
|
|
289
358
|
await this.setConversation(run.id, out.conversation.id);
|
|
290
|
-
// a retried start found the conversation made before: its run may be still to start
|
|
291
|
-
if (!out.created)
|
|
292
|
-
await this.runs.ensureRun(out.conversation.id);
|
|
293
359
|
return;
|
|
294
360
|
}
|
|
295
361
|
let conversationId;
|
|
@@ -349,7 +415,7 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
|
|
|
349
415
|
const { conversationId, created } = await findOrBindThread(t, plan.thread, plan.target, async () => {
|
|
350
416
|
const conv = await this.conversations.create(task.orgId, task.accountId, task.agentId,
|
|
351
417
|
// a chat task takes no device (new mode only): an IM conversation's default
|
|
352
|
-
{ im: true });
|
|
418
|
+
{ im: true }, { source: channelSource(plan.target.chatType) });
|
|
353
419
|
await t
|
|
354
420
|
.updateTable("conversations")
|
|
355
421
|
.set({ scheduled_task_id: task.id })
|
|
@@ -386,15 +452,17 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
|
|
|
386
452
|
return null;
|
|
387
453
|
if (locked.conversation_id !== null)
|
|
388
454
|
return locked.conversation_id;
|
|
455
|
+
const target = { chatId: plan.chatId, chatType: "group" };
|
|
389
456
|
const conv = await this.conversations.create(task.orgId, task.accountId, task.agentId,
|
|
390
457
|
// a chat task takes no device (new mode only): an IM conversation's default
|
|
391
|
-
{ im: true }
|
|
458
|
+
{ im: true },
|
|
459
|
+
// stored now: the thread to derive it from binds only at the run's first post
|
|
460
|
+
{ source: channelSource(target.chatType) });
|
|
392
461
|
await t
|
|
393
462
|
.updateTable("conversations")
|
|
394
463
|
.set({ scheduled_task_id: task.id })
|
|
395
464
|
.where("id", "=", conv.id)
|
|
396
465
|
.execute();
|
|
397
|
-
const target = { chatId: plan.chatId, chatType: "group" };
|
|
398
466
|
await t
|
|
399
467
|
.insertInto("channel_deliveries")
|
|
400
468
|
.values({
|
|
@@ -562,6 +630,9 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
|
|
|
562
630
|
return (await this.locks.holderOf(deviceLeaseKey(deviceId))) === null;
|
|
563
631
|
}
|
|
564
632
|
};
|
|
633
|
+
__decorate([
|
|
634
|
+
Transactional()
|
|
635
|
+
], ScheduledTaskFirer.prototype, "settle", null);
|
|
565
636
|
__decorate([
|
|
566
637
|
Transactional()
|
|
567
638
|
], ScheduledTaskFirer.prototype, "claim", null);
|
|
@@ -593,9 +664,9 @@ ScheduledTaskFirer = __decorate([
|
|
|
593
664
|
__param(8, Inject(ConversationsService)),
|
|
594
665
|
__param(9, Inject(ConversationQueue)),
|
|
595
666
|
__param(10, Inject(ChannelConnections)),
|
|
596
|
-
__param(11, Inject(
|
|
597
|
-
__param(12, Inject(
|
|
598
|
-
__param(13, Inject(
|
|
667
|
+
__param(11, Inject(LockService)),
|
|
668
|
+
__param(12, Inject(KERNEL_LOGGER)),
|
|
669
|
+
__param(13, Inject(OutboxService))
|
|
599
670
|
], ScheduledTaskFirer);
|
|
600
671
|
export { ScheduledTaskFirer };
|
|
601
672
|
/**
|
|
@@ -615,8 +686,6 @@ ScheduledTaskFireModule = __decorate([
|
|
|
615
686
|
OrgLockModule,
|
|
616
687
|
ScheduledTaskStoreModule,
|
|
617
688
|
ScheduledTaskTrackerModule,
|
|
618
|
-
// the scanner enqueues the fire jobs
|
|
619
|
-
QueueModule.registerQueue(MAINTENANCE_QUEUE_NAME),
|
|
620
689
|
],
|
|
621
690
|
providers: [ScheduledTaskFirer, ScheduledTaskScanJob],
|
|
622
691
|
exports: [ScheduledTaskFirer],
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
import type { OutboxService } from "../infra/outbox/outbox.service.js";
|
|
2
|
+
/** The scheduled tasks' jobs (a leaf module: type imports only). */
|
|
2
3
|
/**
|
|
3
4
|
* The once-a-minute `@LockedCron` that turns due tasks into task runs (spec §6); its name enqueued
|
|
4
5
|
* on the maintenance queue runs one scan (`ScheduledRuns.runNow`).
|
|
@@ -13,20 +14,68 @@ export interface SchedFireJobData {
|
|
|
13
14
|
/** The fire job's id: `sched-fire-<taskRunId>` (no `:`, task run ids are UUIDs). */
|
|
14
15
|
export declare const schedFireJobId: (taskRunId: string) => string;
|
|
15
16
|
/**
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
17
|
+
* Owes the fire job of task run `taskRunId` in the transaction in context (transactional outbox
|
|
18
|
+
* spec §8.1): the transaction that writes a pending task run (the scan's tick, a person's "run
|
|
19
|
+
* now") owes its fire, so a tick is never left unfired. The job's options are its outbox registry
|
|
20
|
+
* entry's; the fire is idempotent on the task run's status.
|
|
21
|
+
*/
|
|
22
|
+
export declare function oweSchedFire(outbox: Pick<OutboxService, "add">, taskRunId: string, orgId: string): Promise<boolean>;
|
|
23
|
+
/**
|
|
24
|
+
* The delayed job that settles a task run left pending (ruling R26, fix round 1), on
|
|
25
|
+
* `cap-kernel-maintenance`. Two kinds:
|
|
26
|
+
* - the claimed check, owed by the fire job's claim: the run tracker links, settles or releases a
|
|
27
|
+
* run claimed but never started (`sched-settle-<taskRunId>`, then `-<check>`);
|
|
28
|
+
* - the unfired check, owed with the task run itself (the scan's tick, "run now"): a run its fire
|
|
29
|
+
* job never claimed (the job failed for good before its claim) is skipped, `missed` for a tick,
|
|
30
|
+
* `removed` for a "run now", so it never blocks the task's later ticks as an overlap
|
|
31
|
+
* (`sched-settle-<taskRunId>-unfired`).
|
|
32
|
+
*/
|
|
33
|
+
export declare const SCHED_SETTLE_JOB = "sched-settle";
|
|
34
|
+
/**
|
|
35
|
+
* The data of a {@link SCHED_SETTLE_JOB}: the claimed check (`check` counts the checks before this
|
|
36
|
+
* one) or the unfired check.
|
|
37
|
+
*/
|
|
38
|
+
export type SchedSettleJobData = {
|
|
39
|
+
taskRunId: string;
|
|
40
|
+
check: number;
|
|
41
|
+
} | {
|
|
42
|
+
taskRunId: string;
|
|
43
|
+
unfired: true;
|
|
32
44
|
};
|
|
45
|
+
/**
|
|
46
|
+
* The first claimed check: 10 minutes after the claim, when the start's retries (seconds) are long
|
|
47
|
+
* over and the run tracker's start grace (`START_GRACE_MS`, 5 minutes) has passed, so a run that
|
|
48
|
+
* left no trace is released at once. Also the unfired check's delay after a "run now".
|
|
49
|
+
*/
|
|
50
|
+
export declare const SCHED_SETTLE_AFTER_MS: number;
|
|
51
|
+
/**
|
|
52
|
+
* A claimed run still in flight at a check (its item queued, its message posted and not yet taken
|
|
53
|
+
* by a run) is checked again hourly. The chain ends with the first check that finds the run
|
|
54
|
+
* started, ended or released: the queued item is consumed or removed, the message taken by a run.
|
|
55
|
+
* A chain past {@link SCHED_SETTLE_LONG_CHECKS} checks (a week) is logged as a warning, once.
|
|
56
|
+
*/
|
|
57
|
+
export declare const SCHED_SETTLE_RECHECK_MS: number;
|
|
58
|
+
/** A week of hourly checks: the claimed check warns at this one (a run in flight that long). */
|
|
59
|
+
export declare const SCHED_SETTLE_LONG_CHECKS = 168;
|
|
60
|
+
/**
|
|
61
|
+
* How long past the org's catch-up window a tick's fire may come before the tick is missed (the
|
|
62
|
+
* fire job's own rule, and when a tick's unfired check runs): the scan fires a tick still inside
|
|
63
|
+
* the window, and its fire job follows within seconds.
|
|
64
|
+
*/
|
|
65
|
+
export declare const SCHED_LATE_FIRE_SLACK_MS: number;
|
|
66
|
+
/** The claimed check's id: `sched-settle-<taskRunId>`, then `-<check>` for the later checks. */
|
|
67
|
+
export declare const schedSettleJobId: (taskRunId: string, check: number) => string;
|
|
68
|
+
/** The unfired check's id: `sched-settle-<taskRunId>-unfired`. */
|
|
69
|
+
export declare const schedSettleUnfiredJobId: (taskRunId: string) => string;
|
|
70
|
+
/**
|
|
71
|
+
* Owes claimed check number `check` of task run `taskRunId` in the transaction in context, due
|
|
72
|
+
* `after` milliseconds from now.
|
|
73
|
+
*/
|
|
74
|
+
export declare function oweSchedSettle(outbox: Pick<OutboxService, "add">, taskRunId: string, orgId: string, check: number, after: number): Promise<boolean>;
|
|
75
|
+
/**
|
|
76
|
+
* Owes the unfired check of task run `taskRunId` in the transaction that writes the run: for a
|
|
77
|
+
* tick of the schedule (`catchUpWindowMinutes` given) when the fire job would itself call the
|
|
78
|
+
* tick missed (its tick plus the window plus {@link SCHED_LATE_FIRE_SLACK_MS}, at least
|
|
79
|
+
* {@link SCHED_SETTLE_AFTER_MS} on); for a "run now" (null) {@link SCHED_SETTLE_AFTER_MS} on.
|
|
80
|
+
*/
|
|
81
|
+
export declare function oweSchedSettleUnfired(outbox: Pick<OutboxService, "add">, taskRunId: string, orgId: string, scheduledFor: Date, catchUpWindowMinutes: number | null): Promise<boolean>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
/** The scheduled tasks' jobs (a leaf module:
|
|
1
|
+
/** The scheduled tasks' jobs (a leaf module: type imports only). */
|
|
2
2
|
/**
|
|
3
3
|
* The once-a-minute `@LockedCron` that turns due tasks into task runs (spec §6); its name enqueued
|
|
4
4
|
* on the maintenance queue runs one scan (`ScheduledRuns.runNow`).
|
|
@@ -9,19 +9,87 @@ export const SCHED_FIRE_JOB = "sched-fire";
|
|
|
9
9
|
/** The fire job's id: `sched-fire-<taskRunId>` (no `:`, task run ids are UUIDs). */
|
|
10
10
|
export const schedFireJobId = (taskRunId) => `${SCHED_FIRE_JOB}-${taskRunId}`;
|
|
11
11
|
/**
|
|
12
|
-
*
|
|
13
|
-
*
|
|
12
|
+
* Owes the fire job of task run `taskRunId` in the transaction in context (transactional outbox
|
|
13
|
+
* spec §8.1): the transaction that writes a pending task run (the scan's tick, a person's "run
|
|
14
|
+
* now") owes its fire, so a tick is never left unfired. The job's options are its outbox registry
|
|
15
|
+
* entry's; the fire is idempotent on the task run's status.
|
|
14
16
|
*/
|
|
15
|
-
export function
|
|
16
|
-
return {
|
|
17
|
+
export function oweSchedFire(outbox, taskRunId, orgId) {
|
|
18
|
+
return outbox.add({
|
|
19
|
+
queue: "cap-kernel-maintenance",
|
|
17
20
|
name: SCHED_FIRE_JOB,
|
|
21
|
+
jobId: schedFireJobId(taskRunId),
|
|
18
22
|
data: { taskRunId },
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
23
|
+
orgId,
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The delayed job that settles a task run left pending (ruling R26, fix round 1), on
|
|
28
|
+
* `cap-kernel-maintenance`. Two kinds:
|
|
29
|
+
* - the claimed check, owed by the fire job's claim: the run tracker links, settles or releases a
|
|
30
|
+
* run claimed but never started (`sched-settle-<taskRunId>`, then `-<check>`);
|
|
31
|
+
* - the unfired check, owed with the task run itself (the scan's tick, "run now"): a run its fire
|
|
32
|
+
* job never claimed (the job failed for good before its claim) is skipped, `missed` for a tick,
|
|
33
|
+
* `removed` for a "run now", so it never blocks the task's later ticks as an overlap
|
|
34
|
+
* (`sched-settle-<taskRunId>-unfired`).
|
|
35
|
+
*/
|
|
36
|
+
export const SCHED_SETTLE_JOB = "sched-settle";
|
|
37
|
+
/**
|
|
38
|
+
* The first claimed check: 10 minutes after the claim, when the start's retries (seconds) are long
|
|
39
|
+
* over and the run tracker's start grace (`START_GRACE_MS`, 5 minutes) has passed, so a run that
|
|
40
|
+
* left no trace is released at once. Also the unfired check's delay after a "run now".
|
|
41
|
+
*/
|
|
42
|
+
export const SCHED_SETTLE_AFTER_MS = 10 * 60_000;
|
|
43
|
+
/**
|
|
44
|
+
* A claimed run still in flight at a check (its item queued, its message posted and not yet taken
|
|
45
|
+
* by a run) is checked again hourly. The chain ends with the first check that finds the run
|
|
46
|
+
* started, ended or released: the queued item is consumed or removed, the message taken by a run.
|
|
47
|
+
* A chain past {@link SCHED_SETTLE_LONG_CHECKS} checks (a week) is logged as a warning, once.
|
|
48
|
+
*/
|
|
49
|
+
export const SCHED_SETTLE_RECHECK_MS = 60 * 60_000;
|
|
50
|
+
/** A week of hourly checks: the claimed check warns at this one (a run in flight that long). */
|
|
51
|
+
export const SCHED_SETTLE_LONG_CHECKS = 168;
|
|
52
|
+
/**
|
|
53
|
+
* How long past the org's catch-up window a tick's fire may come before the tick is missed (the
|
|
54
|
+
* fire job's own rule, and when a tick's unfired check runs): the scan fires a tick still inside
|
|
55
|
+
* the window, and its fire job follows within seconds.
|
|
56
|
+
*/
|
|
57
|
+
export const SCHED_LATE_FIRE_SLACK_MS = 10 * 60_000;
|
|
58
|
+
/** The claimed check's id: `sched-settle-<taskRunId>`, then `-<check>` for the later checks. */
|
|
59
|
+
export const schedSettleJobId = (taskRunId, check) => check === 0 ? `${SCHED_SETTLE_JOB}-${taskRunId}` : `${SCHED_SETTLE_JOB}-${taskRunId}-${check}`;
|
|
60
|
+
/** The unfired check's id: `sched-settle-<taskRunId>-unfired`. */
|
|
61
|
+
export const schedSettleUnfiredJobId = (taskRunId) => `${SCHED_SETTLE_JOB}-${taskRunId}-unfired`;
|
|
62
|
+
/**
|
|
63
|
+
* Owes claimed check number `check` of task run `taskRunId` in the transaction in context, due
|
|
64
|
+
* `after` milliseconds from now.
|
|
65
|
+
*/
|
|
66
|
+
export function oweSchedSettle(outbox, taskRunId, orgId, check, after) {
|
|
67
|
+
return outbox.add({
|
|
68
|
+
queue: "cap-kernel-maintenance",
|
|
69
|
+
name: SCHED_SETTLE_JOB,
|
|
70
|
+
jobId: schedSettleJobId(taskRunId, check),
|
|
71
|
+
data: { taskRunId, check },
|
|
72
|
+
runAt: new Date(Date.now() + after),
|
|
73
|
+
orgId,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Owes the unfired check of task run `taskRunId` in the transaction that writes the run: for a
|
|
78
|
+
* tick of the schedule (`catchUpWindowMinutes` given) when the fire job would itself call the
|
|
79
|
+
* tick missed (its tick plus the window plus {@link SCHED_LATE_FIRE_SLACK_MS}, at least
|
|
80
|
+
* {@link SCHED_SETTLE_AFTER_MS} on); for a "run now" (null) {@link SCHED_SETTLE_AFTER_MS} on.
|
|
81
|
+
*/
|
|
82
|
+
export function oweSchedSettleUnfired(outbox, taskRunId, orgId, scheduledFor, catchUpWindowMinutes) {
|
|
83
|
+
const soonest = Date.now() + SCHED_SETTLE_AFTER_MS;
|
|
84
|
+
const at = catchUpWindowMinutes === null
|
|
85
|
+
? soonest
|
|
86
|
+
: Math.max(soonest, scheduledFor.getTime() + catchUpWindowMinutes * 60_000 + SCHED_LATE_FIRE_SLACK_MS);
|
|
87
|
+
return outbox.add({
|
|
88
|
+
queue: "cap-kernel-maintenance",
|
|
89
|
+
name: SCHED_SETTLE_JOB,
|
|
90
|
+
jobId: schedSettleUnfiredJobId(taskRunId),
|
|
91
|
+
data: { taskRunId, unfired: true },
|
|
92
|
+
runAt: new Date(at),
|
|
93
|
+
orgId,
|
|
94
|
+
});
|
|
27
95
|
}
|
|
@@ -59,11 +59,6 @@ export declare class ScheduledTaskRunTracker {
|
|
|
59
59
|
* sub-agent or wake run inheriting the marker) is left alone.
|
|
60
60
|
*/
|
|
61
61
|
export declare function linkStartedRunIn(t: KernelSchemaDb, taskRunId: string, runId: string, conversationId: string): Promise<void>;
|
|
62
|
-
/**
|
|
63
|
-
* A queued run row was taken back before it ran (`RunQueue.deleteQueued`: its job could not be
|
|
64
|
-
* enqueued): the task run it had taken waits for the next run of its message again.
|
|
65
|
-
*/
|
|
66
|
-
export declare function unlinkRunIn(t: KernelSchemaDb, taskRunId: string, runId: string): Promise<void>;
|
|
67
62
|
/**
|
|
68
63
|
* A person removed, edited or sent the queued item that carried `schedule` (the item's stored
|
|
69
64
|
* marker, or null): its task run will never run as the task's, so it becomes `skipped`
|
|
@@ -114,7 +114,8 @@ let ScheduledTaskRunTracker = class ScheduledTaskRunTracker {
|
|
|
114
114
|
await this.settle(tr, run.status);
|
|
115
115
|
return false;
|
|
116
116
|
}
|
|
117
|
-
// not
|
|
117
|
+
// not claimed yet: its fire job is still to come (the run's unfired settle check skips it
|
|
118
|
+
// when the job failed for good before its claim)
|
|
118
119
|
if (tr.fired_at === null)
|
|
119
120
|
return true;
|
|
120
121
|
// only a run with a target conversation is ever queued (`conversation`, `chat`): no scan of
|
|
@@ -281,19 +282,6 @@ export async function linkStartedRunIn(t, taskRunId, runId, conversationId) {
|
|
|
281
282
|
.where("run_id", "is", null)
|
|
282
283
|
.execute();
|
|
283
284
|
}
|
|
284
|
-
/**
|
|
285
|
-
* A queued run row was taken back before it ran (`RunQueue.deleteQueued`: its job could not be
|
|
286
|
-
* enqueued): the task run it had taken waits for the next run of its message again.
|
|
287
|
-
*/
|
|
288
|
-
export async function unlinkRunIn(t, taskRunId, runId) {
|
|
289
|
-
await t
|
|
290
|
-
.updateTable("scheduled_task_runs")
|
|
291
|
-
.set({ status: "pending", run_id: null })
|
|
292
|
-
.where("id", "=", taskRunId)
|
|
293
|
-
.where("run_id", "=", runId)
|
|
294
|
-
.where("status", "=", "running")
|
|
295
|
-
.execute();
|
|
296
|
-
}
|
|
297
285
|
/**
|
|
298
286
|
* A person removed, edited or sent the queued item that carried `schedule` (the item's stored
|
|
299
287
|
* marker, or null): its task run will never run as the task's, so it becomes `skipped`
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import type { Queue } from "bullmq";
|
|
2
1
|
import { KernelConfigService } from "../infra/config/kernel-config.service.js";
|
|
3
2
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
4
3
|
import { TransactionHost } from "../infra/database/transaction-host.js";
|
|
5
4
|
import { LockService } from "../infra/lock/lock.service.js";
|
|
5
|
+
import { OutboxService } from "../infra/outbox/outbox.service.js";
|
|
6
6
|
import type { KernelLogger } from "../log.js";
|
|
7
|
-
import { ScheduledTaskRunTracker } from "./run-tracker.js";
|
|
8
7
|
import { ScheduledTaskStore } from "./store.js";
|
|
9
8
|
/**
|
|
10
9
|
* The scheduled task scanner (spec §6), once a minute across replicas. Each due task in its own
|
|
@@ -12,34 +11,27 @@ import { ScheduledTaskStore } from "./store.js";
|
|
|
12
11
|
* same task): the ticks missed outside the org's `catchUpWindowMinutes` become one `skipped` run
|
|
13
12
|
* (`missed`, at the first missed tick), the latest tick in the window gets its run (the unique
|
|
14
13
|
* `(task_id, scheduled_for)` key makes a second insert a no-op), `next_run_at` moves to the next
|
|
15
|
-
* tick plus the task's jitter (or a `once` task ends, or one past its `expires_at`), and
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* still `pending` (its start failed for good) goes through the run tracker's check, which links,
|
|
23
|
-
* settles or releases it, so a `once` task's last run never stays pending.
|
|
14
|
+
* tick plus the task's jitter (or a `once` task ends, or one past its `expires_at`), and the same
|
|
15
|
+
* transaction owes `sched-fire-<taskRunId>` (transactional outbox spec §8.1), so a written tick is
|
|
16
|
+
* never left unfired and the scan needs no repair of lost fire jobs (a fire delivered too late
|
|
17
|
+
* marks its tick missed itself, `ScheduledTaskFirer`), and the run's unfired settle check (a fire
|
|
18
|
+
* job that failed for good before its claim: the tick is missed, `job-ids.ts`). A task whose stored trigger the schedule
|
|
19
|
+
* math cannot evaluate is paused (`invalid_schedule`) instead of failing the scan; a database
|
|
20
|
+
* error fails the pass (the next minute retries) and pauses nothing.
|
|
24
21
|
*/
|
|
25
22
|
export declare class ScheduledTaskScanJob {
|
|
26
23
|
private readonly kdb;
|
|
27
24
|
readonly txHost: TransactionHost;
|
|
28
25
|
private readonly config;
|
|
29
26
|
private readonly store;
|
|
30
|
-
private readonly
|
|
31
|
-
private readonly maintenance;
|
|
27
|
+
private readonly outbox;
|
|
32
28
|
readonly lockService: LockService;
|
|
33
29
|
readonly log: KernelLogger;
|
|
34
|
-
constructor(kdb: KernelDb, txHost: TransactionHost, config: Pick<KernelConfigService, "get">, store: ScheduledTaskStore,
|
|
30
|
+
constructor(kdb: KernelDb, txHost: TransactionHost, config: Pick<KernelConfigService, "get">, store: ScheduledTaskStore, outbox: Pick<OutboxService, "add">, lockService: LockService, log: KernelLogger);
|
|
35
31
|
private get db();
|
|
36
32
|
scan(): Promise<void>;
|
|
37
|
-
/**
|
|
38
|
-
|
|
39
|
-
* runs even when the scan loop failed (its error is rethrown afterwards). `repairBatch`: how many
|
|
40
|
-
* rows each repair step takes (tests pass a small one).
|
|
41
|
-
*/
|
|
42
|
-
scanAt(now: Date, repairBatch?: number): Promise<number>;
|
|
33
|
+
/** One scan as of `now` (tests pass their own); returns how many due tasks it handled. */
|
|
34
|
+
scanAt(now: Date): Promise<number>;
|
|
43
35
|
/**
|
|
44
36
|
* One due task in its own transaction (a task whose trigger cannot be evaluated rolls back
|
|
45
37
|
* alone and is paused, so it cannot block the others): locked with `SKIP LOCKED`, so replicas take different tasks. False
|
|
@@ -48,17 +40,6 @@ export declare class ScheduledTaskScanJob {
|
|
|
48
40
|
private scanOne;
|
|
49
41
|
/** A task the scan cannot advance stops being due: paused with `invalid_schedule`. */
|
|
50
42
|
private quarantine;
|
|
51
|
-
/** One due task (see the class); returns the
|
|
43
|
+
/** One due task (see the class); returns the run to fire, if any. */
|
|
52
44
|
private advance;
|
|
53
|
-
/** Enqueues the fire jobs (after the commit; a failure is repaired by a later scan). */
|
|
54
|
-
private enqueue;
|
|
55
|
-
/** Lost fire jobs (see the class). */
|
|
56
|
-
private repair;
|
|
57
|
-
/** A run never fired an hour after its tick: given up, task row first (the one lock order). */
|
|
58
|
-
private giveUp;
|
|
59
|
-
/**
|
|
60
|
-
* A started run still pending long after: the tracker links, settles or releases it; one still
|
|
61
|
-
* in flight is stamped (`rechecked_at`) and goes to the back of the next rechecks.
|
|
62
|
-
*/
|
|
63
|
-
private recheck;
|
|
64
45
|
}
|