@capybara-hq/kernel 0.1.0-rc.24 → 0.1.0-rc.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/dist/admin/admin.module.js +2 -0
  2. package/dist/admin/health.service.d.ts +2 -2
  3. package/dist/admin/health.service.js +6 -2
  4. package/dist/agents/agents.module.js +3 -0
  5. package/dist/agents/agents.service.d.ts +6 -1
  6. package/dist/agents/agents.service.js +11 -2
  7. package/dist/catalog/catalog-import.module.js +0 -4
  8. package/dist/catalog/catalog-scope.service.d.ts +63 -26
  9. package/dist/catalog/catalog-scope.service.js +146 -71
  10. package/dist/catalog/catalog.module.js +4 -3
  11. package/dist/catalog/catalog.service.d.ts +11 -4
  12. package/dist/catalog/catalog.service.js +21 -5
  13. package/dist/catalog/cli.d.ts +4 -3
  14. package/dist/catalog/cli.js +8 -13
  15. package/dist/catalog/job-ids.d.ts +8 -20
  16. package/dist/catalog/job-ids.js +11 -16
  17. package/dist/catalog/marketplaces.service.d.ts +1 -1
  18. package/dist/catalog/org-plugin-uploads.d.ts +11 -4
  19. package/dist/catalog/org-plugin-uploads.js +21 -6
  20. package/dist/catalog/owned-items.d.ts +6 -13
  21. package/dist/catalog/owned-items.js +11 -29
  22. package/dist/catalog/owned-plugin-cleanup.d.ts +8 -17
  23. package/dist/catalog/owned-plugin-cleanup.js +12 -66
  24. package/dist/catalog/platform-components.d.ts +56 -34
  25. package/dist/catalog/platform-components.js +68 -78
  26. package/dist/catalog/plugin-import.d.ts +28 -25
  27. package/dist/catalog/plugin-import.js +90 -76
  28. package/dist/catalog/plugin-upgrade.d.ts +4 -4
  29. package/dist/catalog/plugin-upgrade.js +7 -19
  30. package/dist/catalog/scope-jobs.d.ts +59 -0
  31. package/dist/catalog/scope-jobs.js +65 -0
  32. package/dist/catalog/scope-sync-workers.d.ts +22 -0
  33. package/dist/catalog/scope-sync-workers.js +58 -2
  34. package/dist/catalog/scope-sync.d.ts +40 -65
  35. package/dist/catalog/scope-sync.js +114 -122
  36. package/dist/channels/inbound.js +2 -2
  37. package/dist/channels/threads.d.ts +4 -3
  38. package/dist/channels/threads.js +4 -3
  39. package/dist/conversations/conversations.controller.js +4 -2
  40. package/dist/conversations/conversations.service.d.ts +17 -5
  41. package/dist/conversations/conversations.service.js +17 -5
  42. package/dist/conversations/events.d.ts +21 -0
  43. package/dist/conversations/events.js +24 -1
  44. package/dist/conversations/source.d.ts +10 -0
  45. package/dist/conversations/source.js +17 -0
  46. package/dist/conversations/tree.d.ts +10 -0
  47. package/dist/conversations/tree.js +35 -0
  48. package/dist/db/kernel-schema.d.ts +9 -14
  49. package/dist/devices/devices.service.d.ts +11 -8
  50. package/dist/devices/devices.service.js +16 -23
  51. package/dist/evals/run-replay.d.ts +13 -2
  52. package/dist/evals/run-replay.js +28 -1
  53. package/dist/evals/snapshot.d.ts +7 -1
  54. package/dist/evals/snapshot.js +2 -1
  55. package/dist/extensions/hooks.d.ts +22 -3
  56. package/dist/extensions/hooks.js +15 -1
  57. package/dist/extensions/lifecycle-hooks.d.ts +99 -0
  58. package/dist/extensions/lifecycle-hooks.js +273 -0
  59. package/dist/extensions/org-config.d.ts +13 -1
  60. package/dist/extensions/org-config.js +19 -2
  61. package/dist/extensions/permission-view.d.ts +9 -5
  62. package/dist/extensions/permission-view.js +9 -4
  63. package/dist/extensions/point-defaults.d.ts +28 -1
  64. package/dist/extensions/point-defaults.js +44 -13
  65. package/dist/inbox/inbox.service.d.ts +6 -20
  66. package/dist/inbox/inbox.service.js +0 -3
  67. package/dist/inbox/org-notices.d.ts +1 -1
  68. package/dist/inbox/platform-inbox.service.d.ts +3 -32
  69. package/dist/inbox/platform-inbox.service.js +5 -54
  70. package/dist/infra/outbox/registry.d.ts +124 -0
  71. package/dist/infra/outbox/registry.js +126 -0
  72. package/dist/infra/queue/queue-names.d.ts +1 -1
  73. package/dist/infra/queue/queue-names.js +1 -0
  74. package/dist/orgs/directory-sync.js +1 -1
  75. package/dist/orgs/members.service.js +1 -1
  76. package/dist/orgs/offboarding.d.ts +50 -57
  77. package/dist/orgs/offboarding.js +117 -162
  78. package/dist/orgs/orgs.module.js +3 -0
  79. package/dist/orgs/orgs.service.d.ts +11 -5
  80. package/dist/orgs/orgs.service.js +15 -6
  81. package/dist/overrides.d.ts +3 -0
  82. package/dist/overrides.js +2 -0
  83. package/dist/platform/ops/inbox.ops.js +4 -20
  84. package/dist/platform/ops/platform.ops.d.ts +1 -0
  85. package/dist/platform/ops/platform.ops.js +6 -3
  86. package/dist/roles/worker-role.d.ts +10 -5
  87. package/dist/roles/worker-role.js +19 -9
  88. package/dist/roles/worker.module.js +2 -4
  89. package/dist/runtime/context/build.d.ts +3 -0
  90. package/dist/runtime/context/build.js +15 -0
  91. package/dist/runtime/context/plugin-context.d.ts +20 -0
  92. package/dist/runtime/context/plugin-context.js +30 -0
  93. package/dist/runtime/langgraph-loop.js +4 -0
  94. package/dist/runtime/loop.d.ts +8 -0
  95. package/dist/runtime/maintenance.d.ts +5 -3
  96. package/dist/runtime/maintenance.js +36 -11
  97. package/dist/runtime/model-call.d.ts +6 -1
  98. package/dist/runtime/model-call.js +1 -0
  99. package/dist/runtime/plugin-context.service.d.ts +69 -0
  100. package/dist/runtime/plugin-context.service.js +208 -0
  101. package/dist/runtime/run-control.d.ts +6 -1
  102. package/dist/runtime/run-control.js +13 -0
  103. package/dist/runtime/run-queue.d.ts +13 -0
  104. package/dist/runtime/run-queue.js +63 -0
  105. package/dist/runtime/run-worker.d.ts +22 -1
  106. package/dist/runtime/run-worker.js +107 -10
  107. package/dist/runtime/runtime.module.js +2 -0
  108. package/dist/runtime/subagents/dispatch.js +1 -0
  109. package/dist/runtime/subagents/skill-fork.js +3 -0
  110. package/dist/sandbox/jobs.d.ts +20 -10
  111. package/dist/sandbox/jobs.js +23 -17
  112. package/dist/schedules/fire.d.ts +25 -4
  113. package/dist/schedules/fire.js +87 -13
  114. package/dist/schedules/job-ids.d.ts +66 -17
  115. package/dist/schedules/job-ids.js +81 -13
  116. package/dist/schedules/run-tracker.js +2 -1
  117. package/dist/schedules/scan.job.d.ts +13 -32
  118. package/dist/schedules/scan.job.js +35 -144
  119. package/dist/schedules/scheduled-tasks.service.d.ts +6 -6
  120. package/dist/schedules/scheduled-tasks.service.js +17 -14
  121. package/dist/schedules/schedules.module.js +0 -4
  122. package/dist/skills/changes.controller.d.ts +1 -1
  123. package/dist/skills/changes.controller.js +1 -1
  124. package/dist/tools/approvals.service.d.ts +15 -4
  125. package/dist/tools/approvals.service.js +26 -10
  126. package/dist/tools/pipeline.d.ts +11 -2
  127. package/dist/tools/pipeline.js +5 -0
  128. package/dist/tools/questions.service.d.ts +15 -4
  129. package/dist/tools/questions.service.js +21 -19
  130. package/dist/tools/registry.d.ts +8 -1
  131. package/dist/tools/registry.js +9 -0
  132. package/package.json +3 -3
  133. package/src/migrations/0121_drop_platform_migration_runs.sql +12 -0
  134. package/src/migrations/0122_drop_scheduled_task_runs_rechecked_at.sql +5 -0
  135. package/src/migrations/0123_pgvector.sql +27 -0
  136. package/src/migrations/0124_conversation_source.sql +36 -0
  137. package/dist/catalog/platform-migration.d.ts +0 -115
  138. package/dist/catalog/platform-migration.js +0 -583
  139. package/dist/infra/queue/add-job-once.d.ts +0 -14
  140. package/dist/infra/queue/add-job-once.js +0 -16
@@ -25,11 +25,13 @@ import { KernelEvents } from "../infra/events/kernel-events.js";
25
25
  import { KERNEL_BUS, KERNEL_LOGGER } from "../infra/tokens.js";
26
26
  import { SkillsCatalogService } from "../skills/catalog.js";
27
27
  import { normalizeStoredDefinitionIn } from "../tools/legacy-data.js";
28
+ import { ToolRegistry } from "../tools/registry.js";
28
29
  import { RunAbort } from "./abort.js";
29
30
  import { KyselyCheckpointSaver } from "./checkpointer.js";
30
31
  import { ConversationLock } from "./conversation-lock.js";
31
32
  import { runDeadline, tightestLimit } from "./limits.js";
32
33
  import { RunError } from "./model-call.js";
34
+ import { PluginContextService } from "./plugin-context.service.js";
33
35
  import { ProviderListError, ProviderRegistry } from "./providers.js";
34
36
  import { ORG_QUOTA_LEASE_MS, orgRunLimit, RUN_QUOTA } from "./quota.js";
35
37
  import { RunContextService } from "./run-context.service.js";
@@ -64,6 +66,15 @@ async function requeue(job, token, delayMs) {
64
66
  await job.moveToDelayed(Date.now() + delayMs, token);
65
67
  throw new DelayedError();
66
68
  }
69
+ /** An account's display name ("" when it is gone). */
70
+ async function accountName(db, accountId) {
71
+ const row = await db
72
+ .selectFrom("accounts")
73
+ .select("display_name")
74
+ .where("id", "=", accountId)
75
+ .executeTakeFirst();
76
+ return row?.display_name ?? "";
77
+ }
67
78
  /**
68
79
  * 运行 worker(只在含 `worker` 角色的进程消费):领取 `cap-kernel-runs` 与 `cap-kernel-subagent-runs` 的任务
69
80
  * (两个 `@Processor`,见 run-processors.ts),取会话锁与组织配额,调用 AgentLoop,按结果写终态事件并推进
@@ -87,6 +98,8 @@ let RunWorker = class RunWorker {
87
98
  models;
88
99
  skillsCatalog;
89
100
  runContext;
101
+ pluginContext;
102
+ tools;
90
103
  finisher;
91
104
  txHost;
92
105
  drafts;
@@ -114,7 +127,11 @@ let RunWorker = class RunWorker {
114
127
  /** Run start's skills catalog snapshot (reads the pinned skills' bodies, phase B). */
115
128
  skillsCatalog,
116
129
  /** Run start's run context (run context spec §5): who the agent is, the day, the person. */
117
- runContext, finisher, txHost,
130
+ runContext,
131
+ /** Run start's plugin context (memory plugin spec §6.1): the `context@1` providers' text. */
132
+ pluginContext,
133
+ /** The run's resolved tools, for the plugin context's `tools` (chat runs with providers). */
134
+ tools, finisher, txHost,
118
135
  /** The steps' live drafts this process writes for the runs it executes. */
119
136
  drafts, hooks) {
120
137
  this.kdb = kdb;
@@ -134,6 +151,8 @@ let RunWorker = class RunWorker {
134
151
  this.models = models;
135
152
  this.skillsCatalog = skillsCatalog;
136
153
  this.runContext = runContext;
154
+ this.pluginContext = pluginContext;
155
+ this.tools = tools;
137
156
  this.finisher = finisher;
138
157
  this.txHost = txHost;
139
158
  this.drafts = drafts;
@@ -379,9 +398,10 @@ let RunWorker = class RunWorker {
379
398
  // 运行在等人处理(例如暂停后本任务被 stalled 机制重新投递):不执行,续跑由处理 / 超时发起的新任务负责。
380
399
  if (run.status === "waiting")
381
400
  return;
401
+ await this.hooks?.afterRunRead?.(runId, conversationId);
382
402
  const conv = await this.db
383
403
  .selectFrom("conversations")
384
- .select(["org_id", "agent_id", "kind", "subagent_depth"])
404
+ .select(["org_id", "agent_id", "kind", "subagent_depth", "source", "account_id"])
385
405
  .where("id", "=", conversationId)
386
406
  .executeTakeFirst();
387
407
  if (!conv)
@@ -448,8 +468,6 @@ let RunWorker = class RunWorker {
448
468
  delayMs = QUOTA_FULL_DELAY_MS;
449
469
  }
450
470
  else {
451
- // 运行开始执行(含续跑、stalled 后接手):通知旁路(渠道出站投递据此把输出投到 IM),失败不影响运行
452
- void this.events.emit("run.started", { conversationId, runId });
453
471
  const r = await this.execute(run, { id: conversationId, ...conv }, state).catch(async (err) => {
454
472
  if (state.abandoned)
455
473
  return "abandoned";
@@ -574,6 +592,14 @@ let RunWorker = class RunWorker {
574
592
  saved = await normalizeStoredDefinitionIn(this.db, conv.org_id, version.definition);
575
593
  agent = saved;
576
594
  }
595
+ // the trigger account's role, read once per run for the plugin context and plugin tool
596
+ // handlers (memory plugin spec §6.2); an account no longer in the org counts as a guest
597
+ const role = (await this.db
598
+ .selectFrom("memberships")
599
+ .select("role")
600
+ .where("org_id", "=", conv.org_id)
601
+ .where("account_id", "=", run.trigger_account_id)
602
+ .executeTakeFirst())?.role ?? "guest";
577
603
  let startedAt = run.started_at;
578
604
  // The model / effort switch in effect (UI refresh B): the latest `model_switched` before this
579
605
  // run's `run_started`, read under the conversation lock when the run starts, so a replay
@@ -585,9 +611,14 @@ let RunWorker = class RunWorker {
585
611
  }
586
612
  if (run.status === "queued" || startedAt === null) {
587
613
  if (!existingStart) {
614
+ await this.hooks?.beforeRunStarted?.(runId, conversationId);
588
615
  if (saved !== undefined)
589
616
  await this.fallBackIfUnavailable(conv.org_id, conversationId, runId, saved);
590
617
  const started = await this.appendRunStarted(conversationId, run, agent, saved);
618
+ // the status was read before the conversation lock: a stop that ended the run meanwhile
619
+ // (a queued run's direct abort, an orphan abort) wins, and the run never starts
620
+ if (started.terminal)
621
+ return "finished";
591
622
  agent = started.agent;
592
623
  const toSeq = started.toSeq;
593
624
  if (toSeq !== undefined)
@@ -617,14 +648,42 @@ let RunWorker = class RunWorker {
617
648
  },
618
649
  }),
619
650
  });
651
+ // plugin context (memory plugin spec §6.1): after the run context, still before the run is
652
+ // running; the providers are asked outside any transaction, a chat run only
653
+ await this.pluginContext.record({
654
+ orgId: conv.org_id,
655
+ conversationId,
656
+ runId,
657
+ agentId: conv.agent_id,
658
+ agentName: agent.name,
659
+ ownerAccountId: conv.account_id,
660
+ // only a chat run asks the providers
661
+ ownerName: conv.kind === "chat" ? await accountName(this.db, conv.account_id) : "",
662
+ triggerAccountId: run.trigger_account_id,
663
+ source: conv.source,
664
+ role,
665
+ kind: conv.kind,
666
+ toolKeysByPlugin: () => this.pluginToolKeys(conv, agent),
667
+ // the run's own abort watch starts with its execution below: until then, the shutdown
668
+ signal: state.shutdown.signal,
669
+ });
670
+ await this.hooks?.beforeRunning?.(runId, conversationId);
671
+ // never over a terminal status: a run finished meanwhile stays finished
620
672
  const updated = await this.db
621
673
  .updateTable("runs")
622
674
  .set({ status: "running", started_at: sql `COALESCE(started_at, now())` })
623
675
  .where("id", "=", runId)
676
+ .where("status", "in", ["queued", "running"])
624
677
  .returning("started_at")
625
- .executeTakeFirstOrThrow();
678
+ .executeTakeFirst();
679
+ if (updated === undefined)
680
+ return "finished";
626
681
  startedAt = updated.started_at;
627
682
  }
683
+ // the run executes (a first start, a resume, a stalled takeover): a hint to the side paths (the
684
+ // channel outbound delivers its output to IM), after the start went through, never for a run a
685
+ // stop ended first; a failure does not affect the run
686
+ void this.events.emit("run.started", { conversationId, runId });
628
687
  // 截止时间以首次开始为准:停机后在其他 worker 续跑也不重新计时;等人处理的时长(waited_ms)不计入。
629
688
  // `runtime.maxRunSeconds` 0: no deadline, no timer (the run ends when the model stops or is stopped)
630
689
  const deadline = runDeadline(startedAt.getTime() + Number(run.waited_ms), runtime.maxRunSeconds);
@@ -688,6 +747,9 @@ let RunWorker = class RunWorker {
688
747
  runId,
689
748
  agentId: conv.agent_id,
690
749
  triggerAccountId: run.trigger_account_id,
750
+ source: conv.source,
751
+ ownerAccountId: conv.account_id,
752
+ role,
691
753
  agent,
692
754
  agentVersion: run.agent_version,
693
755
  signal: ac.signal,
@@ -788,6 +850,28 @@ let RunWorker = class RunWorker {
788
850
  // a completed run enqueues the conversation's title job (never throws)
789
851
  void this.titles.maybeEnqueue(conversationId);
790
852
  }
853
+ /**
854
+ * The run's extension tool keys by plugin, resolved as a chat run's first step lists them
855
+ * (disabled tools, org switches, the conversation's source): a provider's `tools`.
856
+ */
857
+ async pluginToolKeys(conv, agent) {
858
+ const { tools } = await this.tools.forAgent(conv.org_id, {
859
+ disabledTools: agent.disabledTools,
860
+ subagentTools: agent.subagentTools,
861
+ subagents: agent.subagents,
862
+ sandboxConversationId: conv.id,
863
+ source: conv.source,
864
+ });
865
+ const out = new Map();
866
+ for (const t of tools) {
867
+ if (t.source.kind !== "plugin")
868
+ continue;
869
+ const keys = out.get(t.source.plugin) ?? [];
870
+ keys.push(t.key);
871
+ out.set(t.source.plugin, keys);
872
+ }
873
+ return out;
874
+ }
791
875
  /**
792
876
  * When the conversation was switched to a model that no longer resolves (provider disabled,
793
877
  * model removed), writes `model_switched { model: null, effort: null, reason: "unavailable" }`
@@ -834,6 +918,17 @@ let RunWorker = class RunWorker {
834
918
  .where("id", "=", conversationId)
835
919
  .forUpdate()
836
920
  .execute();
921
+ // the run's status again under the locks (conversation, then run, as `RunFinish.finishIn`):
922
+ // the job read it before taking the conversation lock, and a stop may have ended it since
923
+ const row = await t
924
+ .selectFrom("runs")
925
+ .select("status")
926
+ .where("id", "=", run.id)
927
+ .forUpdate()
928
+ .executeTakeFirst();
929
+ if (!row || TERMINAL_STATUSES.has(row.status)) {
930
+ return { toSeq: undefined, agent: current, terminal: true };
931
+ }
837
932
  const agent = saved !== undefined
838
933
  ? effectiveDefinition(saved, await readSelection(t, conversationId))
839
934
  : current;
@@ -951,10 +1046,12 @@ RunWorker = __decorate([
951
1046
  __param(16, Inject(ProviderRegistry)),
952
1047
  __param(17, Inject(SkillsCatalogService)),
953
1048
  __param(18, Inject(RunContextService)),
954
- __param(19, Inject(RunFinish)),
955
- __param(20, Inject(TransactionHost)),
956
- __param(21, Inject(RunDrafts)),
957
- __param(22, Optional()),
958
- __param(22, Inject(RUN_HOOKS))
1049
+ __param(19, Inject(PluginContextService)),
1050
+ __param(20, Inject(ToolRegistry)),
1051
+ __param(21, Inject(RunFinish)),
1052
+ __param(22, Inject(TransactionHost)),
1053
+ __param(23, Inject(RunDrafts)),
1054
+ __param(24, Optional()),
1055
+ __param(24, Inject(RUN_HOOKS))
959
1056
  ], RunWorker);
960
1057
  export { RunWorker };
@@ -23,6 +23,7 @@ import { RunAbort } from "./abort.js";
23
23
  import { KyselyCheckpointSaver } from "./checkpointer.js";
24
24
  import { ConversationLock } from "./conversation-lock.js";
25
25
  import { MAINTENANCE_QUEUE_NAME } from "./maintenance-queue.js";
26
+ import { PluginContextService } from "./plugin-context.service.js";
26
27
  import { ProvidersModule } from "./providers.js";
27
28
  import { EVAL_QUOTA, EVAL_QUOTA_PREFIX, OrgQuota, RUN_QUOTA, RUN_QUOTA_PREFIX } from "./quota.js";
28
29
  import { RunContextService } from "./run-context.service.js";
@@ -83,6 +84,7 @@ RuntimeModule = __decorate([
83
84
  AgentLoops,
84
85
  RunSteps,
85
86
  RunContextService,
87
+ PluginContextService,
86
88
  RunWorker,
87
89
  // a finished background task wakes an idle conversation (background Bash spec §3)
88
90
  BackgroundWake,
@@ -121,6 +121,7 @@ export async function dispatchSubagent(d, a) {
121
121
  parent_conversation_id: ctx.conversationId,
122
122
  subagent_type: entry.type,
123
123
  subagent_depth: childDepth,
124
+ source: "subagent",
124
125
  })
125
126
  .execute();
126
127
  const [, task] = await d.eventLog.appendIn(childId, [
@@ -125,6 +125,9 @@ export async function advanceSkillForks(d, a) {
125
125
  agentVersion: ctx.agentVersion,
126
126
  agent: ctx.agent,
127
127
  accountId: ctx.triggerAccountId,
128
+ source: ctx.source,
129
+ ownerAccountId: ctx.ownerAccountId,
130
+ role: ctx.role,
128
131
  signal: ctx.signal,
129
132
  ...(ctx.limits.deadline !== undefined && { deadline: ctx.limits.deadline }),
130
133
  ...(ctx.conversationKind !== undefined &&
@@ -1,5 +1,6 @@
1
1
  import type { Job, Queue } from "bullmq";
2
2
  import { LockService } from "../infra/lock/lock.service.js";
3
+ import type { OutboxService } from "../infra/outbox/outbox.service.js";
3
4
  import type { KernelLogger } from "../log.js";
4
5
  import { SandboxService } from "./sandbox.service.js";
5
6
  /** The idle suspension sweep (every minute, once per tick across replicas). */
@@ -7,10 +8,25 @@ export declare const SANDBOX_SUSPEND_JOB = "sandbox-suspend-idle";
7
8
  /** The inactivity destruction sweep (hourly, once per tick across replicas). */
8
9
  export declare const SANDBOX_DESTROY_JOB = "sandbox-destroy-inactive";
9
10
  /**
10
- * One-off job: destroy a batch of devices now, instance and file system (device
11
- * deletion, offboarding; data `{ deviceIds }`).
11
+ * One-off job: destroy devices now, instance and file system (data `{ deviceIds }`). Owed through
12
+ * the outbox by the transaction that soft-deletes the device (a device's deletion, offboarding),
13
+ * one device per job under {@link deviceDestroyJobId} ({@link oweDeviceDestroy}).
12
14
  */
13
15
  export declare const SANDBOX_DESTROY_FOR_JOB = "sandbox-destroy-for";
16
+ /** The data of a {@link SANDBOX_DESTROY_FOR_JOB} job. */
17
+ export interface SandboxDestroyForData {
18
+ deviceIds: string[];
19
+ }
20
+ /**
21
+ * The deterministic id of a deleted device's destroy job (no `:`): a device is deleted once, so
22
+ * its destruction is owed once.
23
+ */
24
+ export declare const deviceDestroyJobId: (deviceId: string) => string;
25
+ /**
26
+ * Owes the destruction of a device in the caller's transaction, the one that soft-deleted it
27
+ * (transactional outbox spec §8.1; throws outside a transaction).
28
+ */
29
+ export declare function oweDeviceDestroy(outbox: Pick<OutboxService, "add">, deviceId: string, orgId: string): Promise<boolean>;
14
30
  /**
15
31
  * One-off job: start a device over with empty files (`resetFileSystem`; data
16
32
  * `{ deviceId }`, job id {@link resetFilesJobId}).
@@ -22,18 +38,12 @@ export declare const SANDBOX_RESET_FILES_JOB = "sandbox-reset-files";
22
38
  */
23
39
  export declare const resetFilesJobId: (deviceId: string) => string;
24
40
  /**
25
- * The producer of the sandbox maintenance jobs: {@link SANDBOX_DESTROY_FOR_JOB} (BullMQ's own job
26
- * ids, as before: every enqueue is a new job, also after a batch failed all its attempts) and
27
- * {@link SANDBOX_RESET_FILES_JOB}; `MaintenanceProcessor` consumes them.
41
+ * The producer of {@link SANDBOX_RESET_FILES_JOB} (destroys are owed through the outbox,
42
+ * {@link oweDeviceDestroy}); `MaintenanceProcessor` consumes both.
28
43
  */
29
44
  export declare class SandboxJobs {
30
45
  private readonly queue;
31
46
  constructor(queue: Pick<Queue, "add">);
32
- /**
33
- * Enqueues the destruction of these devices: a failed batch is retried whole (destroying
34
- * one already destroyed does nothing).
35
- */
36
- destroy(deviceIds: string[]): Promise<void>;
37
47
  /**
38
48
  * Enqueues the reset of a device's files (see {@link resetFilesJobId}): retried on failure
39
49
  * (the exclusive lease not obtained in time, the provider failing); a job that failed all its
@@ -19,10 +19,29 @@ export const SANDBOX_SUSPEND_JOB = "sandbox-suspend-idle";
19
19
  /** The inactivity destruction sweep (hourly, once per tick across replicas). */
20
20
  export const SANDBOX_DESTROY_JOB = "sandbox-destroy-inactive";
21
21
  /**
22
- * One-off job: destroy a batch of devices now, instance and file system (device
23
- * deletion, offboarding; data `{ deviceIds }`).
22
+ * One-off job: destroy devices now, instance and file system (data `{ deviceIds }`). Owed through
23
+ * the outbox by the transaction that soft-deletes the device (a device's deletion, offboarding),
24
+ * one device per job under {@link deviceDestroyJobId} ({@link oweDeviceDestroy}).
24
25
  */
25
26
  export const SANDBOX_DESTROY_FOR_JOB = "sandbox-destroy-for";
27
+ /**
28
+ * The deterministic id of a deleted device's destroy job (no `:`): a device is deleted once, so
29
+ * its destruction is owed once.
30
+ */
31
+ export const deviceDestroyJobId = (deviceId) => `device-destroy-${deviceId}`;
32
+ /**
33
+ * Owes the destruction of a device in the caller's transaction, the one that soft-deleted it
34
+ * (transactional outbox spec §8.1; throws outside a transaction).
35
+ */
36
+ export function oweDeviceDestroy(outbox, deviceId, orgId) {
37
+ return outbox.add({
38
+ queue: MAINTENANCE_QUEUE_NAME,
39
+ name: SANDBOX_DESTROY_FOR_JOB,
40
+ jobId: deviceDestroyJobId(deviceId),
41
+ data: { deviceIds: [deviceId] },
42
+ orgId,
43
+ });
44
+ }
26
45
  /**
27
46
  * One-off job: start a device over with empty files (`resetFileSystem`; data
28
47
  * `{ deviceId }`, job id {@link resetFilesJobId}).
@@ -34,27 +53,14 @@ export const SANDBOX_RESET_FILES_JOB = "sandbox-reset-files";
34
53
  */
35
54
  export const resetFilesJobId = (deviceId) => `${SANDBOX_RESET_FILES_JOB}-${deviceId}`;
36
55
  /**
37
- * The producer of the sandbox maintenance jobs: {@link SANDBOX_DESTROY_FOR_JOB} (BullMQ's own job
38
- * ids, as before: every enqueue is a new job, also after a batch failed all its attempts) and
39
- * {@link SANDBOX_RESET_FILES_JOB}; `MaintenanceProcessor` consumes them.
56
+ * The producer of {@link SANDBOX_RESET_FILES_JOB} (destroys are owed through the outbox,
57
+ * {@link oweDeviceDestroy}); `MaintenanceProcessor` consumes both.
40
58
  */
41
59
  let SandboxJobs = class SandboxJobs {
42
60
  queue;
43
61
  constructor(queue) {
44
62
  this.queue = queue;
45
63
  }
46
- /**
47
- * Enqueues the destruction of these devices: a failed batch is retried whole (destroying
48
- * one already destroyed does nothing).
49
- */
50
- async destroy(deviceIds) {
51
- await this.queue.add(SANDBOX_DESTROY_FOR_JOB, { deviceIds }, {
52
- attempts: 5,
53
- backoff: { type: "exponential", delay: 30_000 },
54
- removeOnComplete: true,
55
- removeOnFail: 100,
56
- });
57
- }
58
64
  /**
59
65
  * Enqueues the reset of a device's files (see {@link resetFilesJobId}): retried on failure
60
66
  * (the exclusive lease not obtained in time, the provider failing); a job that failed all its
@@ -7,8 +7,10 @@ import { KernelConfigService } from "../infra/config/kernel-config.service.js";
7
7
  import { KernelDb } from "../infra/database/kernel-db.js";
8
8
  import { TransactionHost } from "../infra/database/transaction-host.js";
9
9
  import { LockService } from "../infra/lock/lock.service.js";
10
+ import { OutboxService } from "../infra/outbox/outbox.service.js";
10
11
  import type { KernelLogger } from "../log.js";
11
12
  import { OrgLock } from "../orgs/org-lock.js";
13
+ import { type SchedSettleJobData } from "./job-ids.js";
12
14
  import { ScheduledTaskRunTracker } from "./run-tracker.js";
13
15
  import { ScheduledTaskStore } from "./store.js";
14
16
  /** A task run's message client id in a conversation's queue (never read back as "the task's"). */
@@ -21,10 +23,12 @@ export declare function localTimeOf(at: Date, timeZone: string): string;
21
23
  *
22
24
  * 1. The claim, one transaction under the org row lock (which serializes the org's rate limit
23
25
  * count), then the task run and the task rows: the checks in order (task paused since the
24
- * scan, unless a person's "run now"; org disabled, creator
25
- * inactive or not a member, agent gone or not visible, conversation gone, connection inactive,
26
+ * scan, unless a person's "run now"; a tick fired more than 10 minutes past the org's catch-up
27
+ * window, `missed`, unless a "run now"; org disabled, creator inactive or not a member, agent
28
+ * gone or not visible, conversation gone, connection inactive,
26
29
  * overlap, rate limit, device offline, queue paused), each a `skipped` run with its reason, the
27
- * lasting ones also pausing the task; then `fired_at` is written, after the rate limit count.
30
+ * lasting ones also pausing the task; then `fired_at` is written, after the rate limit count,
31
+ * with the run's delayed settle check owed (`sched-settle`, {@link settle}).
28
32
  * A run whose claim committed before (a retried job) goes straight to the start.
29
33
  * 2. The start, idempotent: `new` creates the conversation with its first message keyed by the
30
34
  * task run id (`create_request_id`); `conversation` queues the prompt in the target
@@ -51,12 +55,29 @@ export declare class ScheduledTaskFirer {
51
55
  /** Reads a local device's connection lease (online or not). */
52
56
  private readonly locks;
53
57
  private readonly log;
58
+ /** The claim owes the run's settle check (ruling R26). */
59
+ private readonly outbox;
54
60
  constructor(kdb: KernelDb, txHost: TransactionHost, config: Pick<KernelConfigService, "get">, store: ScheduledTaskStore, tracker: Pick<ScheduledTaskRunTracker, "inFlightIn" | "failStartIn">, orgLock: Pick<OrgLock, "tryLock">, agents: Pick<AgentLookup, "get">, access: Pick<ConversationAccess, "isAccessible">, conversations: Pick<ConversationsService, "createWithMessage" | "create">, queue: Pick<ConversationQueue, "add">, connections: Pick<ChannelConnections, "row" | "channel">,
55
61
  /** Reads a local device's connection lease (online or not). */
56
- locks: Pick<LockService, "holderOf">, log: Pick<KernelLogger, "warn" | "info">);
62
+ locks: Pick<LockService, "holderOf">, log: Pick<KernelLogger, "warn" | "info">,
63
+ /** The claim owes the run's settle check (ruling R26). */
64
+ outbox: Pick<OutboxService, "add">);
57
65
  private get db();
58
66
  /** Fires task run `taskRunId` (see the class); a run not `pending` is left as it is. */
59
67
  fire(taskRunId: string): Promise<void>;
68
+ /**
69
+ * The `sched-settle` job (ruling R26; `job-ids.ts`). The claimed check: a task run claimed
70
+ * (`fired_at`) and still `pending` goes through the run tracker's check, which links it to its
71
+ * run, settles it, or releases a run that left no trace (`skipped`, `removed`), so a `once`
72
+ * task's last run never stays pending; one still in flight (its item queued, its message posted)
73
+ * owes the next check an hour on. The unfired check: a run still `pending` that no claim took
74
+ * (its fire job failed for good before the claim) is skipped, `missed` for a tick, `removed` for
75
+ * a "run now", so the overlap check never waits on it. A run claimed, started, ended or skipped
76
+ * since is left as it is (the claimed check follows a claimed run).
77
+ */
78
+ settle(data: SchedSettleJobData): Promise<void>;
79
+ /** The unfired check (see {@link settle}), task row first (the one lock order). */
80
+ private settleUnfired;
60
81
  /** Step 1 (see the class): a skip, the plan, or null when there is nothing to fire. */
61
82
  private claim;
62
83
  /** Step 2 (see the class). */
@@ -16,19 +16,20 @@ import { ConversationAccess, ConversationAccessModule } from "../conversations/a
16
16
  import { ConversationsModule } from "../conversations/conversations.module.js";
17
17
  import { ConversationsService } from "../conversations/conversations.service.js";
18
18
  import { ConversationQueue } from "../conversations/queue.service.js";
19
+ import { channelSource } from "../conversations/source.js";
19
20
  import { ApiError } from "../errors.js";
20
21
  import { KernelConfigService } from "../infra/config/kernel-config.service.js";
21
22
  import { KernelDb } from "../infra/database/kernel-db.js";
22
23
  import { TransactionHost } from "../infra/database/transaction-host.js";
23
24
  import { Transactional } from "../infra/database/transactional.decorator.js";
24
25
  import { LockService } from "../infra/lock/lock.service.js";
25
- import { QueueModule } from "../infra/queue/queue.module.js";
26
+ import { OutboxService } from "../infra/outbox/outbox.service.js";
26
27
  import { KERNEL_LOGGER } from "../infra/tokens.js";
27
28
  import { deviceLeaseKey } from "../local-devices/keys.js";
28
29
  import { OrgLock, OrgLockModule } from "../orgs/org-lock.js";
29
- import { MAINTENANCE_QUEUE_NAME } from "../runtime/maintenance-queue.js";
30
30
  import { RuntimeModule } from "../runtime/runtime.module.js";
31
31
  import { readScheduleSettings } from "./effective-settings.js";
32
+ import { oweSchedSettle, SCHED_LATE_FIRE_SLACK_MS, SCHED_SETTLE_AFTER_MS, SCHED_SETTLE_LONG_CHECKS, SCHED_SETTLE_RECHECK_MS, } from "./job-ids.js";
32
33
  import { pauseTaskIn } from "./pause.js";
33
34
  import { lockTaskRow, ScheduledTaskRunTracker, ScheduledTaskTrackerModule } from "./run-tracker.js";
34
35
  import { ScheduledTaskScanJob } from "./scan.job.js";
@@ -69,10 +70,12 @@ const DEVICE_GONE = new Set(["device.not_found", "device.unavailable", "sandbox.
69
70
  *
70
71
  * 1. The claim, one transaction under the org row lock (which serializes the org's rate limit
71
72
  * count), then the task run and the task rows: the checks in order (task paused since the
72
- * scan, unless a person's "run now"; org disabled, creator
73
- * inactive or not a member, agent gone or not visible, conversation gone, connection inactive,
73
+ * scan, unless a person's "run now"; a tick fired more than 10 minutes past the org's catch-up
74
+ * window, `missed`, unless a "run now"; org disabled, creator inactive or not a member, agent
75
+ * gone or not visible, conversation gone, connection inactive,
74
76
  * overlap, rate limit, device offline, queue paused), each a `skipped` run with its reason, the
75
- * lasting ones also pausing the task; then `fired_at` is written, after the rate limit count.
77
+ * lasting ones also pausing the task; then `fired_at` is written, after the rate limit count,
78
+ * with the run's delayed settle check owed (`sched-settle`, {@link settle}).
76
79
  * A run whose claim committed before (a retried job) goes straight to the start.
77
80
  * 2. The start, idempotent: `new` creates the conversation with its first message keyed by the
78
81
  * task run id (`create_request_id`); `conversation` queues the prompt in the target
@@ -98,9 +101,12 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
98
101
  connections;
99
102
  locks;
100
103
  log;
104
+ outbox;
101
105
  constructor(kdb, txHost, config, store, tracker, orgLock, agents, access, conversations, queue, connections,
102
106
  /** Reads a local device's connection lease (online or not). */
103
- locks, log) {
107
+ locks, log,
108
+ /** The claim owes the run's settle check (ruling R26). */
109
+ outbox) {
104
110
  this.kdb = kdb;
105
111
  this.txHost = txHost;
106
112
  this.config = config;
@@ -114,6 +120,7 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
114
120
  this.connections = connections;
115
121
  this.locks = locks;
116
122
  this.log = log;
123
+ this.outbox = outbox;
117
124
  }
118
125
  get db() {
119
126
  return this.kdb.db;
@@ -137,6 +144,61 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
137
144
  return;
138
145
  await this.start(claimed.plan);
139
146
  }
147
+ /**
148
+ * The `sched-settle` job (ruling R26; `job-ids.ts`). The claimed check: a task run claimed
149
+ * (`fired_at`) and still `pending` goes through the run tracker's check, which links it to its
150
+ * run, settles it, or releases a run that left no trace (`skipped`, `removed`), so a `once`
151
+ * task's last run never stays pending; one still in flight (its item queued, its message posted)
152
+ * owes the next check an hour on. The unfired check: a run still `pending` that no claim took
153
+ * (its fire job failed for good before the claim) is skipped, `missed` for a tick, `removed` for
154
+ * a "run now", so the overlap check never waits on it. A run claimed, started, ended or skipped
155
+ * since is left as it is (the claimed check follows a claimed run).
156
+ */
157
+ async settle(data) {
158
+ const { taskRunId } = data;
159
+ if ("unfired" in data) {
160
+ await this.settleUnfired(taskRunId);
161
+ return;
162
+ }
163
+ const run = await this.store.updateRun(this.db, taskRunId, {});
164
+ if (run?.status !== "pending" || run.firedAt === null)
165
+ return;
166
+ // inFlightIn takes the task row before the run (the one lock order)
167
+ if (!(await this.tracker.inFlightIn(run)))
168
+ return;
169
+ const now = await this.store.updateRun(this.db, taskRunId, {});
170
+ if (now?.status !== "pending" || now.firedAt === null)
171
+ return;
172
+ const next = data.check + 1;
173
+ if (next === SCHED_SETTLE_LONG_CHECKS) {
174
+ this.log.warn("scheduled task run still in flight a week after its claim", {
175
+ taskId: run.taskId,
176
+ taskRunId,
177
+ });
178
+ }
179
+ await oweSchedSettle(this.outbox, taskRunId, run.orgId, next, SCHED_SETTLE_RECHECK_MS);
180
+ }
181
+ /** The unfired check (see {@link settle}), task row first (the one lock order). */
182
+ async settleUnfired(taskRunId) {
183
+ const t = this.db;
184
+ const owner = await t
185
+ .selectFrom("scheduled_task_runs")
186
+ .select("task_id")
187
+ .where("id", "=", taskRunId)
188
+ .executeTakeFirst();
189
+ if (!owner)
190
+ return;
191
+ await lockTaskRow(t, owner.task_id);
192
+ const run = await t
193
+ .selectFrom("scheduled_task_runs")
194
+ .select(["status", "fired_at", "manual"])
195
+ .where("id", "=", taskRunId)
196
+ .forUpdate()
197
+ .executeTakeFirst();
198
+ if (run?.status !== "pending" || run.fired_at !== null)
199
+ return;
200
+ await this.skipIn(t, taskRunId, owner.task_id, run.manual ? "removed" : "missed");
201
+ }
140
202
  /** Step 1 (see the class): a skip, the plan, or null when there is nothing to fire. */
141
203
  async claim(taskRunId, orgId, deviceOffline) {
142
204
  const t = this.db;
@@ -179,6 +241,12 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
179
241
  if (!resumed && !run.manual && task.status === "paused")
180
242
  return skip("task_paused");
181
243
  const settings = await readScheduleSettings(t, this.config, orgId);
244
+ // a tick whose fire comes this late (workers down for long) is missed, as the scan records a
245
+ // tick it finds outside the window; a person's "run now" still runs
246
+ const lateAfter = settings.catchUpWindowMinutes * 60_000 + SCHED_LATE_FIRE_SLACK_MS;
247
+ if (!resumed && !run.manual && Date.now() - run.scheduledFor.getTime() > lateAfter) {
248
+ return skip("missed");
249
+ }
182
250
  if (!resumed && !settings.enabled)
183
251
  return skip("org_disabled");
184
252
  if (!resumed && !(await this.creatorActive(orgId, task.accountId))) {
@@ -260,6 +328,8 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
260
328
  firedAt: new Date(),
261
329
  ...(plan.mode === "conversation" && { conversationId: plan.conversationId }),
262
330
  });
331
+ // a start that fails for good leaves the run claimed: its settle check releases or ends it
332
+ await oweSchedSettle(this.outbox, run.id, orgId, 0, SCHED_SETTLE_AFTER_MS);
263
333
  return { plan: { ...plan, run: claimed ?? run } };
264
334
  }
265
335
  /** Step 2 (see the class). */
@@ -282,7 +352,7 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
282
352
  ...(task.model !== null && { model: task.model }),
283
353
  ...(task.effort !== null && { effort: task.effort }),
284
354
  };
285
- const out = await this.conversations.createWithMessage(task.orgId, task.accountId, task.agentId, { text: task.prompt, clientMessageId: run.id, ...pick }, task.deviceId === null ? {} : { deviceId: task.deviceId }, internal);
355
+ const out = await this.conversations.createWithMessage(task.orgId, task.accountId, task.agentId, { text: task.prompt, clientMessageId: run.id, ...pick }, task.deviceId === null ? {} : { deviceId: task.deviceId }, { ...internal, source: "web" });
286
356
  // a retried start found the conversation made before: `createWithMessage` already asked
287
357
  // for its run (and the first start's message owes the ensure-run backstop)
288
358
  await this.setConversation(run.id, out.conversation.id);
@@ -345,7 +415,7 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
345
415
  const { conversationId, created } = await findOrBindThread(t, plan.thread, plan.target, async () => {
346
416
  const conv = await this.conversations.create(task.orgId, task.accountId, task.agentId,
347
417
  // a chat task takes no device (new mode only): an IM conversation's default
348
- { im: true });
418
+ { im: true }, { source: channelSource(plan.target.chatType) });
349
419
  await t
350
420
  .updateTable("conversations")
351
421
  .set({ scheduled_task_id: task.id })
@@ -382,15 +452,17 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
382
452
  return null;
383
453
  if (locked.conversation_id !== null)
384
454
  return locked.conversation_id;
455
+ const target = { chatId: plan.chatId, chatType: "group" };
385
456
  const conv = await this.conversations.create(task.orgId, task.accountId, task.agentId,
386
457
  // a chat task takes no device (new mode only): an IM conversation's default
387
- { im: true });
458
+ { im: true },
459
+ // stored now: the thread to derive it from binds only at the run's first post
460
+ { source: channelSource(target.chatType) });
388
461
  await t
389
462
  .updateTable("conversations")
390
463
  .set({ scheduled_task_id: task.id })
391
464
  .where("id", "=", conv.id)
392
465
  .execute();
393
- const target = { chatId: plan.chatId, chatType: "group" };
394
466
  await t
395
467
  .insertInto("channel_deliveries")
396
468
  .values({
@@ -558,6 +630,9 @@ let ScheduledTaskFirer = class ScheduledTaskFirer {
558
630
  return (await this.locks.holderOf(deviceLeaseKey(deviceId))) === null;
559
631
  }
560
632
  };
633
+ __decorate([
634
+ Transactional()
635
+ ], ScheduledTaskFirer.prototype, "settle", null);
561
636
  __decorate([
562
637
  Transactional()
563
638
  ], ScheduledTaskFirer.prototype, "claim", null);
@@ -590,7 +665,8 @@ ScheduledTaskFirer = __decorate([
590
665
  __param(9, Inject(ConversationQueue)),
591
666
  __param(10, Inject(ChannelConnections)),
592
667
  __param(11, Inject(LockService)),
593
- __param(12, Inject(KERNEL_LOGGER))
668
+ __param(12, Inject(KERNEL_LOGGER)),
669
+ __param(13, Inject(OutboxService))
594
670
  ], ScheduledTaskFirer);
595
671
  export { ScheduledTaskFirer };
596
672
  /**
@@ -610,8 +686,6 @@ ScheduledTaskFireModule = __decorate([
610
686
  OrgLockModule,
611
687
  ScheduledTaskStoreModule,
612
688
  ScheduledTaskTrackerModule,
613
- // the scanner enqueues the fire jobs
614
- QueueModule.registerQueue(MAINTENANCE_QUEUE_NAME),
615
689
  ],
616
690
  providers: [ScheduledTaskFirer, ScheduledTaskScanJob],
617
691
  exports: [ScheduledTaskFirer],