@desplega.ai/agent-swarm 2.12.0 → 2.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/acp-adapter-cn53011z.js +17 -0
- package/dist/{acp-targets-311gcyvw.js → acp-targets-86f0sh6g.js} +9 -8
- package/dist/{actions-c4chgxp6.js → actions-zv8vcxvr.js} +24 -23
- package/dist/{agent-fs-provision-0b9awm5n.js → agent-fs-provision-jqshb33y.js} +4 -4
- package/dist/{amp-adapter-r0veq6x5.js → amp-adapter-gt503ywc.js} +2 -2
- package/dist/{app-hn5fftf5.js → app-6t4k5rh1.js} +6 -4
- package/dist/{approval-notifications-3g6yc8nv.js → approval-notifications-fcfw063h.js} +7 -6
- package/dist/{assistant-cpjgpjvp.js → assistant-9dzybh12.js} +25 -24
- package/dist/{boot-args-schema-hce1av1h.js → boot-args-schema-xzfp2feg.js} +1 -1
- package/dist/{boot-reembed-d7xcym0j.js → boot-reembed-3jz0p2jk.js} +1 -1
- package/dist/{boot-reembed-bztbxc4z.js → boot-reembed-r88wngfg.js} +5 -2
- package/dist/{boot-scrub-sweep-mkmm16gk.js → boot-scrub-sweep-scrqhw48.js} +2 -2
- package/dist/{boot-seeds-arrkbe8t.js → boot-seeds-vb2dcrrb.js} +4 -4
- package/dist/{claude-adapter-ykpyr5p8.js → claude-adapter-h3mp9hgs.js} +7 -6
- package/dist/{claude-managed-adapter-ssg6z4bt.js → claude-managed-adapter-kpbyj2vb.js} +3 -3
- package/dist/{claude-managed-setup-trd59v7b.js → claude-managed-setup-0efr3wkg.js} +5 -5
- package/dist/{claude-sdk-session-yjcze0j1.js → claude-sdk-session-03dw669q.js} +7 -6
- package/dist/{cli-e3ahxq5y.js → cli-0b655smz.js} +4 -4
- package/dist/{cli-mcymtvwn.js → cli-0dg7ytxm.js} +89 -11
- package/dist/cli-0jkenasr.js +148 -0
- package/dist/{cli-c7j31hq9.js → cli-0pazkmd4.js} +3 -3
- package/dist/{cli-5tg94mdw.js → cli-117g15t4.js} +8 -11
- package/dist/{cli-r14ac1y0.js → cli-1dbv9mj9.js} +1 -1
- package/dist/{cli-q8tc22bj.js → cli-1v82d2w6.js} +3 -3
- package/dist/{cli-pe3yxedk.js → cli-2jk4s7z1.js} +2 -2
- package/dist/{cli-0pp1yjg2.js → cli-2skf1mdf.js} +8 -8
- package/dist/{cli-2tw284hj.js → cli-48nvk6sb.js} +1 -1
- package/dist/{cli-en6dmkxd.js → cli-49f5jyhh.js} +2 -2
- package/dist/{cli-2z2c4k6q.js → cli-50w82h6m.js} +21 -0
- package/dist/{cli-dd2k2mcd.js → cli-6a40p4kv.js} +1 -1
- package/dist/{cli-yegz2g06.js → cli-6h88mn9q.js} +3 -3
- package/dist/{cli-52nfye5j.js → cli-7dbhnsj9.js} +23 -15
- package/dist/{cli-04zc3f7b.js → cli-9fsagra4.js} +1 -1
- package/dist/{cli-nja15knq.js → cli-9m9kyw77.js} +3 -3
- package/dist/{cli-k5jx8jya.js → cli-ajjvqhf3.js} +1 -1
- package/dist/{cli-hbvgzjq4.js → cli-avqkr20b.js} +2 -2
- package/dist/{cli-yfpc6qy9.js → cli-avxh26pd.js} +46 -15
- package/dist/{cli-q2dywc8h.js → cli-b2jh3dyb.js} +4 -4
- package/dist/{cli-m5qtmrbc.js → cli-c8v2yvej.js} +2 -2
- package/dist/{cli-gfy6hbd9.js → cli-ct8sz3np.js} +17 -10
- package/dist/{cli-bz97ff9n.js → cli-dka73bak.js} +1 -1
- package/dist/{cli-dkvtfgn4.js → cli-dkejt425.js} +75 -16
- package/dist/{cli-sv2r3bw8.js → cli-dnh64e8e.js} +2 -2
- package/dist/{cli-kwr5qv5g.js → cli-eb279wn3.js} +2 -2
- package/dist/{cli-hhc47ack.js → cli-f1952kjy.js} +2 -2
- package/dist/{cli-5z0g0e8v.js → cli-f8rcvf2x.js} +49 -13
- package/dist/{cli-eq4kew31.js → cli-fhvk2nvp.js} +24 -11
- package/dist/{cli-xf4z5bdn.js → cli-fz9pcjsw.js} +2 -2
- package/dist/{cli-fxd1mwtr.js → cli-g6j6ac3h.js} +1 -1
- package/dist/{cli-ya3kf4gs.js → cli-gfcv679k.js} +1 -1
- package/dist/{cli-55tc8cj4.js → cli-hbq0kwxz.js} +4 -4
- package/dist/{cli-39qm0rtm.js → cli-hc6e4tac.js} +28 -149
- package/dist/{cli-j5gtytvg.js → cli-hgsxnjyg.js} +13 -10
- package/dist/{cli-h70a2mxy.js → cli-hpb9x72z.js} +48 -8
- package/dist/{cli-7yn42w6m.js → cli-j53hpbsm.js} +1 -1
- package/dist/{cli-12vek61e.js → cli-jj3c5nyb.js} +471 -119
- package/dist/{cli-fy4gb74g.js → cli-kde4ky90.js} +1 -1
- package/dist/{cli-9qe5y107.js → cli-kspxjftk.js} +1 -1
- package/dist/{cli-bp6k9acz.js → cli-mh7pdxe0.js} +16 -16
- package/dist/{cli-v9ya390t.js → cli-mz34pxbk.js} +8 -8
- package/dist/{cli-ma9ys8rt.js → cli-nc6a8b5b.js} +7 -4
- package/dist/{cli-1htmtqc2.js → cli-prf7kz78.js} +49 -49
- package/dist/{cli-01tvvtad.js → cli-qec9bynj.js} +3 -3
- package/dist/{cli-t75xngj1.js → cli-qhhb30e8.js} +7 -7
- package/dist/{cli-73q4g5nx.js → cli-qm3ntjqs.js} +12 -12
- package/dist/{cli-tm60vtax.js → cli-rfejbpe1.js} +1 -1
- package/dist/{cli-agjbj5x0.js → cli-sdh61g52.js} +37 -19
- package/dist/{cli-9y4tgrkf.js → cli-skd2c4ej.js} +2 -1
- package/dist/{cli-5x03v6en.js → cli-sq913bh1.js} +8 -8
- package/dist/{cli-b93d81yg.js → cli-t35hx078.js} +1 -1
- package/dist/{cli-r0mz92by.js → cli-tn91wc6e.js} +1 -1
- package/dist/{cli-wvjqaqcp.js → cli-tstj6e8e.js} +1113 -227
- package/dist/{cli-xm34qj8m.js → cli-w14j2z4k.js} +2 -2
- package/dist/{cli-eqq0hwvx.js → cli-wndegmns.js} +1 -1
- package/dist/{cli-0k1a0sde.js → cli-wrhbym01.js} +6 -6
- package/dist/{cli-fyaq4zdv.js → cli-wv20kbr3.js} +6 -6
- package/dist/{cli-mq2fbgva.js → cli-wxwv3t11.js} +2 -2
- package/dist/{cli-krbcr00p.js → cli-xfjn2kyq.js} +58 -4
- package/dist/{cli-5r95eqbh.js → cli-y260hgep.js} +1 -1
- package/dist/{cli-cjqyse9q.js → cli-y4a4458j.js} +3 -3
- package/dist/{cli-2amwhyv2.js → cli-y518qkjg.js} +5 -3
- package/dist/{cli-0vcqcwfn.js → cli-yfvhvw8m.js} +2 -0
- package/dist/cli-yn4kt85g.js +148 -0
- package/dist/{cli-n8f0fs3c.js → cli-ytc7fpza.js} +8 -8
- package/dist/{cli-e8n0hdzd.js → cli-z64t9pe6.js} +1 -1
- package/dist/{cli-72yms3ey.js → cli-zqmmy073.js} +198 -18
- package/dist/cli.js +18 -16
- package/dist/{codex-adapter-wc6e4wct.js → codex-adapter-f3yp0kaj.js} +5 -5
- package/dist/{codex-hook-aqtj17yr.js → codex-hook-262cekfp.js} +73 -2
- package/dist/{codex-session-runner-ppfgjd97.js → codex-session-runner-ns4mv70p.js} +5 -5
- package/dist/{commands-tsv2vx5b.js → commands-5bzf2tfw.js} +3 -3
- package/dist/{controller-ye4sdwsy.js → controller-5xt7kvfm.js} +2 -2
- package/dist/{controllers-connect-zn4dg69h.js → controllers-connect-0gg13mzp.js} +24 -23
- package/dist/core-0fqtbrxa.js +42 -0
- package/dist/{cursor-adapter-q2g9vwts.js → cursor-adapter-p0c5zv6r.js} +1 -1
- package/dist/{db-g111qn9j.js → db-2s68ycq7.js} +3 -3
- package/dist/{db-3r1fyr2n.js → db-q7q639k5.js} +4 -2
- package/dist/{dispatcher-4q8s0mx8.js → dispatcher-j2y1q7ae.js} +24 -21
- package/dist/{dsh-adapter-mmcw1z5s.js → dsh-adapter-pgnkf4st.js} +1 -1
- package/dist/{easter-egg-3d-04ke6hcp.js → easter-egg-3d-0kea97jf.js} +1 -1
- package/dist/{embedding-space-kcdn52vz.js → embedding-space-3n6mhhmy.js} +6 -6
- package/dist/{embeddings-jnzcd91a.js → embeddings-jfmj6yme.js} +1 -1
- package/dist/{execute-74amghty.js → execute-tjrej979.js} +1 -1
- package/dist/{external-channel-v9zgxnxz.js → external-channel-jy8w2gg3.js} +2 -2
- package/dist/fx-adapter-k6w3jvz9.js +20 -0
- package/dist/grok-adapter-4q5tssw0.js +20 -0
- package/dist/handlers-qxgfvvsf.js +41 -0
- package/dist/{heartbeat-p8pk1msj.js → heartbeat-6tr2w2da.js} +25 -24
- package/dist/{hook-p5reemds.js → hook-bpnncsec.js} +76 -17
- package/dist/{hook-k1h73zd7.js → hook-fv1jwrv7.js} +6 -5
- package/dist/{http-ms1z5xxy.js → http-bpa26czn.js} +785 -181
- package/dist/index-0mcxjdsa.js +44 -0
- package/dist/{index-kn63q1gr.js → index-52tgbhrw.js} +29 -28
- package/dist/index-dz4q23ag.js +40 -0
- package/dist/{index-76zzxtzf.js → index-mpj0z9e0.js} +1 -1
- package/dist/index-s175jckx.js +39 -0
- package/dist/{index-gjn6qfgn.js → index-s8re7mvq.js} +1 -1
- package/dist/{index-fbkqhvvj.js → index-w216g9y9.js} +198 -160
- package/dist/{keepalive-031js8jz.js → keepalive-d6z12s04.js} +3 -3
- package/dist/lead-p3hx7h35.js +79 -0
- package/dist/{maintenance-sq9v3qx9.js → maintenance-7kac6g2g.js} +1 -1
- package/dist/{oauth-refresh-sweep-zqmnyvs8.js → oauth-refresh-sweep-t1xjetbh.js} +2 -2
- package/dist/{object-webhooks-r24wwb5y.js → object-webhooks-6rdagg0r.js} +7 -7
- package/dist/{onboard-p25yhh7t.js → onboard-32y1cfxy.js} +3 -3
- package/dist/{opencode-adapter-wcjc4zdk.js → opencode-adapter-enp3v31n.js} +7 -6
- package/dist/{otel-impl-ff1gf20m.js → otel-impl-k0tm5ptj.js} +1 -1
- package/dist/{pending-hxj5yatb.js → pending-24zebgvs.js} +23 -22
- package/dist/{pi-mono-adapter-vkp4zz7p.js → pi-mono-adapter-yj72xxxe.js} +85 -16
- package/dist/{pricing-refresh-9w7gx41w.js → pricing-refresh-cf25x38j.js} +2 -2
- package/dist/{rbac-roles-84aw10nz.js → rbac-roles-0wydd22m.js} +3 -3
- package/dist/{rbac-roles-hz74vckp.js → rbac-roles-at6crb3r.js} +7 -4
- package/dist/{registry-eeyx1201.js → registry-zv0zvxsx.js} +1 -1
- package/dist/{render-v2-nrdkg9gf.js → render-v2-nvcw0jd7.js} +8 -4
- package/dist/{resume-s6ey36rh.js → resume-y4w0k5v3.js} +9 -8
- package/dist/{runtime-x0b4pyam.js → runtime-vpe43hx0.js} +1 -1
- package/dist/{session-access-xv49tk14.js → session-access-gkyvnfye.js} +5 -4
- package/dist/{session-source-backfill-r5j988ah.js → session-source-backfill-3w2st4m2.js} +2 -2
- package/dist/{setup-mj0krk1p.js → setup-axxvqtys.js} +1 -1
- package/dist/thread-buffer-rwa7d4dx.js +42 -0
- package/dist/{tree-visibility-cbrsfaar.js → tree-visibility-y4fqvab5.js} +2 -2
- package/dist/{user-merge-jobs-f9d8nwnw.js → user-merge-jobs-kcmwvrc6.js} +3 -3
- package/dist/{virtual-modules-ns0ryw05.js → virtual-modules-7d92dct1.js} +3 -3
- package/dist/{work-objects-pydhzast.js → work-objects-f708e069.js} +27 -25
- package/dist/worker-8rg9gz74.js +79 -0
- package/dist/{worker-follow-up-9z2xec5e.js → worker-follow-up-0by8wztc.js} +22 -21
- package/dist/xlsx-9acfmxr1.js +31534 -0
- package/openapi.json +1385 -204
- package/package.json +5 -3
- package/src/agentmail/handlers.ts +26 -19
- package/src/apps/catalog.generated.json +31 -0
- package/src/apps/row-store.ts +37 -14
- package/src/azure-devops/handlers.ts +2 -0
- package/src/be/audit-user.ts +46 -6
- package/src/be/db/kv-windows.ts +22 -0
- package/src/be/db/session-moves.ts +3 -3
- package/src/be/db/session-tree.ts +52 -5
- package/src/be/db/skill-audit.ts +17 -0
- package/src/be/db/tasks/write.ts +18 -3
- package/src/be/db.ts +114 -23
- package/src/be/extensions/assets.ts +11 -7
- package/src/be/memory/demotion-evidence.ts +2 -2
- package/src/be/memory/providers/sqlite-store.ts +151 -18
- package/src/be/memory/search-read-worker.ts +60 -0
- package/src/be/memory/search-reader.ts +223 -0
- package/src/be/migrations/337_favorites_session_hidden_key.sql +39 -0
- package/src/be/migrations/338_audit_agent_columns.sql +30 -0
- package/src/be/migrations/339_deferred_event_waits.sql +26 -0
- package/src/be/model-tier-resolution.ts +18 -1
- package/src/be/multi-runtime.ts +17 -0
- package/src/be/primitives/object-embed.ts +250 -0
- package/src/be/primitives/objects.ts +29 -9
- package/src/be/primitives/sheets/service.ts +138 -6
- package/src/be/primitives/sheets/xlsx.ts +367 -0
- package/src/be/rbac-roles.ts +3 -0
- package/src/be/repo-checked-update.ts +4 -4
- package/src/be/repo-merge-policy.ts +26 -9
- package/src/be/scripts/boot-reembed.ts +6 -0
- package/src/be/scripts/db.ts +17 -5
- package/src/be/scripts/embeddings.ts +6 -0
- package/src/be/scripts/typecheck.ts +43 -2
- package/src/be/seed/schedules-seeder.ts +9 -1
- package/src/be/seed/workflows-seeder.ts +9 -1
- package/src/be/seed-scripts/index.ts +2 -0
- package/src/be/seed-skills/index.ts +12 -0
- package/src/be/swarm-config-guard.ts +1 -0
- package/src/be/unmapped-identities.ts +95 -1
- package/src/bitbucket/handlers.ts +2 -0
- package/src/commands/onboard/agent-fs-version.ts +1 -1
- package/src/event-waits/github.ts +43 -0
- package/src/event-waits/index.ts +29 -0
- package/src/event-waits/types.ts +51 -0
- package/src/extensions/contract-types.generated.ts +1 -1
- package/src/extensions/dispatcher.ts +9 -0
- package/src/fs/agent-fs-provider.ts +4 -1
- package/src/github/handlers.ts +3 -1
- package/src/github/pull-request-state.ts +34 -0
- package/src/gitlab/handlers.ts +12 -7
- package/src/hooks/codex-hook.ts +99 -1
- package/src/hooks/hook.ts +85 -11
- package/src/hooks/native-tool-hooks.ts +245 -0
- package/src/http/all-routes.ts +1 -0
- package/src/http/assets.ts +4 -3
- package/src/http/comb.ts +2 -2
- package/src/http/favorite-owner.ts +2 -2
- package/src/http/favorites.ts +3 -2
- package/src/http/fs.ts +13 -4
- package/src/http/index.ts +2 -0
- package/src/http/mcp-bridge.ts +4 -0
- package/src/http/objects.ts +265 -0
- package/src/http/pages.ts +3 -3
- package/src/http/poll.ts +43 -12
- package/src/http/repos.ts +7 -6
- package/src/http/schedules.ts +10 -6
- package/src/http/script-connections.ts +2 -2
- package/src/http/scripts.ts +9 -0
- package/src/http/sessions.ts +30 -12
- package/src/http/skills.ts +17 -0
- package/src/http/task-requester.ts +2 -2
- package/src/http/tasks.ts +13 -3
- package/src/http/tool-hooks.ts +24 -2
- package/src/http/trackers/authorize-gate.ts +2 -2
- package/src/http/users.ts +136 -1
- package/src/http/workflow-events.ts +13 -0
- package/src/http/workflows.ts +21 -11
- package/src/linear/sync.ts +3 -1
- package/src/prompts/base-prompt.ts +35 -13
- package/src/prompts/session-templates.ts +25 -0
- package/src/providers/acp-targets.ts +1 -1
- package/src/providers/amp-adapter.ts +1 -1
- package/src/providers/dsh-adapter.ts +1 -1
- package/src/providers/opencode-adapter.ts +1 -0
- package/src/providers/opencode-swarm-plugin.ts +50 -1
- package/src/providers/pi-mono-adapter.ts +139 -9
- package/src/providers/pi-mono-extension.ts +69 -0
- package/src/providers/runtime-dependencies.ts +1 -1
- package/src/rbac/legacy-policy.ts +3 -0
- package/src/rbac/permissions.ts +11 -1
- package/src/scheduler/deferred-task-waits.ts +170 -13
- package/src/scripts-runtime/executors/types.ts +5 -0
- package/src/scripts-runtime/loader.ts +8 -0
- package/src/scripts-runtime/mcp-client.ts +4 -0
- package/src/scripts-runtime/sdk-allowlist.ts +14 -1
- package/src/scripts-runtime/swarm-config.ts +8 -0
- package/src/scripts-runtime/swarm-sdk.ts +4 -0
- package/src/scripts-runtime/types/stdlib.d.ts +47 -4
- package/src/scripts-runtime/types/swarm-extension.d.ts +1 -0
- package/src/scripts-runtime/types/swarm-sdk.d.ts +47 -4
- package/src/server.ts +4 -0
- package/src/slack/enrich.ts +81 -21
- package/src/slack/handlers.test.ts +55 -2
- package/src/slack/handlers.ts +38 -3
- package/src/slack/render-v2.ts +19 -0
- package/src/slack/session-mirror.ts +208 -0
- package/src/slack/thread-participants.ts +0 -5
- package/src/slack/watcher.ts +3 -0
- package/src/slack/work-objects.test.ts +2 -1
- package/src/slack/work-objects.ts +7 -2
- package/src/tests/agentmail-filters.test.ts +4 -3
- package/src/tests/audit-agent-person.test.ts +825 -0
- package/src/tests/audit-updated-by.test.ts +19 -12
- package/src/tests/claude-hook-output.test.ts +107 -2
- package/src/tests/claude-managed-setup.test.ts +1 -1
- package/src/tests/codex-adapter.test.ts +6 -6
- package/src/tests/codex-gpt6-release.test.ts +4 -4
- package/src/tests/db-task-read-contract.test.ts +1 -0
- package/src/tests/db-task-write-contract.test.ts +1 -0
- package/src/tests/dsh-adapter.test.ts +1 -1
- package/src/tests/extensions-catalog.test.ts +33 -25
- package/src/tests/fixtures/sheets/eocd-count-mismatch.xlsx +0 -0
- package/src/tests/fixtures/sheets/rich-budget.xlsx +0 -0
- package/src/tests/fixtures/sheets/tracker-export.xlsx +0 -0
- package/src/tests/kv-storage.test.ts +3 -0
- package/src/tests/memory-search-reader.test.ts +258 -0
- package/src/tests/memory-search-stall.test.ts +197 -20
- package/src/tests/model-control.test.ts +4 -4
- package/src/tests/model-tier-resolution.test.ts +2 -2
- package/src/tests/multi-runtime-registration.test.ts +8 -2
- package/src/tests/native-tool-hooks.test.ts +188 -0
- package/src/tests/opencode-swarm-plugin.test.ts +6 -1
- package/src/tests/pi-provider-runtime-keys.test.ts +163 -0
- package/src/tests/poll-probe-readonly.test.ts +108 -0
- package/src/tests/primitives-sheets-xlsx.test.ts +549 -0
- package/src/tests/prompt-template-session.test.ts +3 -2
- package/src/tests/rbac-roles.test.ts +3 -0
- package/src/tests/script-connections.test.ts +4 -0
- package/src/tests/seed-model-profiles.test.ts +1 -1
- package/src/tests/sessions-tree.test.ts +10 -2
- package/src/tests/slack-thread-participants.test.ts +66 -4
- package/src/tests/tool-registrar-no-input.test.ts +1 -0
- package/src/tools/defer-task.ts +154 -12
- package/src/tools/objects.ts +10 -5
- package/src/tools/repos/update-repo.ts +6 -6
- package/src/tools/schedules/create-schedule.ts +1 -0
- package/src/tools/schedules/list-schedules.ts +3 -0
- package/src/tools/schedules/patch-schedule.ts +12 -3
- package/src/tools/schedules/update-schedule.ts +12 -3
- package/src/tools/sheets.ts +66 -0
- package/src/tools/skills/skill-create.ts +6 -0
- package/src/tools/skills/skill-install-remote.ts +6 -0
- package/src/tools/skills/skill-sync-remote.ts +9 -0
- package/src/tools/skills/skill-update.ts +9 -1
- package/src/tools/tool-config.ts +3 -1
- package/src/tools/utils.ts +15 -0
- package/src/tools/workflows/create-workflow.ts +9 -4
- package/src/tools/workflows/patch-workflow-node.ts +1 -1
- package/src/tools/workflows/patch-workflow.ts +5 -2
- package/src/tools/workflows/update-workflow.ts +13 -7
- package/src/types.ts +96 -12
- package/src/workflows/input.ts +12 -5
- package/src/workflows/patch-definition.ts +13 -5
- package/src/workflows/triggers.ts +11 -6
- package/src/workflows/version.ts +7 -1
- package/templates/skills/boards-and-sheets/config.json +14 -0
- package/templates/skills/boards-and-sheets/content.md +163 -0
- package/templates/skills/swarm-docs/content.md +9 -0
- package/dist/acp-adapter-11m13kx4.js +0 -16
- package/dist/core-dxcy02sc.js +0 -41
- package/dist/fx-adapter-0ar6pqm3.js +0 -19
- package/dist/grok-adapter-t6m74fb1.js +0 -19
- package/dist/handlers-9kwcrc4k.js +0 -38
- package/dist/index-23cdfzr6.js +0 -39
- package/dist/index-g7kkjs5b.js +0 -38
- package/dist/index-stk36pa5.js +0 -43
- package/dist/lead-mfggjtfk.js +0 -77
- package/dist/thread-buffer-temgaqsy.js +0 -41
- package/dist/worker-efcemc22.js +0 -77
- /package/dist/{cli-cah298e5.js → cli-m4ejz4nj.js} +0 -0
|
@@ -3,6 +3,7 @@ import { cosineSimilarity, deserializeEmbedding, serializeEmbedding } from "@/be
|
|
|
3
3
|
import { contentSha256 } from "@/commands/profile-sync";
|
|
4
4
|
import type { AgentMemory, AgentMemoryScope, AgentMemorySource } from "@/types";
|
|
5
5
|
import { scrubSecrets } from "@/utils/secret-scrubber";
|
|
6
|
+
import { isInTransaction } from "../../db-client";
|
|
6
7
|
import { currentLoopPath, withLoopPath } from "../../event-loop-lag";
|
|
7
8
|
import {
|
|
8
9
|
EMBEDDING_DIMENSIONS,
|
|
@@ -17,6 +18,7 @@ import { isLearningKey, isLongtermKey, LONGTERM_ROOT, tierSource } from "../key-
|
|
|
17
18
|
import { recencyDecay } from "../reranker";
|
|
18
19
|
import { notReviewTaskCompletionSql } from "../review-recall-filter";
|
|
19
20
|
import { MAX_CANDIDATE_FETCH } from "../search-candidates";
|
|
21
|
+
import { memorySearchReader } from "../search-reader";
|
|
20
22
|
import type {
|
|
21
23
|
MemoryCandidate,
|
|
22
24
|
MemoryDocumentRef,
|
|
@@ -62,6 +64,10 @@ export const MEMORY_ID_BATCH_SQL = `SELECT rowid AS rowid, id FROM agent_memory
|
|
|
62
64
|
WHERE rowid > ? ORDER BY rowid LIMIT ?`;
|
|
63
65
|
/** How long search reuses one health result. Stale by at most this long. */
|
|
64
66
|
export const SEARCH_HEALTH_TTL_MS = 60_000;
|
|
67
|
+
/** Off-loop health refreshes a search retries when writes invalidate them, before it counts in-process. */
|
|
68
|
+
const SEARCH_HEALTH_MAX_REFRESHES = 3;
|
|
69
|
+
/** An off-loop health refresh, for the DB and generation it started under. */
|
|
70
|
+
type SearchHealthRefresh = { db: unknown; generation: number; promise: Promise<MemoryHealth> };
|
|
65
71
|
const FTS_SCAN_BATCH_SIZE = 1000;
|
|
66
72
|
const FTS_POPULATE_BATCH_SIZE = 500;
|
|
67
73
|
|
|
@@ -210,6 +216,13 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
210
216
|
private ftsPopulate: Promise<void> | null = null;
|
|
211
217
|
private lastPopulate: MemoryVecPopulateStats | null = null;
|
|
212
218
|
private searchHealthCache: { health: MemoryHealth; db: unknown; at: number } | null = null;
|
|
219
|
+
/** The newest refresh. An older one may still run; its result is checked, not shared. */
|
|
220
|
+
private searchHealthRefresh: SearchHealthRefresh | null = null;
|
|
221
|
+
/**
|
|
222
|
+
* Bumped when the cache is dropped. A refresh that started under an older
|
|
223
|
+
* generation is neither stored nor returned to a search waiting on it.
|
|
224
|
+
*/
|
|
225
|
+
private searchHealthGeneration = 0;
|
|
213
226
|
|
|
214
227
|
constructor() {
|
|
215
228
|
this.ensureVecTable();
|
|
@@ -476,7 +489,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
476
489
|
}
|
|
477
490
|
|
|
478
491
|
const afterCount = this.getVecCount();
|
|
479
|
-
this.
|
|
492
|
+
this.dropSearchHealth();
|
|
480
493
|
this.lastPopulate = {
|
|
481
494
|
attempted,
|
|
482
495
|
inserted,
|
|
@@ -672,7 +685,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
672
685
|
});
|
|
673
686
|
}
|
|
674
687
|
|
|
675
|
-
const health = this.searchHealth();
|
|
688
|
+
const health = await this.searchHealth();
|
|
676
689
|
if (
|
|
677
690
|
isHybridSearchEnabled() &&
|
|
678
691
|
options.queryText &&
|
|
@@ -812,7 +825,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
812
825
|
): Promise<void> {
|
|
813
826
|
const missing = ids.filter((id) => !cosines.has(id));
|
|
814
827
|
if (missing.length === 0) return;
|
|
815
|
-
const rows = await
|
|
828
|
+
const rows = await memorySearchReader.query<{ id: string; embedding: Buffer | null }>(
|
|
816
829
|
`SELECT id, embedding FROM agent_memory WHERE id IN (${missing.map(() => "?").join(",")})`,
|
|
817
830
|
missing,
|
|
818
831
|
);
|
|
@@ -859,7 +872,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
859
872
|
|
|
860
873
|
try {
|
|
861
874
|
const sqlLimit = armFetchLimit(limit);
|
|
862
|
-
const rows = await
|
|
875
|
+
const rows = await memorySearchReader.query<AgentMemoryRow & { rank: number }>(
|
|
863
876
|
`SELECT m.*, bm25(memory_fts) AS rank
|
|
864
877
|
FROM memory_fts
|
|
865
878
|
JOIN agent_memory m ON m.id = memory_fts.memory_id
|
|
@@ -944,7 +957,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
944
957
|
conditions.push("v.k = ?");
|
|
945
958
|
params.push(knnLimit);
|
|
946
959
|
|
|
947
|
-
const rows = await
|
|
960
|
+
const rows = await memorySearchReader.query<AgentMemoryRow & { distance: number }>(
|
|
948
961
|
`SELECT m.*, v.distance
|
|
949
962
|
FROM memory_vec v
|
|
950
963
|
JOIN agent_memory m ON m.id = v.memory_id
|
|
@@ -1020,7 +1033,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
1020
1033
|
);
|
|
1021
1034
|
if (top.length === 0) return [];
|
|
1022
1035
|
|
|
1023
|
-
const rows = await
|
|
1036
|
+
const rows = await memorySearchReader.query<AgentMemoryRow>(
|
|
1024
1037
|
`SELECT * FROM agent_memory WHERE id IN (${top.map(() => "?").join(", ")})`,
|
|
1025
1038
|
top.map((hit) => hit.id),
|
|
1026
1039
|
);
|
|
@@ -1042,7 +1055,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
1042
1055
|
floor: number,
|
|
1043
1056
|
limit: number,
|
|
1044
1057
|
): Promise<{ scanned: number; top: { id: string; similarity: number }[] }> {
|
|
1045
|
-
const db =
|
|
1058
|
+
const db = memorySearchReader;
|
|
1046
1059
|
const counted = await db.get<{ n: number }>(
|
|
1047
1060
|
`SELECT COUNT(*) AS n FROM (SELECT 1 ${capped})`,
|
|
1048
1061
|
cappedParams,
|
|
@@ -1073,7 +1086,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
1073
1086
|
floor: number,
|
|
1074
1087
|
limit: number,
|
|
1075
1088
|
): Promise<{ scanned: number; top: { id: string; similarity: number }[] }> {
|
|
1076
|
-
const rows = await
|
|
1089
|
+
const rows = await memorySearchReader.query<{ id: string; embedding: Buffer }>(
|
|
1077
1090
|
`SELECT id, embedding ${capped}`,
|
|
1078
1091
|
cappedParams,
|
|
1079
1092
|
);
|
|
@@ -1123,7 +1136,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
1123
1136
|
|
|
1124
1137
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
1125
1138
|
|
|
1126
|
-
const rows = await
|
|
1139
|
+
const rows = await memorySearchReader.query<AgentMemoryRow>(
|
|
1127
1140
|
`SELECT * FROM agent_memory ${whereClause}`,
|
|
1128
1141
|
params,
|
|
1129
1142
|
);
|
|
@@ -1591,7 +1604,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
1591
1604
|
} catch (err) {
|
|
1592
1605
|
console.error(`[memory-vec] update failed memory_id=${id}: ${(err as Error).message}`);
|
|
1593
1606
|
// The row now misses from memory_vec: search must see the fallback.
|
|
1594
|
-
this.
|
|
1607
|
+
this.invalidateSearchHealth();
|
|
1595
1608
|
}
|
|
1596
1609
|
}
|
|
1597
1610
|
});
|
|
@@ -1694,6 +1707,51 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
1694
1707
|
};
|
|
1695
1708
|
}
|
|
1696
1709
|
|
|
1710
|
+
/**
|
|
1711
|
+
* The same eight counts as getHealthCounts(), as one statement on a read
|
|
1712
|
+
* worker so the scans block that thread and not the API event loop. One
|
|
1713
|
+
* statement keeps the other worker free for searches while it runs.
|
|
1714
|
+
*/
|
|
1715
|
+
private async getHealthCountsOffLoop(): Promise<MemoryHealth["counts"]> {
|
|
1716
|
+
const tableUsable = this.getVecTableSchema() !== null && isSqliteVecAvailable();
|
|
1717
|
+
const valid = `embedding IS NOT NULL AND length(embedding) = ${VECTOR_BYTES}`;
|
|
1718
|
+
const row = await memorySearchReader.get<Record<keyof MemoryHealth["counts"], number>>(
|
|
1719
|
+
`SELECT
|
|
1720
|
+
(SELECT COUNT(*) FROM agent_memory) AS total,
|
|
1721
|
+
(SELECT COUNT(*) FROM agent_memory WHERE embedding IS NOT NULL) AS withEmbedding,
|
|
1722
|
+
(SELECT COUNT(*) FROM agent_memory WHERE ${valid}) AS validEmbedding,
|
|
1723
|
+
(SELECT COUNT(*) FROM agent_memory
|
|
1724
|
+
WHERE embedding IS NOT NULL AND length(embedding) != ${VECTOR_BYTES}) AS invalidEmbedding,
|
|
1725
|
+
(SELECT COUNT(*) FROM agent_memory
|
|
1726
|
+
WHERE ${valid} AND (expiresAt IS NULL OR expiresAt > datetime('now'))) AS searchable,
|
|
1727
|
+
${tableUsable ? "(SELECT COUNT(*) FROM memory_vec)" : "0"} AS memoryVec,
|
|
1728
|
+
${
|
|
1729
|
+
tableUsable
|
|
1730
|
+
? `(SELECT COUNT(*) FROM agent_memory WHERE ${valid})
|
|
1731
|
+
- (SELECT COUNT(*) FROM memory_vec v CROSS JOIN agent_memory m ON m.id = v.memory_id
|
|
1732
|
+
WHERE m.embedding IS NOT NULL AND length(m.embedding) = ${VECTOR_BYTES})`
|
|
1733
|
+
: `(SELECT COUNT(*) FROM agent_memory WHERE ${valid})`
|
|
1734
|
+
} AS missingFromVec,
|
|
1735
|
+
${
|
|
1736
|
+
tableUsable
|
|
1737
|
+
? `(SELECT COUNT(*) FROM memory_vec v LEFT JOIN agent_memory m ON m.id = v.memory_id
|
|
1738
|
+
WHERE m.id IS NULL)`
|
|
1739
|
+
: "0"
|
|
1740
|
+
} AS extraInVec`,
|
|
1741
|
+
);
|
|
1742
|
+
if (!row) throw new Error("health counts returned no row");
|
|
1743
|
+
return {
|
|
1744
|
+
total: row.total,
|
|
1745
|
+
withEmbedding: row.withEmbedding,
|
|
1746
|
+
validEmbedding: row.validEmbedding,
|
|
1747
|
+
invalidEmbedding: row.invalidEmbedding,
|
|
1748
|
+
searchable: row.searchable,
|
|
1749
|
+
memoryVec: row.memoryVec,
|
|
1750
|
+
missingFromVec: row.missingFromVec,
|
|
1751
|
+
extraInVec: row.extraInVec,
|
|
1752
|
+
};
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1697
1755
|
private getHealthCounts(): MemoryHealth["counts"] {
|
|
1698
1756
|
const db = getDb();
|
|
1699
1757
|
const tableExists = this.getVecTableSchema() !== null;
|
|
@@ -1747,33 +1805,108 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
1747
1805
|
|
|
1748
1806
|
/**
|
|
1749
1807
|
* getHealth() for the search path. The counts scan agent_memory and
|
|
1750
|
-
* memory_vec
|
|
1751
|
-
*
|
|
1808
|
+
* memory_vec (about 0.8 s at 63k memories), so search never runs them on the
|
|
1809
|
+
* event loop: one read worker computes them, search reuses one result per
|
|
1810
|
+
* window, and an expired "vec" result is served while a refresh runs in the
|
|
1811
|
+
* background. The health endpoint keeps calling getHealth() for fresh counts.
|
|
1752
1812
|
*/
|
|
1753
|
-
private searchHealth(): MemoryHealth {
|
|
1813
|
+
private async searchHealth(): Promise<MemoryHealth> {
|
|
1754
1814
|
const db = getDb();
|
|
1755
1815
|
const cached = this.searchHealthCache;
|
|
1816
|
+
const inTransaction = isInTransaction();
|
|
1756
1817
|
// The db check drops a result that outlived a DB swap (tests reinit it).
|
|
1757
|
-
if (cached && cached.db === db
|
|
1758
|
-
return cached.health;
|
|
1818
|
+
if (cached && cached.db === db) {
|
|
1819
|
+
if (Date.now() - cached.at < SEARCH_HEALTH_TTL_MS) return cached.health;
|
|
1820
|
+
// A stale "vec" is at worst one refresh old; a stale fallback is
|
|
1821
|
+
// awaited so a repaired index is seen on the next search.
|
|
1822
|
+
if (cached.health.retrievalMode === "vec" && !inTransaction) {
|
|
1823
|
+
if (!this.searchHealthRefresh) this.startSearchHealthRefresh(db).promise.catch(() => {});
|
|
1824
|
+
return cached.health;
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
1827
|
+
// A transaction must see its own writes, which may still roll back, and
|
|
1828
|
+
// its search reads run in-process anyway: count there, share and store nothing.
|
|
1829
|
+
if (inTransaction) return this.getHealth();
|
|
1830
|
+
return this.currentSearchHealth();
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
/**
|
|
1834
|
+
* A health result that no write invalidated before it returns. Searches
|
|
1835
|
+
* share one refresh per DB and generation; one a write overtook is retried,
|
|
1836
|
+
* for every search waiting on it.
|
|
1837
|
+
*/
|
|
1838
|
+
private async currentSearchHealth(): Promise<MemoryHealth> {
|
|
1839
|
+
for (let attempt = 0; attempt < SEARCH_HEALTH_MAX_REFRESHES; attempt++) {
|
|
1840
|
+
const db = getDb();
|
|
1841
|
+
const generation = this.searchHealthGeneration;
|
|
1842
|
+
const inFlight = this.searchHealthRefresh;
|
|
1843
|
+
const refresh =
|
|
1844
|
+
inFlight && inFlight.db === db && inFlight.generation === generation
|
|
1845
|
+
? inFlight
|
|
1846
|
+
: this.startSearchHealthRefresh(db);
|
|
1847
|
+
const health = await refresh.promise;
|
|
1848
|
+
if (getDb() === db && this.searchHealthGeneration === generation) return health;
|
|
1759
1849
|
}
|
|
1850
|
+
// Writes kept overtaking the off-loop counts. In-process, no write can
|
|
1851
|
+
// land between the count and the return.
|
|
1760
1852
|
const health = this.getHealth();
|
|
1761
|
-
this.searchHealthCache = { health, db, at: Date.now() };
|
|
1853
|
+
this.searchHealthCache = { health, db: getDb(), at: Date.now() };
|
|
1762
1854
|
return health;
|
|
1763
1855
|
}
|
|
1764
1856
|
|
|
1857
|
+
private startSearchHealthRefresh(db: unknown): SearchHealthRefresh {
|
|
1858
|
+
const generation = this.searchHealthGeneration;
|
|
1859
|
+
const promise = (async () => {
|
|
1860
|
+
let health: MemoryHealth;
|
|
1861
|
+
try {
|
|
1862
|
+
health = this.healthFromCounts(await this.getHealthCountsOffLoop());
|
|
1863
|
+
} catch (err) {
|
|
1864
|
+
console.warn("[memory-search] health counts failed:", (err as Error).message);
|
|
1865
|
+
health = this.getHealth();
|
|
1866
|
+
}
|
|
1867
|
+
// A DB swap or a vec write mid-refresh invalidated this result.
|
|
1868
|
+
if (getDb() === db && generation === this.searchHealthGeneration)
|
|
1869
|
+
this.searchHealthCache = { health, db, at: Date.now() };
|
|
1870
|
+
return health;
|
|
1871
|
+
})().finally(() => {
|
|
1872
|
+
// A newer refresh may have replaced this one: leave it in place.
|
|
1873
|
+
if (this.searchHealthRefresh === entry) this.searchHealthRefresh = null;
|
|
1874
|
+
});
|
|
1875
|
+
const entry: SearchHealthRefresh = { db, generation, promise };
|
|
1876
|
+
this.searchHealthRefresh = entry;
|
|
1877
|
+
return entry;
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1765
1880
|
/**
|
|
1766
1881
|
* A vec write can heal a cached fallback (first vector, backfilled row), so
|
|
1767
1882
|
* it drops one. A cached "vec" stays: dropping it on every write would
|
|
1768
1883
|
* recount on the next search, and a stale "vec" is at worst one TTL old.
|
|
1769
1884
|
*/
|
|
1770
1885
|
private noteVecWrite(): void {
|
|
1771
|
-
if (this.searchHealthCache?.health.retrievalMode !== "vec") this.
|
|
1886
|
+
if (this.searchHealthCache?.health.retrievalMode !== "vec") this.invalidateSearchHealth();
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
/**
|
|
1890
|
+
* Drop now, so no refresh in flight is stored or returned, and again once
|
|
1891
|
+
* the writing transaction commits: a refresh that starts before the commit
|
|
1892
|
+
* reads the snapshot without this write.
|
|
1893
|
+
*/
|
|
1894
|
+
private invalidateSearchHealth(): void {
|
|
1895
|
+
this.dropSearchHealth();
|
|
1896
|
+
if (isInTransaction()) getDbClient().afterCommit(() => this.dropSearchHealth());
|
|
1897
|
+
}
|
|
1898
|
+
|
|
1899
|
+
private dropSearchHealth(): void {
|
|
1900
|
+
this.searchHealthCache = null;
|
|
1901
|
+
this.searchHealthGeneration++;
|
|
1772
1902
|
}
|
|
1773
1903
|
|
|
1774
1904
|
getHealth(): MemoryHealth {
|
|
1905
|
+
return this.healthFromCounts(this.getHealthCounts());
|
|
1906
|
+
}
|
|
1907
|
+
|
|
1908
|
+
private healthFromCounts(counts: MemoryHealth["counts"]): MemoryHealth {
|
|
1775
1909
|
const schema = this.getVecTableSchema();
|
|
1776
|
-
const counts = this.getHealthCounts();
|
|
1777
1910
|
const reasons: string[] = [];
|
|
1778
1911
|
|
|
1779
1912
|
if (!isSqliteVecAvailable()) reasons.push("sqlite_vec_extension_unavailable");
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker thread entry for memory search reads. It holds its own read-only
|
|
3
|
+
* connection to the API's WAL database file (with sqlite-vec when the host
|
|
4
|
+
* has it), so the vec KNN and bm25 statements block this thread, not the API
|
|
5
|
+
* event loop. Host side: search-reader.ts.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { Database } from "bun:sqlite";
|
|
9
|
+
import type { DbParam } from "../db-client";
|
|
10
|
+
|
|
11
|
+
declare const self: Worker;
|
|
12
|
+
|
|
13
|
+
export type SearchReadInit = { type: "init"; file: string; vecExtensionPath?: string };
|
|
14
|
+
export type SearchReadRequest = {
|
|
15
|
+
type: "read";
|
|
16
|
+
id: number;
|
|
17
|
+
mode: "all" | "get";
|
|
18
|
+
sql: string;
|
|
19
|
+
params: DbParam[];
|
|
20
|
+
};
|
|
21
|
+
export type SearchReadMessage =
|
|
22
|
+
| { type: "ready" }
|
|
23
|
+
| { type: "fatal"; message: string }
|
|
24
|
+
| { type: "result"; id: number; rows: unknown }
|
|
25
|
+
| { type: "error"; id: number; message: string };
|
|
26
|
+
|
|
27
|
+
let db: Database | null = null;
|
|
28
|
+
|
|
29
|
+
self.onmessage = (event: MessageEvent<SearchReadInit | SearchReadRequest>) => {
|
|
30
|
+
const message = event.data;
|
|
31
|
+
if (message.type === "init") {
|
|
32
|
+
try {
|
|
33
|
+
db = new Database(message.file, { readonly: true });
|
|
34
|
+
// Same lock wait as the main connection; WAL readers rarely wait at all.
|
|
35
|
+
db.run("PRAGMA busy_timeout = 5000;");
|
|
36
|
+
db.run("PRAGMA mmap_size = 268435456;");
|
|
37
|
+
if (message.vecExtensionPath) db.loadExtension(message.vecExtensionPath);
|
|
38
|
+
self.postMessage({ type: "ready" } satisfies SearchReadMessage);
|
|
39
|
+
} catch (err) {
|
|
40
|
+
self.postMessage({
|
|
41
|
+
type: "fatal",
|
|
42
|
+
message: (err as Error).message,
|
|
43
|
+
} satisfies SearchReadMessage);
|
|
44
|
+
}
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const { id, mode, sql, params } = message;
|
|
48
|
+
try {
|
|
49
|
+
// db.query caches the statement, as the main connection's client does.
|
|
50
|
+
const statement = db!.query<unknown, DbParam[]>(sql);
|
|
51
|
+
const rows = mode === "all" ? statement.all(...params) : statement.get(...params);
|
|
52
|
+
self.postMessage({ type: "result", id, rows } satisfies SearchReadMessage);
|
|
53
|
+
} catch (err) {
|
|
54
|
+
self.postMessage({
|
|
55
|
+
type: "error",
|
|
56
|
+
id,
|
|
57
|
+
message: (err as Error).message,
|
|
58
|
+
} satisfies SearchReadMessage);
|
|
59
|
+
}
|
|
60
|
+
};
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory search reads off the API event loop.
|
|
3
|
+
*
|
|
4
|
+
* bun:sqlite is synchronous: at ~56k memories one search's vec KNN and bm25
|
|
5
|
+
* statements held the loop ~1.3 s, and every other request waited. These
|
|
6
|
+
* reads run in a small pool of Bun workers, each with its own read-only
|
|
7
|
+
* connection to the same WAL file, so the loop only awaits a message. The
|
|
8
|
+
* SQL and params are the ones the in-process path runs, so results match.
|
|
9
|
+
*
|
|
10
|
+
* The in-process path (getDbClient()) still serves a read when:
|
|
11
|
+
* - it runs inside a transaction (it must see that transaction's writes),
|
|
12
|
+
* - the database is not a file (tests restore an in-memory template),
|
|
13
|
+
* - the workers failed to start or one died (logged once).
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { getDb, getDbClient, isSqliteVecAvailable, resolveSqliteVecExtensionPath } from "@/be/db";
|
|
17
|
+
import { type DbParam, isInTransaction } from "../db-client";
|
|
18
|
+
import type { SearchReadMessage, SearchReadRequest } from "./search-read-worker";
|
|
19
|
+
|
|
20
|
+
/** Workers in the pool. Reads beyond this wait in a FIFO queue, so a burst never piles onto SQLite. */
|
|
21
|
+
export const SEARCH_READ_WORKERS = 2;
|
|
22
|
+
const START_TIMEOUT_MS = 10_000;
|
|
23
|
+
|
|
24
|
+
type Mode = SearchReadRequest["mode"];
|
|
25
|
+
type Job = {
|
|
26
|
+
id: number;
|
|
27
|
+
mode: Mode;
|
|
28
|
+
sql: string;
|
|
29
|
+
params: DbParam[];
|
|
30
|
+
resolve: (rows: unknown) => void;
|
|
31
|
+
reject: (err: Error) => void;
|
|
32
|
+
};
|
|
33
|
+
/** Bun's Worker has ref/unref; the DOM lib the extension type bundle compiles against does not. */
|
|
34
|
+
type RefWorker = Worker & { ref?: () => void; unref?: () => void };
|
|
35
|
+
type Slot = { worker: RefWorker; job: Job | null };
|
|
36
|
+
|
|
37
|
+
/** A worker died or was stopped with this read in flight; the read is safe to rerun in-process. */
|
|
38
|
+
class WorkerGoneError extends Error {}
|
|
39
|
+
|
|
40
|
+
/** In the compiled API binary every `import.meta.url` is the binary; see the Dockerfile compile step. */
|
|
41
|
+
function workerSpecifier(): string {
|
|
42
|
+
if (import.meta.url.includes("/$bunfs/")) return "./be/memory/search-read-worker.ts";
|
|
43
|
+
return new URL("./search-read-worker.ts", import.meta.url).href;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function spawnWorker(file: string, vecExtensionPath: string | undefined): Promise<Worker> {
|
|
47
|
+
return new Promise<Worker>((resolve, reject) => {
|
|
48
|
+
// Inside the executor, so a constructor that throws rejects like a failed boot.
|
|
49
|
+
const worker: RefWorker = new Worker(workerSpecifier());
|
|
50
|
+
// An idle pool must not keep the API process (or a test run) alive.
|
|
51
|
+
worker.unref?.();
|
|
52
|
+
const fail = (message: string) => {
|
|
53
|
+
clearTimeout(timer);
|
|
54
|
+
worker.terminate();
|
|
55
|
+
reject(new Error(message));
|
|
56
|
+
};
|
|
57
|
+
const timer = setTimeout(() => fail("timed out starting"), START_TIMEOUT_MS);
|
|
58
|
+
worker.addEventListener("message", function onReady(event: MessageEvent<SearchReadMessage>) {
|
|
59
|
+
worker.removeEventListener("message", onReady);
|
|
60
|
+
if (event.data.type === "ready") {
|
|
61
|
+
clearTimeout(timer);
|
|
62
|
+
resolve(worker);
|
|
63
|
+
} else fail(event.data.type === "fatal" ? event.data.message : "unexpected first message");
|
|
64
|
+
});
|
|
65
|
+
worker.addEventListener("error", (event) => fail(event.message), { once: true });
|
|
66
|
+
worker.postMessage({ type: "init", file, vecExtensionPath });
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
class SearchReadPool {
|
|
71
|
+
private readonly idle: Slot[];
|
|
72
|
+
private readonly queue: Job[] = [];
|
|
73
|
+
private readonly slots: Slot[];
|
|
74
|
+
private nextId = 1;
|
|
75
|
+
broken = false;
|
|
76
|
+
|
|
77
|
+
constructor(workers: Worker[]) {
|
|
78
|
+
this.slots = workers.map((worker) => ({ worker, job: null }));
|
|
79
|
+
this.idle = [...this.slots];
|
|
80
|
+
for (const slot of this.slots) {
|
|
81
|
+
slot.worker.addEventListener("message", (event: MessageEvent<SearchReadMessage>) =>
|
|
82
|
+
this.settle(slot, event.data),
|
|
83
|
+
);
|
|
84
|
+
slot.worker.addEventListener("error", (event) => this.stop(`worker error: ${event.message}`));
|
|
85
|
+
slot.worker.addEventListener("close", () => this.stop("worker exited"));
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
get stats() {
|
|
90
|
+
return {
|
|
91
|
+
workers: this.slots.length,
|
|
92
|
+
busy: this.slots.filter((slot) => slot.job).length,
|
|
93
|
+
queued: this.queue.length,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
read(mode: Mode, sql: string, params: DbParam[]): Promise<unknown> {
|
|
98
|
+
if (this.broken) return Promise.reject(new WorkerGoneError("pool stopped"));
|
|
99
|
+
return new Promise((resolve, reject) => {
|
|
100
|
+
this.queue.push({ id: this.nextId++, mode, sql, params, resolve, reject });
|
|
101
|
+
this.pump();
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
private pump(): void {
|
|
106
|
+
while (this.idle.length > 0 && this.queue.length > 0) {
|
|
107
|
+
const slot = this.idle.pop()!;
|
|
108
|
+
const job = this.queue.shift()!;
|
|
109
|
+
slot.job = job;
|
|
110
|
+
// Busy workers hold the process open, so an awaited read always settles.
|
|
111
|
+
slot.worker.ref?.();
|
|
112
|
+
const request: SearchReadRequest = {
|
|
113
|
+
type: "read",
|
|
114
|
+
id: job.id,
|
|
115
|
+
mode: job.mode,
|
|
116
|
+
sql: job.sql,
|
|
117
|
+
params: job.params,
|
|
118
|
+
};
|
|
119
|
+
slot.worker.postMessage(request);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
private settle(slot: Slot, message: SearchReadMessage): void {
|
|
124
|
+
const job = slot.job;
|
|
125
|
+
if (!job || (message.type !== "result" && message.type !== "error") || message.id !== job.id) {
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
slot.job = null;
|
|
129
|
+
slot.worker.unref?.();
|
|
130
|
+
this.idle.push(slot);
|
|
131
|
+
if (message.type === "result") job.resolve(message.rows);
|
|
132
|
+
else job.reject(new Error(message.message));
|
|
133
|
+
this.pump();
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Fail every pending read over to the in-process path and stop the workers. */
|
|
137
|
+
stop(reason: string, log = true): void {
|
|
138
|
+
if (this.broken) return;
|
|
139
|
+
this.broken = true;
|
|
140
|
+
if (log) console.warn(`[memory-search] read workers stopped (${reason}); reads run in-process`);
|
|
141
|
+
const pending = [...this.slots.flatMap((slot) => (slot.job ? [slot.job] : [])), ...this.queue];
|
|
142
|
+
this.queue.length = 0;
|
|
143
|
+
for (const slot of this.slots) {
|
|
144
|
+
slot.job = null;
|
|
145
|
+
slot.worker.terminate();
|
|
146
|
+
}
|
|
147
|
+
for (const job of pending) job.reject(new WorkerGoneError(reason));
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// One pool per database handle: a closed and reopened DB (tests) gets fresh workers.
|
|
152
|
+
let current: { db: unknown; pool: Promise<SearchReadPool | null> } | null = null;
|
|
153
|
+
let inProcessOnly = false;
|
|
154
|
+
|
|
155
|
+
function poolForCurrentDb(): Promise<SearchReadPool | null> {
|
|
156
|
+
const db = getDb();
|
|
157
|
+
if (current?.db === db) return current.pool;
|
|
158
|
+
void current?.pool.then((pool) => pool?.stop("database handle changed", false));
|
|
159
|
+
const file = db.filename;
|
|
160
|
+
const pool =
|
|
161
|
+
!file || file === ":memory:"
|
|
162
|
+
? Promise.resolve(null)
|
|
163
|
+
: startPool(file, isSqliteVecAvailable() ? resolveSqliteVecExtensionPath() : undefined);
|
|
164
|
+
current = { db, pool };
|
|
165
|
+
return pool;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
async function startPool(
|
|
169
|
+
file: string,
|
|
170
|
+
vecExtensionPath: string | undefined,
|
|
171
|
+
): Promise<SearchReadPool | null> {
|
|
172
|
+
const started = await Promise.allSettled(
|
|
173
|
+
Array.from({ length: SEARCH_READ_WORKERS }, () => spawnWorker(file, vecExtensionPath)),
|
|
174
|
+
);
|
|
175
|
+
const failed = started.find((result) => result.status === "rejected");
|
|
176
|
+
if (failed) {
|
|
177
|
+
for (const result of started) if (result.status === "fulfilled") result.value.terminate();
|
|
178
|
+
console.warn(
|
|
179
|
+
`[memory-search] read workers failed to start (${(failed.reason as Error).message}); reads run in-process`,
|
|
180
|
+
);
|
|
181
|
+
return null;
|
|
182
|
+
}
|
|
183
|
+
console.log(`[memory-search] read workers started workers=${SEARCH_READ_WORKERS}`);
|
|
184
|
+
return new SearchReadPool(
|
|
185
|
+
started.map((result) => (result as PromiseFulfilledResult<Worker>).value),
|
|
186
|
+
);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function readInProcess(mode: Mode, sql: string, params: DbParam[]): Promise<unknown> {
|
|
190
|
+
return mode === "all" ? getDbClient().query(sql, params) : getDbClient().get(sql, params);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
async function read(mode: Mode, sql: string, params: DbParam[]): Promise<unknown> {
|
|
194
|
+
if (inProcessOnly || isInTransaction()) return readInProcess(mode, sql, params);
|
|
195
|
+
const pool = await poolForCurrentDb();
|
|
196
|
+
if (!pool || pool.broken) return readInProcess(mode, sql, params);
|
|
197
|
+
try {
|
|
198
|
+
return await pool.read(mode, sql, params);
|
|
199
|
+
} catch (err) {
|
|
200
|
+
// shortcut: a dead worker stops the pool for this DB handle, no respawn; add one if crashes show up.
|
|
201
|
+
if (err instanceof WorkerGoneError) return readInProcess(mode, sql, params);
|
|
202
|
+
throw err;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** Read-only executor for memory search statements; same contract as getDbClient().query/get. */
|
|
207
|
+
export const memorySearchReader = {
|
|
208
|
+
query: <T>(sql: string, params: DbParam[] = []) => read("all", sql, params) as Promise<T[]>,
|
|
209
|
+
get: <T>(sql: string, params: DbParam[] = []) => read("get", sql, params) as Promise<T | null>,
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
/** Pool state for tests and diagnostics; null while no pool is running. */
|
|
213
|
+
export async function memorySearchReaderStats(): Promise<SearchReadPool["stats"] | null> {
|
|
214
|
+
const pool = await current?.pool;
|
|
215
|
+
return pool && !pool.broken ? pool.stats : null;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** Test-only: stop the pool; `inProcess` forces every read onto the main connection. */
|
|
219
|
+
export async function __resetMemorySearchReaderForTests(options: { inProcess?: boolean } = {}) {
|
|
220
|
+
(await current?.pool)?.stop("reset", false);
|
|
221
|
+
current = null;
|
|
222
|
+
inProcessOnly = options.inProcess ?? false;
|
|
223
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
-- Sessions sidebar: a single session can be starred (`session`, item id = the
|
|
2
|
+
-- root task id), and a group can be hidden from the sidebar (`hidden-key`,
|
|
3
|
+
-- item id = the normalized group key, like `key`). Both are per-principal
|
|
4
|
+
-- flags, so they reuse the favorites table and its owner scoping.
|
|
5
|
+
-- SQLite cannot change a CHECK in place, so rebuild the table with the 301
|
|
6
|
+
-- shape plus the new item types. Every row is copied unchanged.
|
|
7
|
+
PRAGMA defer_foreign_keys = ON;
|
|
8
|
+
|
|
9
|
+
CREATE TABLE user_favorites_new (
|
|
10
|
+
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
|
|
11
|
+
favoriteScope TEXT NOT NULL,
|
|
12
|
+
userId TEXT REFERENCES users(id) ON DELETE CASCADE,
|
|
13
|
+
itemType TEXT NOT NULL CHECK (itemType IN ('page','workflow','schedule','agent-fs-path','key','session','hidden-key')),
|
|
14
|
+
itemId TEXT NOT NULL,
|
|
15
|
+
createdAt TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
16
|
+
lastUpdatedAt TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
17
|
+
created_by TEXT,
|
|
18
|
+
updated_by TEXT,
|
|
19
|
+
UNIQUE (favoriteScope, itemType, itemId)
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
INSERT INTO user_favorites_new (
|
|
23
|
+
id, favoriteScope, userId, itemType, itemId,
|
|
24
|
+
createdAt, lastUpdatedAt, created_by, updated_by
|
|
25
|
+
)
|
|
26
|
+
SELECT
|
|
27
|
+
id, favoriteScope, userId, itemType, itemId,
|
|
28
|
+
createdAt, lastUpdatedAt, created_by, updated_by
|
|
29
|
+
FROM user_favorites;
|
|
30
|
+
|
|
31
|
+
DROP TABLE user_favorites;
|
|
32
|
+
ALTER TABLE user_favorites_new RENAME TO user_favorites;
|
|
33
|
+
|
|
34
|
+
CREATE INDEX idx_user_favorites_scope_type
|
|
35
|
+
ON user_favorites(favoriteScope, itemType);
|
|
36
|
+
CREATE INDEX idx_user_favorites_user_type
|
|
37
|
+
ON user_favorites(userId, itemType);
|
|
38
|
+
CREATE INDEX idx_user_favorites_item
|
|
39
|
+
ON user_favorites(itemType, itemId);
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
-- Who created / last updated a script, skill, workflow or schedule, as an
|
|
2
|
+
-- agent next to the existing person columns (created_by / updated_by, 082).
|
|
3
|
+
-- The agent columns carry the calling agent; NULL means a human, a seed or a
|
|
4
|
+
-- system write. skills.ownerAgentId keeps meaning ownership, not creator.
|
|
5
|
+
|
|
6
|
+
ALTER TABLE scripts ADD COLUMN updatedByAgentId TEXT;
|
|
7
|
+
ALTER TABLE workflows ADD COLUMN updatedByAgentId TEXT;
|
|
8
|
+
ALTER TABLE scheduled_tasks ADD COLUMN updatedByAgentId TEXT;
|
|
9
|
+
ALTER TABLE skills ADD COLUMN createdByAgentId TEXT;
|
|
10
|
+
ALTER TABLE skills ADD COLUMN updatedByAgentId TEXT;
|
|
11
|
+
|
|
12
|
+
-- Deterministic backfill only. No person backfill: nothing records it.
|
|
13
|
+
-- An agent-owned skill was created by its owner (skill-create and extension
|
|
14
|
+
-- installs set ownerAgentId to the caller; seeds and remote installs leave it NULL).
|
|
15
|
+
UPDATE skills SET createdByAgentId = ownerAgentId WHERE ownerAgentId IS NOT NULL;
|
|
16
|
+
|
|
17
|
+
-- Every script content write records a script_versions row, so the newest
|
|
18
|
+
-- one names the last agent to change it.
|
|
19
|
+
UPDATE scripts SET updatedByAgentId = (
|
|
20
|
+
SELECT v.changedByAgentId FROM script_versions v
|
|
21
|
+
WHERE v.scriptId = scripts.id
|
|
22
|
+
ORDER BY v.version DESC LIMIT 1
|
|
23
|
+
);
|
|
24
|
+
|
|
25
|
+
-- Versioned workflow updates snapshot the prior state with the updating agent.
|
|
26
|
+
UPDATE workflows SET updatedByAgentId = (
|
|
27
|
+
SELECT v.changedByAgentId FROM workflow_versions v
|
|
28
|
+
WHERE v.workflowId = workflows.id
|
|
29
|
+
ORDER BY v.version DESC LIMIT 1
|
|
30
|
+
);
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
-- Deferrals can wake on an external event (a VCS or tracker webhook) as well as
|
|
2
|
+
-- a task outcome. This table is a sibling of deferred_task_waits (150): that
|
|
3
|
+
-- table's eventName CHECK would need a rebuild to take a new event, so external
|
|
4
|
+
-- waits live here and the old one stays untouched. Same shape otherwise: one
|
|
5
|
+
-- wait per one-time schedule, the schedule's ceiling stays the deadline, and a
|
|
6
|
+
-- claim flips pending to fired.
|
|
7
|
+
-- Provider-neutral on purpose: `source` and `event` are plain text (no CHECK) and
|
|
8
|
+
-- the target is a canonical `matchKey` plus its `matchJson`, both produced by the
|
|
9
|
+
-- source's adapter in src/event-waits/. A new provider adds an adapter, not a migration.
|
|
10
|
+
CREATE TABLE IF NOT EXISTS deferred_event_waits (
|
|
11
|
+
scheduleId TEXT PRIMARY KEY REFERENCES scheduled_tasks(id) ON DELETE CASCADE,
|
|
12
|
+
source TEXT NOT NULL,
|
|
13
|
+
event TEXT NOT NULL,
|
|
14
|
+
matchKey TEXT NOT NULL,
|
|
15
|
+
matchJson TEXT NOT NULL,
|
|
16
|
+
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'fired')),
|
|
17
|
+
firedBy TEXT,
|
|
18
|
+
childTaskId TEXT,
|
|
19
|
+
resolvedAt TEXT,
|
|
20
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
21
|
+
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
22
|
+
created_by TEXT,
|
|
23
|
+
updated_by TEXT
|
|
24
|
+
);
|
|
25
|
+
CREATE INDEX IF NOT EXISTS idx_deferred_event_waits_pending
|
|
26
|
+
ON deferred_event_waits(source, event, matchKey) WHERE status = 'pending';
|