mixdog 0.9.3 → 0.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -38
- package/package.json +10 -3
- package/scripts/bench/lead-review-tasks-r3.json +20 -0
- package/scripts/bench/lead-review-tasks.json +20 -0
- package/scripts/bench/r4-mixed-tasks.json +20 -0
- package/scripts/bench/r5-orchestrated-task.json +7 -0
- package/scripts/bench/review-tasks.json +20 -0
- package/scripts/bench/round-codex.json +114 -0
- package/scripts/bench/round-mixdog-lead-r3.json +269 -0
- package/scripts/bench/round-mixdog-lead.json +269 -0
- package/scripts/bench/round-mixdog.json +126 -0
- package/scripts/bench/round-r10-bigsample.json +679 -0
- package/scripts/bench/round-r11-codexalign.json +257 -0
- package/scripts/bench/round-r13-clientmeta.json +464 -0
- package/scripts/bench/round-r14-betafeatures.json +466 -0
- package/scripts/bench/round-r15-fulldefault.json +462 -0
- package/scripts/bench/round-r16-sessionid.json +466 -0
- package/scripts/bench/round-r17-wirebytes.json +456 -0
- package/scripts/bench/round-r18-prewarm.json +468 -0
- package/scripts/bench/round-r19-clean.json +472 -0
- package/scripts/bench/round-r20-prewarm-clean.json +475 -0
- package/scripts/bench/round-r21-delta-retry.json +473 -0
- package/scripts/bench/round-r22-full-probe.json +693 -0
- package/scripts/bench/round-r23-itemprobe.json +701 -0
- package/scripts/bench/round-r24-shapefix.json +677 -0
- package/scripts/bench/round-r25-serial.json +464 -0
- package/scripts/bench/round-r26-parallel3.json +671 -0
- package/scripts/bench/round-r27-parallel10.json +894 -0
- package/scripts/bench/round-r28-parallel10-stagger.json +882 -0
- package/scripts/bench/round-r29-parallel10-stagger166.json +886 -0
- package/scripts/bench/round-r30-instid.json +253 -0
- package/scripts/bench/round-r31-upgradeprobe.json +256 -0
- package/scripts/bench/round-r32-vs-codex-lead.json +254 -0
- package/scripts/bench/round-r33-vs-codex-codex.json +115 -0
- package/scripts/bench/round-r34-orchestrated.json +120 -0
- package/scripts/bench/round-r35-orchestrated-codex.json +61 -0
- package/scripts/bench/round-r36-orchestrated-capped.json +128 -0
- package/scripts/bench/round-r4-codex.json +114 -0
- package/scripts/bench/round-r4-mixed.json +225 -0
- package/scripts/bench/round-r5-gpt-lead.json +259 -0
- package/scripts/bench/round-r6-codex.json +114 -0
- package/scripts/bench/round-r6-solo.json +257 -0
- package/scripts/bench/round-r7-full.json +254 -0
- package/scripts/bench/round-r8-fulldefault.json +255 -0
- package/scripts/bench-run.mjs +251 -32
- package/scripts/freevar-smoke.mjs +95 -0
- package/scripts/internal-comms-bench.mjs +3 -4
- package/scripts/internal-comms-smoke.mjs +10 -9
- package/scripts/model-catalog-audit.mjs +209 -0
- package/scripts/model-list-sanitize-test.mjs +37 -0
- package/scripts/mouse-probe.mjs +45 -0
- package/scripts/output-style-bench.mjs +13 -6
- package/scripts/output-style-smoke.mjs +4 -4
- package/scripts/provider-toolcall-test.mjs +7 -3
- package/scripts/recall-bench.mjs +76 -13
- package/scripts/recall-quality-cases.json +12 -0
- package/scripts/recall-usecase-cases.json +18 -0
- package/scripts/session-bench.mjs +152 -6
- package/scripts/tool-smoke.mjs +25 -65
- package/scripts/tui-render-smoke.mjs +90 -0
- package/scripts/webhook-smoke.mjs +208 -0
- package/src/agents/debugger/AGENT.md +4 -1
- package/src/agents/heavy-worker/AGENT.md +9 -8
- package/src/agents/maintainer/AGENT.md +4 -0
- package/src/agents/reviewer/AGENT.md +2 -1
- package/src/agents/scheduler-task/AGENT.md +2 -3
- package/src/agents/webhook-handler/AGENT.md +2 -3
- package/src/agents/worker/AGENT.md +10 -7
- package/src/app.mjs +12 -1
- package/src/headless-role.mjs +7 -1
- package/src/lib/rules-builder.cjs +4 -0
- package/src/mixdog-session-runtime.mjs +647 -2056
- package/src/output-styles/default.md +30 -9
- package/src/output-styles/{oneline.md → extreme-minimal.md} +5 -4
- package/src/output-styles/minimal.md +8 -6
- package/src/output-styles/simple.md +21 -7
- package/src/rules/agent/00-common.md +6 -3
- package/src/rules/agent/30-explorer.md +16 -5
- package/src/rules/lead/01-general.md +5 -5
- package/src/rules/lead/lead-brief.md +15 -0
- package/src/rules/lead/lead-tool.md +6 -15
- package/src/rules/shared/01-tool.md +17 -21
- package/src/runtime/agent/orchestrator/agent-runtime/agent-dispatch.mjs +8 -3
- package/src/runtime/agent/orchestrator/agent-runtime/agent-loop-policy.mjs +25 -0
- package/src/runtime/agent/orchestrator/agent-runtime/cache-strategy.mjs +100 -23
- package/src/runtime/agent/orchestrator/agent-runtime/session-builder.mjs +6 -15
- package/src/runtime/agent/orchestrator/agent-trace-format.mjs +362 -0
- package/src/runtime/agent/orchestrator/agent-trace-io.mjs +410 -0
- package/src/runtime/agent/orchestrator/agent-trace.mjs +16 -735
- package/src/runtime/agent/orchestrator/config.mjs +69 -2
- package/src/runtime/agent/orchestrator/providers/anthropic-effort.mjs +62 -20
- package/src/runtime/agent/orchestrator/providers/anthropic-model-resolve.mjs +209 -0
- package/src/runtime/agent/orchestrator/providers/anthropic-oauth-credentials.mjs +489 -0
- package/src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs +81 -1281
- package/src/runtime/agent/orchestrator/providers/anthropic-sse.mjs +607 -0
- package/src/runtime/agent/orchestrator/providers/anthropic.mjs +32 -3
- package/src/runtime/agent/orchestrator/providers/codex-client-meta.mjs +81 -0
- package/src/runtime/agent/orchestrator/providers/gemini-cache.mjs +248 -0
- package/src/runtime/agent/orchestrator/providers/gemini-schema.mjs +303 -0
- package/src/runtime/agent/orchestrator/providers/gemini-stream.mjs +505 -0
- package/src/runtime/agent/orchestrator/providers/gemini.mjs +43 -1013
- package/src/runtime/agent/orchestrator/providers/grok-oauth.mjs +17 -3
- package/src/runtime/agent/orchestrator/providers/model-catalog.mjs +105 -11
- package/src/runtime/agent/orchestrator/providers/model-list-sanitize.mjs +356 -0
- package/src/runtime/agent/orchestrator/providers/openai-codex-model.mjs +108 -0
- package/src/runtime/agent/orchestrator/providers/openai-compat-trace.mjs +58 -0
- package/src/runtime/agent/orchestrator/providers/openai-compat-wire.mjs +368 -0
- package/src/runtime/agent/orchestrator/providers/openai-compat-xai.mjs +760 -0
- package/src/runtime/agent/orchestrator/providers/openai-compat.mjs +40 -1143
- package/src/runtime/agent/orchestrator/providers/openai-oauth-http-sse.mjs +740 -0
- package/src/runtime/agent/orchestrator/providers/openai-oauth-login.mjs +193 -0
- package/src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs +349 -2131
- package/src/runtime/agent/orchestrator/providers/openai-oauth.mjs +143 -1002
- package/src/runtime/agent/orchestrator/providers/openai-ws-delta.mjs +229 -0
- package/src/runtime/agent/orchestrator/providers/openai-ws-events.mjs +67 -0
- package/src/runtime/agent/orchestrator/providers/openai-ws-pool.mjs +465 -0
- package/src/runtime/agent/orchestrator/providers/openai-ws-stream.mjs +1105 -0
- package/src/runtime/agent/orchestrator/providers/openai-ws.mjs +2 -1
- package/src/runtime/agent/orchestrator/providers/provider-catalog-cache.mjs +80 -0
- package/src/runtime/agent/orchestrator/session/compact/budget.mjs +288 -0
- package/src/runtime/agent/orchestrator/session/compact/constants.mjs +85 -0
- package/src/runtime/agent/orchestrator/session/compact/engine.mjs +749 -0
- package/src/runtime/agent/orchestrator/session/compact/messages.mjs +82 -0
- package/src/runtime/agent/orchestrator/session/compact/summary-schema.mjs +315 -0
- package/src/runtime/agent/orchestrator/session/compact/summary.mjs +643 -0
- package/src/runtime/agent/orchestrator/session/compact/text-utils.mjs +326 -0
- package/src/runtime/agent/orchestrator/session/compact.mjs +40 -2282
- package/src/runtime/agent/orchestrator/session/loop/compact-policy.mjs +14 -2
- package/src/runtime/agent/orchestrator/session/loop/completion-guards.mjs +61 -0
- package/src/runtime/agent/orchestrator/session/loop/pre-dispatch-deny.mjs +1 -3
- package/src/runtime/agent/orchestrator/session/loop/recall-fasttrack.mjs +275 -0
- package/src/runtime/agent/orchestrator/session/loop/steering-ladder.mjs +173 -0
- package/src/runtime/agent/orchestrator/session/loop/termination.mjs +58 -0
- package/src/runtime/agent/orchestrator/session/loop/tool-exec.mjs +239 -0
- package/src/runtime/agent/orchestrator/session/loop.mjs +278 -402
- package/src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs +471 -0
- package/src/runtime/agent/orchestrator/session/manager/context-meta.mjs +7 -4
- package/src/runtime/agent/orchestrator/session/manager/prompt-utils.mjs +12 -0
- package/src/runtime/agent/orchestrator/session/manager/runtime-liveness.mjs +406 -0
- package/src/runtime/agent/orchestrator/session/manager/status-telemetry.mjs +80 -0
- package/src/runtime/agent/orchestrator/session/manager/usage-metrics.mjs +210 -0
- package/src/runtime/agent/orchestrator/session/manager.mjs +166 -1087
- package/src/runtime/agent/orchestrator/session/store-summary-index.mjs +189 -0
- package/src/runtime/agent/orchestrator/session/store.mjs +74 -179
- package/src/runtime/agent/orchestrator/stall-policy.mjs +20 -1
- package/src/runtime/agent/orchestrator/tools/builtin/arg-guard.mjs +70 -20
- package/src/runtime/agent/orchestrator/tools/builtin/bash-tool.mjs +22 -2
- package/src/runtime/agent/orchestrator/tools/builtin/builtin-tools.mjs +35 -44
- package/src/runtime/agent/orchestrator/tools/builtin/list-tool.mjs +40 -0
- package/src/runtime/agent/orchestrator/tools/builtin/rg-runner.mjs +29 -0
- package/src/runtime/agent/orchestrator/tools/builtin/search-builders.mjs +8 -0
- package/src/runtime/agent/orchestrator/tools/builtin/search-path-diagnostics.mjs +126 -0
- package/src/runtime/agent/orchestrator/tools/builtin/search-tool.mjs +81 -92
- package/src/runtime/agent/orchestrator/tools/builtin/shell-job-paths.mjs +161 -0
- package/src/runtime/agent/orchestrator/tools/builtin/shell-job-process.mjs +108 -0
- package/src/runtime/agent/orchestrator/tools/builtin/shell-jobs.mjs +28 -265
- package/src/runtime/agent/orchestrator/tools/builtin.mjs +0 -6
- package/src/runtime/agent/orchestrator/tools/code-graph/dispatch.mjs +57 -3
- package/src/runtime/agent/orchestrator/tools/code-graph/keyword-match.mjs +82 -0
- package/src/runtime/agent/orchestrator/tools/code-graph/search.mjs +10 -122
- package/src/runtime/agent/orchestrator/tools/code-graph/text-columns.mjs +45 -0
- package/src/runtime/agent/orchestrator/tools/code-graph-tool-defs.mjs +6 -6
- package/src/runtime/agent/orchestrator/tools/graph-binary-fetcher.mjs +6 -3
- package/src/runtime/agent/orchestrator/tools/patch/constants.mjs +9 -0
- package/src/runtime/agent/orchestrator/tools/patch/dispatch.mjs +171 -0
- package/src/runtime/agent/orchestrator/tools/patch/matcher.mjs +471 -0
- package/src/runtime/agent/orchestrator/tools/patch/native-server.mjs +436 -0
- package/src/runtime/agent/orchestrator/tools/patch/orchestrator.mjs +342 -0
- package/src/runtime/agent/orchestrator/tools/patch/parsing.mjs +359 -0
- package/src/runtime/agent/orchestrator/tools/patch/paths.mjs +340 -0
- package/src/runtime/agent/orchestrator/tools/patch/v4a-convert.mjs +643 -0
- package/src/runtime/agent/orchestrator/tools/patch.mjs +36 -2959
- package/src/runtime/agent/orchestrator/tools/progress-message.mjs +0 -21
- package/src/runtime/agent/orchestrator/tools/shell-command.mjs +9 -72
- package/src/runtime/agent/orchestrator/tools/shell-powershell.mjs +77 -0
- package/src/runtime/agent/orchestrator/tools/shell-state.mjs +154 -0
- package/src/runtime/channels/backends/discord-access.mjs +32 -0
- package/src/runtime/channels/backends/discord-attachments.mjs +65 -0
- package/src/runtime/channels/backends/discord-gateway.mjs +233 -0
- package/src/runtime/channels/backends/discord.mjs +27 -318
- package/src/runtime/channels/backends/telegram.mjs +8 -12
- package/src/runtime/channels/index.mjs +247 -701
- package/src/runtime/channels/lib/backend-dispatch.mjs +46 -0
- package/src/runtime/channels/lib/config.mjs +37 -149
- package/src/runtime/channels/lib/event-pipeline.mjs +22 -5
- package/src/runtime/channels/lib/event-queue.mjs +78 -13
- package/src/runtime/channels/lib/inbound-routing.mjs +74 -0
- package/src/runtime/channels/lib/interaction-workflows.mjs +5 -113
- package/src/runtime/channels/lib/output-forwarder.mjs +1 -1
- package/src/runtime/channels/lib/owner-heartbeat.mjs +75 -0
- package/src/runtime/channels/lib/parent-bridge.mjs +88 -0
- package/src/runtime/channels/lib/runtime-paths.mjs +14 -4
- package/src/runtime/channels/lib/scheduler.mjs +27 -113
- package/src/runtime/channels/lib/session-discovery.mjs +56 -4
- package/src/runtime/channels/lib/tool-dispatch.mjs +158 -0
- package/src/runtime/channels/lib/tool-format.mjs +1 -1
- package/src/runtime/channels/lib/transcript-discovery.mjs +4 -4
- package/src/runtime/channels/lib/voice-runtime-fetcher.mjs +6 -3
- package/src/runtime/channels/lib/voice-transcription.mjs +179 -0
- package/src/runtime/channels/lib/webhook/deliveries.mjs +313 -0
- package/src/runtime/channels/lib/webhook/log.mjs +42 -0
- package/src/runtime/channels/lib/webhook/ngrok.mjs +181 -0
- package/src/runtime/channels/lib/webhook/signature.mjs +60 -0
- package/src/runtime/channels/lib/webhook.mjs +43 -616
- package/src/runtime/channels/tool-defs.mjs +11 -130
- package/src/runtime/memory/index.mjs +210 -1948
- package/src/runtime/memory/lib/core-memory-store.mjs +5 -1
- package/src/runtime/memory/lib/cycle-llm-adapters.mjs +58 -0
- package/src/runtime/memory/lib/cycle-scheduler.mjs +497 -0
- package/src/runtime/memory/lib/embedding-warmup.mjs +58 -0
- package/src/runtime/memory/lib/ko-morph.mjs +195 -0
- package/src/runtime/memory/lib/memory-config-flags.mjs +91 -0
- package/src/runtime/memory/lib/memory-cycle.mjs +1 -1
- package/src/runtime/memory/lib/memory-cycle2-gate.mjs +515 -0
- package/src/runtime/memory/lib/memory-cycle2-mutations.mjs +324 -0
- package/src/runtime/memory/lib/memory-cycle2-shared.mjs +18 -0
- package/src/runtime/memory/lib/memory-cycle2.mjs +24 -842
- package/src/runtime/memory/lib/memory-embed.mjs +149 -0
- package/src/runtime/memory/lib/memory-process-lock.mjs +162 -0
- package/src/runtime/memory/lib/memory-recall-store.mjs +69 -12
- package/src/runtime/memory/lib/memory-text-utils.mjs +46 -0
- package/src/runtime/memory/lib/pg/supervisor.mjs +1 -1
- package/src/runtime/memory/lib/query-handlers.mjs +802 -0
- package/src/runtime/memory/lib/recall-format.mjs +55 -0
- package/src/runtime/memory/lib/runtime-fetcher.mjs +8 -3
- package/src/runtime/memory/lib/transcript-ingest.mjs +425 -0
- package/src/runtime/memory/tool-defs.mjs +5 -13
- package/src/runtime/search/lib/http-fetch.mjs +274 -0
- package/src/runtime/search/lib/ssrf-guard.mjs +333 -0
- package/src/runtime/search/lib/web-tools.mjs +24 -602
- package/src/runtime/shared/atomic-file.mjs +26 -1
- package/src/runtime/shared/config.mjs +14 -4
- package/src/runtime/shared/launcher-control.mjs +2 -2
- package/src/runtime/shared/markdown-frontmatter.mjs +19 -0
- package/src/runtime/shared/schedules-store.mjs +13 -3
- package/src/runtime/shared/tool-execution-contract.mjs +2 -2
- package/src/runtime/shared/tool-primitives.mjs +308 -0
- package/src/runtime/shared/tool-result-summary.mjs +515 -0
- package/src/runtime/shared/tool-surface.mjs +80 -898
- package/src/runtime/shared/transcript-writer.mjs +23 -0
- package/src/runtime/shared/update-checker.mjs +7 -4
- package/src/session-runtime/config-helpers.mjs +119 -2
- package/src/session-runtime/config-lifecycle.mjs +232 -0
- package/src/session-runtime/cwd-plugins.mjs +226 -0
- package/src/session-runtime/mcp-glue.mjs +177 -0
- package/src/session-runtime/model-recency.mjs +111 -0
- package/src/session-runtime/native-search.mjs +247 -0
- package/src/session-runtime/output-styles.mjs +11 -9
- package/src/session-runtime/prewarm.mjs +142 -0
- package/src/session-runtime/provider-models.mjs +278 -0
- package/src/session-runtime/provider-usage.mjs +120 -0
- package/src/session-runtime/quick-model-rows.mjs +205 -0
- package/src/session-runtime/quick-search-models.mjs +47 -0
- package/src/session-runtime/session-hooks.mjs +93 -0
- package/src/session-runtime/settings-api.mjs +352 -0
- package/src/session-runtime/tool-catalog.mjs +29 -29
- package/src/session-runtime/tool-defs.mjs +84 -0
- package/src/session-runtime/warmup-schedulers.mjs +201 -0
- package/src/session-runtime/workflow.mjs +1 -1
- package/src/standalone/agent-tool/helpers.mjs +237 -0
- package/src/standalone/agent-tool/notify.mjs +107 -0
- package/src/standalone/agent-tool/provider-init.mjs +143 -0
- package/src/standalone/agent-tool/render.mjs +152 -0
- package/src/standalone/agent-tool/tool-def.mjs +55 -0
- package/src/standalone/agent-tool.mjs +138 -669
- package/src/standalone/channel-admin.mjs +102 -90
- package/src/standalone/channel-worker.mjs +4 -7
- package/src/standalone/explore-tool.mjs +64 -14
- package/src/standalone/hook-bus/config.mjs +207 -0
- package/src/standalone/hook-bus/constants.mjs +90 -0
- package/src/standalone/hook-bus/handlers.mjs +481 -0
- package/src/standalone/hook-bus/payload.mjs +31 -0
- package/src/standalone/hook-bus/rules.mjs +77 -0
- package/src/standalone/hook-bus.mjs +77 -870
- package/src/standalone/memory-runtime-proxy.mjs +7 -0
- package/src/standalone/opencode-go-login.mjs +5 -1
- package/src/standalone/provider-admin.mjs +1 -16
- package/src/standalone/usage-dashboard.mjs +3 -1
- package/src/tui/App.jsx +1059 -8110
- package/src/tui/app/app-format.mjs +213 -0
- package/src/tui/app/channel-pickers.mjs +508 -0
- package/src/tui/app/clipboard.mjs +67 -0
- package/src/tui/app/core-memory-picker.mjs +210 -0
- package/src/tui/app/extension-pickers.mjs +506 -0
- package/src/tui/app/input-parsers.mjs +193 -0
- package/src/tui/app/maintenance-pickers.mjs +356 -0
- package/src/tui/app/model-options.mjs +334 -0
- package/src/tui/app/model-picker.mjs +365 -0
- package/src/tui/app/onboarding-steps.mjs +400 -0
- package/src/tui/app/project-picker.mjs +247 -0
- package/src/tui/app/provider-setup-picker.mjs +580 -0
- package/src/tui/app/resume-picker.mjs +55 -0
- package/src/tui/app/route-pickers.mjs +419 -0
- package/src/tui/app/settings-picker.mjs +489 -0
- package/src/tui/app/slash-commands.mjs +101 -0
- package/src/tui/app/slash-dispatch.mjs +427 -0
- package/src/tui/app/text-layout.mjs +46 -0
- package/src/tui/app/theme-effort-pickers.mjs +154 -0
- package/src/tui/app/transcript-window.mjs +677 -0
- package/src/tui/app/use-mouse-input.mjs +460 -0
- package/src/tui/app/use-prompt-handlers.mjs +310 -0
- package/src/tui/app/use-transcript-scroll.mjs +512 -0
- package/src/tui/app/use-transcript-window.mjs +607 -0
- package/src/tui/components/ConfirmBar.jsx +10 -7
- package/src/tui/components/Picker.jsx +64 -15
- package/src/tui/components/PromptInput.jsx +33 -102
- package/src/tui/components/SlashCommandPalette.jsx +8 -1
- package/src/tui/components/StatusLine.jsx +69 -15
- package/src/tui/components/TextEntryPanel.jsx +11 -0
- package/src/tui/components/ToolExecution.jsx +52 -594
- package/src/tui/components/TranscriptItem.jsx +105 -0
- package/src/tui/components/UsagePanel.jsx +18 -4
- package/src/tui/components/prompt-input/edit-helpers.mjs +72 -0
- package/src/tui/components/prompt-input/voice-indicator.mjs +39 -0
- package/src/tui/components/tool-execution/ResultBody.jsx +56 -0
- package/src/tui/components/tool-execution/surface-detail.mjs +405 -0
- package/src/tui/components/tool-execution/text-format.mjs +161 -0
- package/src/tui/display-width.mjs +20 -3
- package/src/tui/dist/index.mjs +13553 -12384
- package/src/tui/engine/agent-job-feed.mjs +133 -0
- package/src/tui/engine/notification-plan.mjs +76 -0
- package/src/tui/engine/render-timing.mjs +17 -0
- package/src/tui/engine/tool-approval.mjs +94 -0
- package/src/tui/engine/tool-card-results.mjs +234 -0
- package/src/tui/engine/tool-result-status.mjs +135 -0
- package/src/tui/engine.mjs +170 -574
- package/src/tui/figures.mjs +5 -0
- package/src/tui/index.jsx +65 -1
- package/src/tui/input-editing.mjs +2 -2
- package/src/tui/markdown/format-token.mjs +4 -1
- package/src/tui/statusline-ansi-bridge.mjs +11 -3
- package/src/tui/theme.mjs +6 -0
- package/src/ui/statusline-agents.mjs +213 -0
- package/src/ui/statusline-format.mjs +146 -0
- package/src/ui/statusline-segments.mjs +148 -0
- package/src/ui/statusline.mjs +77 -501
- package/src/ui/tool-card.mjs +0 -1
- package/src/vendor/statusline/bin/statusline-route.mjs +15 -2
- package/src/workflows/default/WORKFLOW.md +16 -18
- package/src/workflows/sequential/WORKFLOW.md +16 -18
- package/vendor/ink/build/display-width.js +19 -3
- package/vendor/ink/build/ink.js +112 -7
- package/vendor/ink/build/log-update.js +17 -3
- package/vendor/ink/build/wrap-text.js +125 -0
- package/scripts/_test-folder-dialog.mjs +0 -30
- package/scripts/fix-brief-fn.mjs +0 -35
- package/scripts/fix-format-tool-surface.mjs +0 -24
- package/scripts/fix-tool-exec-visible.mjs +0 -42
- package/scripts/patch-agent-brief.mjs +0 -48
- package/scripts/patch-app.mjs +0 -21
- package/scripts/patch-app2.mjs +0 -18
- package/scripts/patch-dist-brief.mjs +0 -96
- package/scripts/patch-tool-exec.mjs +0 -70
- package/src/examples/schedules/SCHEDULE.example.md +0 -32
- package/src/examples/webhooks/WEBHOOK.example.md +0 -40
- package/src/runtime/agent/orchestrator/session/manager.reactive-persist.test.mjs +0 -107
- package/src/runtime/agent/orchestrator/tools/builtin/arg-guard.test.mjs +0 -143
- package/src/runtime/agent/orchestrator/tools/builtin/diagnostics-tool.mjs +0 -285
- package/src/runtime/agent/orchestrator/tools/builtin/external-tool-adapters.test.mjs +0 -162
- package/src/runtime/agent/orchestrator/tools/builtin/open-config-tool.mjs +0 -26
- package/src/runtime/channels/lib/holidays.mjs +0 -138
- package/src/runtime/shared/channel-notification-routing.test.mjs +0 -45
- package/src/runtime/shared/task-notification-envelope.test.mjs +0 -107
- package/src/runtime/shared/tool-execution-contract.test.mjs +0 -183
- package/src/standalone/agent-task-status.test.mjs +0 -76
- package/src/tui/components/tool-output-format.test.mjs +0 -399
- package/src/tui/display-width.test.mjs +0 -35
- package/src/tui/engine-runtime-notification.test.mjs +0 -115
- package/src/tui/engine-tool-result-text.test.mjs +0 -75
- package/src/tui/input-editing.selection.test.mjs +0 -75
- package/src/tui/markdown/format-token.test.mjs +0 -354
- package/src/tui/markdown/render-ansi.test.mjs +0 -108
- package/src/tui/markdown/stream-fence.test.mjs +0 -26
- package/src/tui/markdown/streaming-markdown.test.mjs +0 -70
- package/src/tui/paste-fix.test.mjs +0 -119
- package/src/tui/prompt-history-store.test.mjs +0 -52
- package/src/tui/statusline-ansi-bridge.test.mjs +0 -159
- package/src/tui/transcript-tool-failures.test.mjs +0 -111
- package/src/ui/markdown.test.mjs +0 -70
- package/src/ui/statusline-context-label.test.mjs +0 -15
- package/src/vendor/statusline/bin/statusline-lib.mjs +0 -186
- package/src/vendor/statusline/bin/statusline-route.test.mjs +0 -80
|
@@ -56,12 +56,11 @@ import {
|
|
|
56
56
|
} from './lib/session-ingest.mjs'
|
|
57
57
|
import { configureEmbedding, embedText, embedTexts, getEmbeddingDims, getEmbeddingDtype, getEmbeddingModelId, getKnownDimsForCurrentModel, isEmbeddingModelReady, primeEmbeddingDims, warmupEmbeddingProvider } from './lib/embedding-provider.mjs'
|
|
58
58
|
import { startLlmWorker, stopLlmWorker } from './lib/llm-worker-host.mjs'
|
|
59
|
-
import { runCycle1, runCycle2, runCycle3, runUnifiedGate, parseInterval, syncRootEmbedding, applySimpleStatus, applyUpdate, applyMerge, CYCLE2_ACTIVE_TARGET_CAP } from './lib/memory-cycle.mjs'
|
|
59
|
+
import { runCycle1, runCycle2, runCycle3, runUnifiedGate, parseInterval, syncRootEmbedding, flushRawEmbeddings, applySimpleStatus, applyUpdate, applyMerge, CYCLE2_ACTIVE_TARGET_CAP } from './lib/memory-cycle.mjs'
|
|
60
60
|
import { loadConfig as loadAgentConfig } from '../agent/orchestrator/config.mjs'
|
|
61
61
|
import { initProviders } from '../agent/orchestrator/providers/registry.mjs'
|
|
62
62
|
import { makeAgentDispatch } from '../agent/orchestrator/agent-runtime/agent-dispatch.mjs'
|
|
63
63
|
import { getInFlightCycle1 } from './lib/memory-cycle1.mjs'
|
|
64
|
-
import { drainSessionCycle1 } from '../agent/orchestrator/session/compact.mjs'
|
|
65
64
|
import { claimAndMarkScheduledCycle, resolveCoalesceMaxRetries, scheduleCoalescedCycleRetry } from './lib/memory-cycle-requests.mjs'
|
|
66
65
|
import { searchRelevantHybrid } from './lib/memory-recall-store.mjs'
|
|
67
66
|
import { fetchEntriesByIdsScoped } from './lib/memory-recall-id-patch.mjs'
|
|
@@ -74,9 +73,27 @@ import { resolveProjectId, resolveProjectScope } from './lib/project-id-resolver
|
|
|
74
73
|
import { openTraceDatabase, closeTraceDatabase, insertTraceEvents, enqueueTraceEvents, insertAgentCalls, registerTraceExitDrain } from './lib/trace-store.mjs'
|
|
75
74
|
import { updateJsonAtomicSync, writeJsonAtomicSync } from '../shared/atomic-file.mjs'
|
|
76
75
|
import { resolvePluginData, mixdogHome } from '../shared/plugin-paths.mjs'
|
|
77
|
-
import { parsePeriod, formatTs, coreRecallTerms, normalizeRecallProjectScope, sessionRecallTerms, interleaveRawRows, renderEntryLines } from './lib/recall-format.mjs'
|
|
76
|
+
import { parsePeriod, formatTs, coreRecallTerms, normalizeRecallProjectScope, sessionRecallTerms, interleaveRawRows, renderEntryLines, renderSessionGroupedLines } from './lib/recall-format.mjs'
|
|
78
77
|
import { readBody, sendJson, sendError, isLocalOrigin, normalizeCoreProjectId } from './lib/http-wire.mjs'
|
|
79
78
|
import { scheduledCycle1Signature, scheduledCycle2Signature, scheduledCycle3Signature } from './lib/cycle-signatures.mjs'
|
|
79
|
+
import { createTranscriptIngest } from './lib/transcript-ingest.mjs'
|
|
80
|
+
import { createEmbeddingWarmup } from './lib/embedding-warmup.mjs'
|
|
81
|
+
import { init as initKoMorph } from './lib/ko-morph.mjs'
|
|
82
|
+
import { createCycleLlmAdapters } from './lib/cycle-llm-adapters.mjs'
|
|
83
|
+
import { createCycleScheduler } from './lib/cycle-scheduler.mjs'
|
|
84
|
+
import { createQueryHandlers } from './lib/query-handlers.mjs'
|
|
85
|
+
import {
|
|
86
|
+
readMainConfig,
|
|
87
|
+
readRecapEnabled,
|
|
88
|
+
embeddingWarmupEnabled,
|
|
89
|
+
envFlagEnabled,
|
|
90
|
+
memorySecondaryMode,
|
|
91
|
+
embeddingWarmupCanStart,
|
|
92
|
+
memoryLlmWorkerEnabled,
|
|
93
|
+
memoryCyclesEnabled,
|
|
94
|
+
secondaryPgAdvertised as _secondaryPgAdvertised,
|
|
95
|
+
assertSecondaryPgAttachable as _assertSecondaryPgAttachable,
|
|
96
|
+
} from './lib/memory-config-flags.mjs'
|
|
80
97
|
const IS_MEMORY_ENTRY = !!process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href
|
|
81
98
|
const USE_ARG_DATA_DIR = IS_MEMORY_ENTRY || process.env.MIXDOG_WORKER_MODE === '1'
|
|
82
99
|
const DATA_DIR = process.env.MIXDOG_DATA_DIR || (USE_ARG_DATA_DIR ? process.argv[2] : '') || resolvePluginData()
|
|
@@ -86,7 +103,15 @@ if (!DATA_DIR) {
|
|
|
86
103
|
}
|
|
87
104
|
__mixdogMemoryLog(`[memory-service] DATA_DIR=${DATA_DIR}\n`)
|
|
88
105
|
|
|
89
|
-
import {
|
|
106
|
+
import {
|
|
107
|
+
parsePositivePid,
|
|
108
|
+
isPidAliveLocal,
|
|
109
|
+
tryAcquireMemoryOwnerLock as _tryAcquireMemoryOwnerLock,
|
|
110
|
+
releaseMemoryOwnerLock as _releaseMemoryOwnerLock,
|
|
111
|
+
killPreviousServer as _killPreviousServer,
|
|
112
|
+
acquireLock as _acquireLock,
|
|
113
|
+
releaseLock as _releaseLock,
|
|
114
|
+
} from './lib/memory-process-lock.mjs'
|
|
90
115
|
|
|
91
116
|
const RUNTIME_ROOT = process.env.MIXDOG_RUNTIME_ROOT
|
|
92
117
|
? path.resolve(process.env.MIXDOG_RUNTIME_ROOT)
|
|
@@ -100,12 +125,8 @@ let _periodicAdvertiseTimer = null
|
|
|
100
125
|
// port after fork-proxy promotion swaps in our own locally-bound port.
|
|
101
126
|
let _currentAdvertisedPort = null
|
|
102
127
|
|
|
103
|
-
function parsePositivePid(value) {
|
|
104
|
-
const pid = Number(value)
|
|
105
|
-
return Number.isFinite(pid) && pid > 0 ? pid : null
|
|
106
|
-
}
|
|
107
|
-
|
|
108
128
|
const MEMORY_SERVER_PID = parsePositivePid(process.env.MIXDOG_SERVER_PID) ?? process.pid
|
|
129
|
+
const _isPidAliveLocal = isPidAliveLocal
|
|
109
130
|
const MEMORY_DAEMON_MODE = process.env.MIXDOG_MEMORY_DAEMON === '1'
|
|
110
131
|
const MEMORY_IDLE_TTL_MS = Math.max(0, Number(process.env.MIXDOG_MEMORY_IDLE_TTL_MS) || 10 * 60_000)
|
|
111
132
|
let _idleShutdownTimer = null
|
|
@@ -155,7 +176,7 @@ function advertiseMemoryPort(boundPort, attempt = 0) {
|
|
|
155
176
|
updatedAt: Date.now(),
|
|
156
177
|
}
|
|
157
178
|
return next
|
|
158
|
-
}, { compact: true, fsyncDir: true })
|
|
179
|
+
}, { compact: true, fsyncDir: true, renameFallback: 'truncate' })
|
|
159
180
|
if (!_periodicAdvertiseInstalled) {
|
|
160
181
|
_periodicAdvertiseInstalled = true
|
|
161
182
|
_periodicAdvertiseTimer = setInterval(() => {
|
|
@@ -185,58 +206,12 @@ const LOCK_FILE = path.join(DATA_DIR, '.memory-service.lock')
|
|
|
185
206
|
// EEXIST when another process won the race.
|
|
186
207
|
const OWNER_LOCK_FILE = path.join(DATA_DIR, '.memory-owner.lock')
|
|
187
208
|
|
|
188
|
-
function _isPidAliveLocal(pid) {
|
|
189
|
-
if (!Number.isFinite(pid) || pid <= 0) return false
|
|
190
|
-
try { process.kill(pid, 0); return true }
|
|
191
|
-
catch (e) { return e.code !== 'ESRCH' }
|
|
192
|
-
}
|
|
193
|
-
|
|
194
209
|
function tryAcquireMemoryOwnerLock() {
|
|
195
|
-
|
|
196
|
-
// dir), false when a live peer holds the lock. Stale locks (dead PID) are
|
|
197
|
-
// unlinked and retried atomically. Throws on unexpected fs errors so callers
|
|
198
|
-
// surface lock-system corruption rather than silently downgrading.
|
|
199
|
-
//
|
|
200
|
-
// EPERM/EBUSY/EACCES at openSync are transient — AV scanners (SignKorea /
|
|
201
|
-
// SKCert / ezPDFWS etc) briefly lock newly-created files during inspection.
|
|
202
|
-
// The 0.1.x baseline threw immediately and the worker promoted to
|
|
203
|
-
// permanentlyDegraded, killing memory tools for the rest of the session.
|
|
204
|
-
// Treat the AV error codes as retryable with bounded backoff (~750ms total)
|
|
205
|
-
// before giving up and rethrowing.
|
|
206
|
-
for (let attempt = 0; attempt < 5; attempt++) {
|
|
207
|
-
try {
|
|
208
|
-
const fd = fs.openSync(OWNER_LOCK_FILE, 'wx')
|
|
209
|
-
fs.writeSync(fd, String(process.pid))
|
|
210
|
-
fs.closeSync(fd)
|
|
211
|
-
return true
|
|
212
|
-
} catch (e) {
|
|
213
|
-
if (e.code === 'EEXIST') {
|
|
214
|
-
let ownerPid = NaN
|
|
215
|
-
try { ownerPid = Number(fs.readFileSync(OWNER_LOCK_FILE, 'utf8').trim()) } catch {}
|
|
216
|
-
if (_isPidAliveLocal(ownerPid)) return false
|
|
217
|
-
// Stale lock: dead owner — unlink and retry exclusive create.
|
|
218
|
-
try { fs.unlinkSync(OWNER_LOCK_FILE) } catch {}
|
|
219
|
-
continue
|
|
220
|
-
}
|
|
221
|
-
const transient = e.code === 'EPERM' || e.code === 'EBUSY' || e.code === 'EACCES'
|
|
222
|
-
if (transient && attempt < 4) {
|
|
223
|
-
// Sync busy-wait acceptable here: this runs on memory worker boot
|
|
224
|
-
// path, once per process; the parent handler is not blocked.
|
|
225
|
-
const end = Date.now() + 50 * (attempt + 1)
|
|
226
|
-
while (Date.now() < end) {}
|
|
227
|
-
continue
|
|
228
|
-
}
|
|
229
|
-
throw e
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
return false
|
|
210
|
+
return _tryAcquireMemoryOwnerLock(OWNER_LOCK_FILE, __mixdogMemoryLog)
|
|
233
211
|
}
|
|
234
212
|
|
|
235
213
|
function releaseMemoryOwnerLock() {
|
|
236
|
-
|
|
237
|
-
const ownerPid = Number(fs.readFileSync(OWNER_LOCK_FILE, 'utf8').trim())
|
|
238
|
-
if (ownerPid === process.pid) fs.unlinkSync(OWNER_LOCK_FILE)
|
|
239
|
-
} catch {}
|
|
214
|
+
return _releaseMemoryOwnerLock(OWNER_LOCK_FILE)
|
|
240
215
|
}
|
|
241
216
|
|
|
242
217
|
const BASE_PORT = 3350
|
|
@@ -247,139 +222,39 @@ let _traceDb = null
|
|
|
247
222
|
const MEMORY_INSTRUCTIONS_TEXT = ''
|
|
248
223
|
|
|
249
224
|
function killPreviousServer(pid) {
|
|
250
|
-
|
|
251
|
-
if (process.platform === 'win32') {
|
|
252
|
-
try {
|
|
253
|
-
execFileSync('taskkill', ['/F', '/T', '/PID', String(pid)], { encoding: 'utf8', timeout: 5000, windowsHide: true })
|
|
254
|
-
__mixdogMemoryLog(`[memory-service] Killed previous server PID ${pid}\n`)
|
|
255
|
-
return true
|
|
256
|
-
} catch (e) {
|
|
257
|
-
// Exit code 128 = process not found; treat stale lock as already-dead = success.
|
|
258
|
-
// Status 128 reliably means "process not found" regardless of locale; no text match needed.
|
|
259
|
-
// Status 1 with English text match handles edge cases on some Windows versions.
|
|
260
|
-
const notFoundText = /not found|no running instance/i.test(e.stdout || '')
|
|
261
|
-
|| /not found|no running instance/i.test(e.stderr || '')
|
|
262
|
-
|| /not found|no running instance/i.test(e.message || '')
|
|
263
|
-
const alreadyDead = e.status === 128 || (e.status === 1 && notFoundText)
|
|
264
|
-
if (alreadyDead) {
|
|
265
|
-
__mixdogMemoryLog(`[memory-service] PID ${pid} already dead (stale lock), proceeding\n`)
|
|
266
|
-
return true
|
|
267
|
-
}
|
|
268
|
-
__mixdogMemoryLog(`[memory-service] taskkill failed for PID ${pid}: ${e.message}\n`)
|
|
269
|
-
return false
|
|
270
|
-
}
|
|
271
|
-
} else {
|
|
272
|
-
// Pre-flight: if the process is already gone, treat stale lock as success.
|
|
273
|
-
try {
|
|
274
|
-
process.kill(pid, 0)
|
|
275
|
-
} catch (e) {
|
|
276
|
-
if (e.code === 'ESRCH') {
|
|
277
|
-
__mixdogMemoryLog(`[memory-service] PID ${pid} already dead (stale lock), proceeding\n`)
|
|
278
|
-
return true
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
try { process.kill(pid, 'SIGTERM') } catch {}
|
|
282
|
-
try { process.kill(pid, 'SIGKILL') } catch {}
|
|
283
|
-
// Poll for death up to 2s
|
|
284
|
-
const deadline = Date.now() + 2000
|
|
285
|
-
while (Date.now() < deadline) {
|
|
286
|
-
try {
|
|
287
|
-
process.kill(pid, 0)
|
|
288
|
-
} catch (e) {
|
|
289
|
-
if (e.code === 'ESRCH') {
|
|
290
|
-
__mixdogMemoryLog(`[memory-service] Killed previous server PID ${pid}\n`)
|
|
291
|
-
return true
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
// Synchronous 50ms sleep via shared buffer spin
|
|
295
|
-
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50)
|
|
296
|
-
}
|
|
297
|
-
__mixdogMemoryLog(`[memory-service] PID ${pid} still alive after SIGKILL\n`)
|
|
298
|
-
return false
|
|
299
|
-
}
|
|
225
|
+
return _killPreviousServer(pid, __mixdogMemoryLog)
|
|
300
226
|
}
|
|
301
227
|
|
|
302
228
|
function acquireLock() {
|
|
303
|
-
|
|
304
|
-
// memory worker serving recall for another CC session. killPreviousServer
|
|
305
|
-
// would taskkill /F that healthy peer mid-flight, then this fork-proxy
|
|
306
|
-
// mode wouldn't even need a lock anyway. Skip the entire kill-the-previous
|
|
307
|
-
// protocol; fork-proxy detection in init() takes priority. If neither
|
|
308
|
-
// proxy nor lock-owner path applies (race window during simultaneous
|
|
309
|
-
// boot), the worker simply continues without the lock — server-main /
|
|
310
|
-
// PG / port-listen handle the actual conflict cases.
|
|
311
|
-
if (process.env.MIXDOG_MULTI_INSTANCE === '1') return
|
|
312
|
-
try {
|
|
313
|
-
if (fs.existsSync(LOCK_FILE)) {
|
|
314
|
-
const lockedPid = Number(fs.readFileSync(LOCK_FILE, 'utf8').trim())
|
|
315
|
-
if (lockedPid > 0 && lockedPid !== process.pid) {
|
|
316
|
-
const killed = killPreviousServer(lockedPid)
|
|
317
|
-
if (!killed) {
|
|
318
|
-
__mixdogMemoryLog(`[memory-service] Could not kill previous server PID ${lockedPid}, aborting\n`)
|
|
319
|
-
process.exit(1)
|
|
320
|
-
}
|
|
321
|
-
try { fs.unlinkSync(LOCK_FILE) } catch {}
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
const fd = fs.openSync(LOCK_FILE, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, 0o600)
|
|
325
|
-
try {
|
|
326
|
-
fs.writeSync(fd, String(process.pid))
|
|
327
|
-
} finally {
|
|
328
|
-
fs.closeSync(fd)
|
|
329
|
-
}
|
|
330
|
-
} catch (e) {
|
|
331
|
-
if (e.code === 'EEXIST') {
|
|
332
|
-
__mixdogMemoryLog(`[memory-service] Lock file exists (EEXIST) — another instance is already running, exiting\n`)
|
|
333
|
-
process.exit(0)
|
|
334
|
-
}
|
|
335
|
-
__mixdogMemoryLog(`[memory-service] Lock acquisition failed: ${e.message}\n`)
|
|
336
|
-
process.exit(1)
|
|
337
|
-
}
|
|
229
|
+
return _acquireLock(LOCK_FILE, __mixdogMemoryLog)
|
|
338
230
|
}
|
|
339
231
|
|
|
340
232
|
function releaseLock() {
|
|
341
|
-
|
|
342
|
-
const content = fs.readFileSync(LOCK_FILE, 'utf8').trim()
|
|
343
|
-
if (Number(content) === process.pid) fs.unlinkSync(LOCK_FILE)
|
|
344
|
-
} catch {}
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
import { readSection } from '../shared/config.mjs'
|
|
348
|
-
|
|
349
|
-
function readMainConfig() {
|
|
350
|
-
return readSection('memory')
|
|
233
|
+
return _releaseLock(LOCK_FILE)
|
|
351
234
|
}
|
|
352
235
|
|
|
353
236
|
let db = null
|
|
354
237
|
let mainConfig = null
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
//
|
|
358
|
-
//
|
|
359
|
-
// The AUTHORITATIVE guard lives in memory-cycle.mjs:runCycle1 itself — that
|
|
360
|
-
// one catches every caller, including direct imports (setup-server backfill,
|
|
361
|
-
// policy-layer backfill). This outer tracker is kept as a defense-in-depth
|
|
362
|
-
// layer local to the MCP server process: it coalesces simultaneous
|
|
363
|
-
// _awaitCycle1Run callers (MCP action, scheduler, flush) onto a shared
|
|
364
|
-
// promise so they all observe the SAME result object rather than some
|
|
365
|
-
// getting the real stats and others getting `skippedInFlight: true` from
|
|
366
|
-
// the inner guard.
|
|
367
|
-
let _cycle1InFlight = null // shared cycle1 promise (outer coalesce layer)
|
|
368
|
-
// Per-session recall-drain in-flight map. Multiple terminals share one
|
|
369
|
-
// memory daemon (active-instance.json memory_port); concurrent recall calls
|
|
370
|
-
// for the SAME session must share one drain promise (dedup), while DIFFERENT
|
|
371
|
-
// sessions drain in parallel (cycle1's own advisory-lock/coalesce guard in
|
|
372
|
-
// memory-cycle1.mjs still serializes the actual DB work). A drain never
|
|
373
|
-
// blocks recall for sessions other than its own.
|
|
374
|
-
const _recallDrainInFlight = new Map() // sessionId -> Promise
|
|
238
|
+
// NOTE: cycle tick timers + the cycle1 outer-coalesce in-flight tracker now
|
|
239
|
+
// live inside the cycle scheduler factory (lib/cycle-scheduler.mjs). The
|
|
240
|
+
// AUTHORITATIVE cycle1 guard is still memory-cycle.mjs:runCycle1; the
|
|
241
|
+
// scheduler's outer layer coalesces simultaneous awaitCycle1Run callers.
|
|
375
242
|
let _initialized = false
|
|
376
243
|
let _initPromise = null
|
|
377
244
|
let _stopPromise = null
|
|
378
245
|
let _bootTimestamp = null
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
246
|
+
// Post-ingest raw-embedding flush chain (facade-local). The checkCycles tick
|
|
247
|
+
// keeps its OWN independent guard inside the scheduler; this chain serializes
|
|
248
|
+
// flushes kicked from ingestSessionMessages so ingest bursts never stack
|
|
249
|
+
// embedding work, while still guaranteeing every ingest's rows get their own
|
|
250
|
+
// flush pass (the old boolean guard silently SKIPPED bursts, leaving rows
|
|
251
|
+
// embedding-less until the ~60s tick — a recall (no results) window).
|
|
252
|
+
let _rawEmbedFlushChain = Promise.resolve()
|
|
253
|
+
// Ingest now WAITS for its embedding flush (bounded) so freshly ingested rows
|
|
254
|
+
// are dense-searchable the moment ingest_session returns. On a cold ONNX
|
|
255
|
+
// model the flush can stall on warmup; the race below caps the wait and lets
|
|
256
|
+
// the flush finish in the background (tick fallback still sweeps stragglers).
|
|
257
|
+
const INGEST_EMBED_WAIT_MS = 15_000
|
|
383
258
|
// Boot-edge background warmup. ONNX session creation on the embedding worker
|
|
384
259
|
// thread is CPU-heavy, so it must not overlap the worker's own init (DB open,
|
|
385
260
|
// schema, cycle wiring). Previously this was gated behind a fixed setTimeout —
|
|
@@ -387,83 +262,47 @@ let _transcriptOffsetsPersistTail = Promise.resolve()
|
|
|
387
262
|
// _initStore and fired at the _initRuntime completion edge (see _initRuntime),
|
|
388
263
|
// so it starts the instant boot's CPU-heavy work is done — no magic-number
|
|
389
264
|
// delay. MIXDOG_EMBED_WARMUP=0 disables it (model loads lazily on first use).
|
|
390
|
-
let _pendingEmbeddingWarmup = null
|
|
391
|
-
let _embeddingColdRecallLogAt = 0
|
|
392
265
|
|
|
393
266
|
const TRANSCRIPT_OFFSETS_KEY = 'state.transcript_offsets'
|
|
394
267
|
const CYCLE_LAST_RUN_KEY = 'state.cycle_last_run'
|
|
395
268
|
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
}
|
|
269
|
+
// Transcript ingest cluster (extracted to lib/transcript-ingest.mjs). Live
|
|
270
|
+
// db/config coupling is injected so index.mjs keeps lifecycle ownership.
|
|
271
|
+
const _transcriptIngest = createTranscriptIngest({
|
|
272
|
+
getDb: () => db,
|
|
273
|
+
loadMeta: () => getMetaValue(db, TRANSCRIPT_OFFSETS_KEY, '{}'),
|
|
274
|
+
persistMeta: (json) => setMetaValue(db, TRANSCRIPT_OFFSETS_KEY, json),
|
|
275
|
+
projectsRoot: () => path.join(mixdogHome(), 'projects'),
|
|
276
|
+
resolveProjectId,
|
|
277
|
+
firstTextContent,
|
|
278
|
+
cleanMemoryText,
|
|
279
|
+
log: __mixdogMemoryLog,
|
|
280
|
+
})
|
|
281
|
+
const {
|
|
282
|
+
loadTranscriptOffsets,
|
|
283
|
+
ingestTranscriptFile,
|
|
284
|
+
cwdFromTranscriptPath,
|
|
285
|
+
parseTsToMs,
|
|
286
|
+
} = _transcriptIngest
|
|
287
|
+
|
|
288
|
+
// Boot-edge embedding warmup queue (extracted to lib/embedding-warmup.mjs).
|
|
289
|
+
const _embeddingWarmup = createEmbeddingWarmup({
|
|
290
|
+
canStart: embeddingWarmupCanStart,
|
|
291
|
+
warmup: warmupEmbeddingProvider,
|
|
292
|
+
getDims: getEmbeddingDims,
|
|
293
|
+
persistMeta: (metaPath, value) => writeJsonAtomicSync(metaPath, value, { lock: true }),
|
|
294
|
+
log: __mixdogMemoryLog,
|
|
295
|
+
})
|
|
421
296
|
|
|
297
|
+
// DATA_DIR-bound wrappers over the extracted pure flag helpers (see
|
|
298
|
+
// ./lib/memory-config-flags.mjs). The pg-attach check needs DATA_DIR, which is
|
|
299
|
+
// module-local here.
|
|
422
300
|
function secondaryPgAdvertised() {
|
|
423
|
-
|
|
424
|
-
const runtimeRoot = process.env.MIXDOG_RUNTIME_ROOT
|
|
425
|
-
? path.resolve(process.env.MIXDOG_RUNTIME_ROOT)
|
|
426
|
-
: path.join(os.tmpdir(), 'mixdog')
|
|
427
|
-
try {
|
|
428
|
-
const cur = JSON.parse(fs.readFileSync(path.join(runtimeRoot, 'active-instance.json'), 'utf8'))
|
|
429
|
-
const port = Number(cur?.pg_port)
|
|
430
|
-
const pgdata = cur?.pg_pgdata ? path.resolve(String(cur.pg_pgdata)) : ''
|
|
431
|
-
return Number.isInteger(port) && port > 0 && pgdata === path.resolve(path.join(DATA_DIR, 'pgdata'))
|
|
432
|
-
} catch {
|
|
433
|
-
return false
|
|
434
|
-
}
|
|
301
|
+
return _secondaryPgAdvertised(DATA_DIR)
|
|
435
302
|
}
|
|
436
303
|
|
|
437
304
|
function assertSecondaryPgAttachable() {
|
|
438
|
-
|
|
439
|
-
throw new Error('memory-service: secondary mode requires an existing primary PG instance')
|
|
440
|
-
}
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
function scheduleBackgroundEmbeddingWarmup(metaPath, metaKey) {
|
|
444
|
-
if (!embeddingWarmupCanStart()) return
|
|
445
|
-
// Queue the warmup; _initRuntime fires it once boot completes.
|
|
446
|
-
_pendingEmbeddingWarmup = () => {
|
|
447
|
-
warmupEmbeddingProvider()
|
|
448
|
-
.then(() => {
|
|
449
|
-
const measured = Number(getEmbeddingDims())
|
|
450
|
-
try {
|
|
451
|
-
writeJsonAtomicSync(metaPath, { ...metaKey, dims: measured }, { lock: true })
|
|
452
|
-
} catch (e) {
|
|
453
|
-
__mixdogMemoryLog(`[memory-service] could not persist embedding-meta: ${e?.message || e}\n`)
|
|
454
|
-
}
|
|
455
|
-
})
|
|
456
|
-
.catch(err => {
|
|
457
|
-
__mixdogMemoryLog(`[memory-service] background warmup failed: ${err?.message || err}\n`)
|
|
458
|
-
})
|
|
459
|
-
}
|
|
460
|
-
}
|
|
461
|
-
|
|
462
|
-
function fireDeferredEmbeddingWarmup() {
|
|
463
|
-
const fire = _pendingEmbeddingWarmup
|
|
464
|
-
if (!fire) return
|
|
465
|
-
_pendingEmbeddingWarmup = null
|
|
466
|
-
fire()
|
|
305
|
+
return _assertSecondaryPgAttachable(DATA_DIR)
|
|
467
306
|
}
|
|
468
307
|
|
|
469
308
|
async function _initStore() {
|
|
@@ -509,7 +348,7 @@ async function _initStore() {
|
|
|
509
348
|
primeEmbeddingDims(dimsResolved)
|
|
510
349
|
assertSecondaryPgAttachable()
|
|
511
350
|
db = await openDatabase(DATA_DIR, dimsResolved)
|
|
512
|
-
|
|
351
|
+
_embeddingWarmup.schedule(EMBEDDING_META_PATH, metaKey)
|
|
513
352
|
} else {
|
|
514
353
|
if (!embeddingWarmupCanStart()) {
|
|
515
354
|
throw new Error('memory-service: embedding dims unavailable while warmup is disabled')
|
|
@@ -547,7 +386,14 @@ async function _initStore() {
|
|
|
547
386
|
// MIXDOG_MEMORY_DISABLE_LLM_WORKER=1 + cycles enabled hits an empty registry
|
|
548
387
|
// and fails with "Provider not found". Non-fatal: a failure here is logged
|
|
549
388
|
// and cycle1's own callLlm surfaces the unresolved-provider error per call.
|
|
550
|
-
|
|
389
|
+
// Registry must be available whenever cycles COULD run in this process, not
|
|
390
|
+
// just when they are currently enabled: recap can be toggled on at runtime
|
|
391
|
+
// (memoryCyclesEnabled() now includes the recap flag), so gate registry init
|
|
392
|
+
// on the cycle-capable conditions (not secondary, env not hard-disabled) OR
|
|
393
|
+
// the llm worker gate — never on the runtime recap flag itself, or enabling
|
|
394
|
+
// recap later would hit an empty registry and fail with "Provider not found".
|
|
395
|
+
const cyclesCapable = !memorySecondaryMode() && !envFlagEnabled('MIXDOG_MEMORY_DISABLE_CYCLES')
|
|
396
|
+
if (cyclesCapable || memoryLlmWorkerEnabled()) {
|
|
551
397
|
try {
|
|
552
398
|
const agentCfg = loadAgentConfig()
|
|
553
399
|
await initProviders(agentCfg.providers || {})
|
|
@@ -559,29 +405,6 @@ async function _initStore() {
|
|
|
559
405
|
await loadTranscriptOffsets()
|
|
560
406
|
}
|
|
561
407
|
|
|
562
|
-
async function loadTranscriptOffsets() {
|
|
563
|
-
try {
|
|
564
|
-
const raw = await getMetaValue(db, TRANSCRIPT_OFFSETS_KEY, '{}')
|
|
565
|
-
const obj = JSON.parse(raw)
|
|
566
|
-
_transcriptOffsets = new Map(Object.entries(obj))
|
|
567
|
-
} catch {
|
|
568
|
-
_transcriptOffsets = new Map()
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
|
|
572
|
-
async function persistTranscriptOffsets() {
|
|
573
|
-
const run = _transcriptOffsetsPersistTail.catch(() => {}).then(async () => {
|
|
574
|
-
try {
|
|
575
|
-
const obj = Object.fromEntries(_transcriptOffsets)
|
|
576
|
-
await setMetaValue(db, TRANSCRIPT_OFFSETS_KEY, JSON.stringify(obj))
|
|
577
|
-
} catch (e) {
|
|
578
|
-
__mixdogMemoryLog(`[memory] persist transcript offsets failed: ${e.message}\n`)
|
|
579
|
-
}
|
|
580
|
-
})
|
|
581
|
-
_transcriptOffsetsPersistTail = run.catch(() => {})
|
|
582
|
-
return run
|
|
583
|
-
}
|
|
584
|
-
|
|
585
408
|
async function getCycleLastRun() {
|
|
586
409
|
try {
|
|
587
410
|
const raw = await getMetaValue(db, CYCLE_LAST_RUN_KEY, '{}')
|
|
@@ -616,146 +439,6 @@ async function setCycleLastRun(kind, ts) {
|
|
|
616
439
|
await mergeMetaValue(db, CYCLE_LAST_RUN_KEY, { [kind]: ts })
|
|
617
440
|
}
|
|
618
441
|
|
|
619
|
-
// Raw-row priority lookup for narrow-window queries. Raw rows (is_root=0,
|
|
620
|
-
// chunk_root IS NULL) are inserted immediately by ingestTranscriptFile before
|
|
621
|
-
// cycle1 runs, so they always carry the freshest turns in the DB.
|
|
622
|
-
async function readRawRowsInWindow(db, tsFromMs, tsToMs, hardLimit = 10, { projectScope, sessionId } = {}) {
|
|
623
|
-
try {
|
|
624
|
-
// Composable WHERE assembly (mirrors retrieveEntries' filter semantics so
|
|
625
|
-
// raw and chunked legs stay in filter parity: projectScope AND sessionId
|
|
626
|
-
// apply identically to both pools).
|
|
627
|
-
const where = ['chunk_root IS NULL', 'is_root = 0', 'ts >= $1', 'ts <= $2']
|
|
628
|
-
const params = [tsFromMs ?? 0, tsToMs ?? Date.now()]
|
|
629
|
-
if (projectScope === 'common') {
|
|
630
|
-
where.push('project_id IS NULL')
|
|
631
|
-
} else if (projectScope && projectScope !== 'all') {
|
|
632
|
-
params.push(projectScope)
|
|
633
|
-
where.push(`(project_id IS NULL OR project_id = $${params.length})`)
|
|
634
|
-
}
|
|
635
|
-
const sid = String(sessionId || '').trim()
|
|
636
|
-
if (sid) {
|
|
637
|
-
params.push(sid)
|
|
638
|
-
where.push(`session_id = $${params.length}`)
|
|
639
|
-
}
|
|
640
|
-
params.push(hardLimit)
|
|
641
|
-
const sql = `SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root,
|
|
642
|
-
element, category, summary, status, score, last_seen_at, project_id
|
|
643
|
-
FROM entries
|
|
644
|
-
WHERE ${where.join(' AND ')}
|
|
645
|
-
ORDER BY ts DESC
|
|
646
|
-
LIMIT $${params.length}`
|
|
647
|
-
const rows = (await db.query(sql, params)).rows
|
|
648
|
-
return rows.map(r => ({ ...r, retrievalScore: 0, rrf: 0 }))
|
|
649
|
-
} catch { return [] }
|
|
650
|
-
}
|
|
651
|
-
|
|
652
|
-
async function recallSessionRows(args = {}) {
|
|
653
|
-
const sessionId = String(args.sessionId || args.session_id || '').trim()
|
|
654
|
-
if (!sessionId) return { text: '(no current session)' }
|
|
655
|
-
const limit = Math.max(1, Math.min(100, Number(args.limit) || 20))
|
|
656
|
-
const terms = sessionRecallTerms(args.query)
|
|
657
|
-
const params = [sessionId]
|
|
658
|
-
// Roots + not-yet-chunked leaves only. Once cycle1 turns raw leaves into
|
|
659
|
-
// (root, members) pairs, selecting every row unfiltered emitted the root's
|
|
660
|
-
// summary AND its own member rows in the same browse — duplicate content.
|
|
661
|
-
// A committed member (is_root=0 with a chunk_root) is always reachable via
|
|
662
|
-
// its root's `members` expansion below, so it never needs to be selected
|
|
663
|
-
// directly here.
|
|
664
|
-
const where = ['session_id = $1', '(is_root = 1 OR chunk_root IS NULL OR chunk_root = id)']
|
|
665
|
-
// Current-turn cutoff: the newest unchunked row is very often the calling
|
|
666
|
-
// turn's OWN recall request/tool-args, still being written when this query
|
|
667
|
-
// runs. Exclude it from a bare (no-query) browse so the in-flight turn
|
|
668
|
-
// doesn't self-echo; a query browse (explicit search intent) keeps it.
|
|
669
|
-
// Only treat the newest unchunked turn as "in-flight" when its latest row
|
|
670
|
-
// is fresh (within FRESHNESS_MS of now) — an older newest-unchunked-turn
|
|
671
|
-
// is completed history (cycle1 just hasn't gotten to it, or drain timed
|
|
672
|
-
// out) and must stay visible, not be silently hidden every browse.
|
|
673
|
-
const IN_FLIGHT_TURN_FRESHNESS_MS = 5 * 60 * 1000
|
|
674
|
-
let excludeSourceTurnId = null
|
|
675
|
-
if (terms.length === 0) {
|
|
676
|
-
try {
|
|
677
|
-
const r = await db.query(
|
|
678
|
-
`SELECT source_turn t, MAX(ts) last_ts FROM entries
|
|
679
|
-
WHERE session_id = $1 AND chunk_root IS NULL
|
|
680
|
-
GROUP BY source_turn ORDER BY source_turn DESC LIMIT 1`,
|
|
681
|
-
[sessionId],
|
|
682
|
-
)
|
|
683
|
-
const t = r.rows?.[0]?.t
|
|
684
|
-
const lastTs = Number(r.rows?.[0]?.last_ts)
|
|
685
|
-
if (t != null && Number.isFinite(lastTs) && (Date.now() - lastTs) <= IN_FLIGHT_TURN_FRESHNESS_MS) {
|
|
686
|
-
excludeSourceTurnId = Number(t)
|
|
687
|
-
}
|
|
688
|
-
} catch {}
|
|
689
|
-
}
|
|
690
|
-
if (Number.isFinite(excludeSourceTurnId)) {
|
|
691
|
-
params.push(excludeSourceTurnId)
|
|
692
|
-
where.push(`NOT (chunk_root IS NULL AND source_turn = $${params.length})`)
|
|
693
|
-
}
|
|
694
|
-
if (terms.length > 0) {
|
|
695
|
-
const textExpr = `lower(coalesce(content, '') || ' ' || coalesce(element, '') || ' ' || coalesce(summary, ''))`
|
|
696
|
-
const clauses = terms.map((term) => {
|
|
697
|
-
params.push(`%${term}%`)
|
|
698
|
-
return `${textExpr} LIKE $${params.length}`
|
|
699
|
-
})
|
|
700
|
-
where.push(`(${clauses.join(' OR ')})`)
|
|
701
|
-
}
|
|
702
|
-
params.push(limit)
|
|
703
|
-
let rows = (await db.query(`
|
|
704
|
-
SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root,
|
|
705
|
-
element, category, summary, status, score, last_seen_at, project_id
|
|
706
|
-
FROM entries
|
|
707
|
-
WHERE ${where.join(' AND ')}
|
|
708
|
-
ORDER BY ts DESC, id DESC
|
|
709
|
-
LIMIT $${params.length}
|
|
710
|
-
`, params)).rows
|
|
711
|
-
if (rows.length < limit) {
|
|
712
|
-
const seen = new Set(rows.map((row) => Number(row.id)).filter((id) => Number.isFinite(id)))
|
|
713
|
-
const fillLimit = Math.max(0, limit - rows.length)
|
|
714
|
-
const fillWhere = ['session_id = $1', 'id <> ALL($2::bigint[])', '(is_root = 1 OR chunk_root IS NULL OR chunk_root = id)']
|
|
715
|
-
const fillParams = [sessionId, [...seen]]
|
|
716
|
-
if (Number.isFinite(excludeSourceTurnId)) {
|
|
717
|
-
fillParams.push(excludeSourceTurnId)
|
|
718
|
-
fillWhere.push(`NOT (chunk_root IS NULL AND source_turn = $${fillParams.length})`)
|
|
719
|
-
}
|
|
720
|
-
fillParams.push(fillLimit)
|
|
721
|
-
const fillRows = fillLimit > 0
|
|
722
|
-
? (await db.query(`
|
|
723
|
-
SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root,
|
|
724
|
-
element, category, summary, status, score, last_seen_at, project_id
|
|
725
|
-
FROM entries
|
|
726
|
-
WHERE ${fillWhere.join(' AND ')}
|
|
727
|
-
ORDER BY ts DESC, id DESC
|
|
728
|
-
LIMIT $${fillParams.length}
|
|
729
|
-
`, fillParams)).rows
|
|
730
|
-
: []
|
|
731
|
-
if (fillRows.length > 0) rows = [...rows, ...fillRows]
|
|
732
|
-
}
|
|
733
|
-
if (args.includeMembers === true) {
|
|
734
|
-
const rootIds = rows
|
|
735
|
-
.filter((row) => Number(row.is_root) === 1)
|
|
736
|
-
.map((row) => Number(row.id))
|
|
737
|
-
.filter((id) => Number.isFinite(id))
|
|
738
|
-
if (rootIds.length > 0) {
|
|
739
|
-
const members = (await db.query(`
|
|
740
|
-
SELECT id, ts, role, content, session_id, source_turn, project_id, chunk_root
|
|
741
|
-
FROM entries
|
|
742
|
-
WHERE chunk_root = ANY($1::bigint[]) AND is_root = 0
|
|
743
|
-
ORDER BY chunk_root ASC, COALESCE(source_turn, 2147483647) ASC, ts ASC, id ASC
|
|
744
|
-
`, [rootIds])).rows
|
|
745
|
-
const byRoot = new Map(rootIds.map((id) => [id, []]))
|
|
746
|
-
for (const member of members) {
|
|
747
|
-
const root = Number(member.chunk_root)
|
|
748
|
-
if (byRoot.has(root)) byRoot.get(root).push(member)
|
|
749
|
-
}
|
|
750
|
-
for (const row of rows) {
|
|
751
|
-
const id = Number(row.id)
|
|
752
|
-
if (byRoot.has(id)) row.members = byRoot.get(id)
|
|
753
|
-
}
|
|
754
|
-
}
|
|
755
|
-
}
|
|
756
|
-
return { text: renderEntryLines(rows) }
|
|
757
|
-
}
|
|
758
|
-
|
|
759
442
|
async function ingestSessionMessages(args = {}) {
|
|
760
443
|
const sessionId = String(args.sessionId || args.session_id || `session-${Date.now()}`).trim()
|
|
761
444
|
const messages = Array.isArray(args.messages) ? args.messages : []
|
|
@@ -828,886 +511,85 @@ async function ingestSessionMessages(args = {}) {
|
|
|
828
511
|
turnAllocator.next()
|
|
829
512
|
}
|
|
830
513
|
}
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
async function ingestTranscriptFileImpl(transcriptPath, { cwd } = {}) {
|
|
849
|
-
let stat
|
|
850
|
-
try { stat = await fs.promises.stat(transcriptPath) } catch { return 0 }
|
|
851
|
-
const sessionUuid = path.basename(transcriptPath, '.jsonl')
|
|
852
|
-
const prev = snapshotTranscriptOffset(transcriptPath)
|
|
853
|
-
if (stat.size < prev.bytes) {
|
|
854
|
-
prev.bytes = 0
|
|
855
|
-
prev.lineIndex = 0
|
|
856
|
-
}
|
|
857
|
-
if (stat.size <= prev.bytes) return 0
|
|
858
|
-
|
|
859
|
-
const fh = await fs.promises.open(transcriptPath, 'r')
|
|
860
|
-
const buf = Buffer.alloc(stat.size - prev.bytes)
|
|
861
|
-
try {
|
|
862
|
-
await fh.read(buf, 0, buf.length, prev.bytes)
|
|
863
|
-
} finally {
|
|
864
|
-
await fh.close()
|
|
865
|
-
}
|
|
866
|
-
const text = buf.toString('utf8')
|
|
867
|
-
|
|
868
|
-
const resolvedCwd = typeof cwd === 'string' && cwd ? cwd : cwdFromTranscriptPath(transcriptPath)
|
|
869
|
-
// No cwd resolved -> classify as COMMON (project_id NULL). Falling back to
|
|
870
|
-
// process.cwd() would misclassify rows under the service/plugin cwd.
|
|
871
|
-
const projectId = resolvedCwd ? resolveProjectId(resolvedCwd) : null
|
|
872
|
-
|
|
873
|
-
let count = 0
|
|
874
|
-
let index = prev.lineIndex
|
|
875
|
-
// Track the byte boundary of the LAST line we fully consumed (parsed +
|
|
876
|
-
// either inserted or intentionally skipped). On parse failure or
|
|
877
|
-
// transient insert error we stop and leave the boundary untouched so the
|
|
878
|
-
// next sweep retries from the same position. This prevents malformed
|
|
879
|
-
// trailing JSONL (mid-write partial lines) and DB hiccups from being
|
|
880
|
-
// silently consumed forever.
|
|
881
|
-
let lastGoodBytes = prev.bytes
|
|
882
|
-
let lastGoodLineIndex = prev.lineIndex
|
|
883
|
-
let cursor = 0
|
|
884
|
-
while (cursor < text.length) {
|
|
885
|
-
const nl = text.indexOf('\n', cursor)
|
|
886
|
-
// No trailing newline -> partial line still being written; stop here
|
|
887
|
-
// without advancing so the rest is re-read once the writer flushes.
|
|
888
|
-
if (nl === -1) break
|
|
889
|
-
const rawLine = text.slice(cursor, nl)
|
|
890
|
-
const consumedBytes = Buffer.byteLength(rawLine, 'utf8') + 1
|
|
891
|
-
cursor = nl + 1
|
|
892
|
-
const line = rawLine.replace(/\r$/, '')
|
|
893
|
-
if (!line) {
|
|
894
|
-
lastGoodBytes += consumedBytes
|
|
895
|
-
continue
|
|
896
|
-
}
|
|
897
|
-
index += 1
|
|
898
|
-
let parsed
|
|
899
|
-
try { parsed = JSON.parse(line) } catch {
|
|
900
|
-
// Malformed line: do not advance past it; retry on next sweep.
|
|
901
|
-
index -= 1
|
|
902
|
-
break
|
|
903
|
-
}
|
|
904
|
-
const role = parsed.message?.role
|
|
905
|
-
if (role !== 'user' && role !== 'assistant') {
|
|
906
|
-
lastGoodBytes += consumedBytes
|
|
907
|
-
lastGoodLineIndex = index
|
|
908
|
-
continue
|
|
909
|
-
}
|
|
910
|
-
const content = firstTextContent(parsed.message?.content)
|
|
911
|
-
if (!content || !content.trim()) {
|
|
912
|
-
lastGoodBytes += consumedBytes
|
|
913
|
-
lastGoodLineIndex = index
|
|
914
|
-
continue
|
|
915
|
-
}
|
|
916
|
-
const cleaned = cleanMemoryText(content)
|
|
917
|
-
if (!cleaned) {
|
|
918
|
-
lastGoodBytes += consumedBytes
|
|
919
|
-
lastGoodLineIndex = index
|
|
920
|
-
continue
|
|
921
|
-
}
|
|
922
|
-
const tsMs = parseTsToMs(parsed.timestamp ?? parsed.ts ?? Date.now())
|
|
923
|
-
const sourceRef = `transcript:${sessionUuid}#${index}`
|
|
924
|
-
try {
|
|
925
|
-
const result = await db.query(
|
|
926
|
-
`INSERT INTO entries(ts, role, content, source_ref, session_id, source_turn, project_id)
|
|
927
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
|
928
|
-
ON CONFLICT DO NOTHING`,
|
|
929
|
-
[tsMs, role, cleaned, sourceRef, sessionUuid, index, projectId]
|
|
930
|
-
)
|
|
931
|
-
if (Number(result.rowCount ?? result.affectedRows ?? 0) > 0) count += 1
|
|
932
|
-
lastGoodBytes += consumedBytes
|
|
933
|
-
lastGoodLineIndex = index
|
|
934
|
-
} catch (e) {
|
|
935
|
-
__mixdogMemoryLog(`[transcript-watch] insert error (${sourceRef}): ${e.message}\n`)
|
|
936
|
-
// Transient insert failure: leave the boundary before this line so
|
|
937
|
-
// the next sweep retries it. Roll back the line counter too.
|
|
938
|
-
index -= 1
|
|
939
|
-
break
|
|
940
|
-
}
|
|
941
|
-
}
|
|
942
|
-
_transcriptOffsets.set(transcriptPath, {
|
|
943
|
-
bytes: lastGoodBytes,
|
|
944
|
-
lineIndex: lastGoodLineIndex,
|
|
945
|
-
})
|
|
946
|
-
await persistTranscriptOffsets()
|
|
947
|
-
return count
|
|
948
|
-
}
|
|
949
|
-
|
|
950
|
-
async function ingestTranscriptFile(transcriptPath, options = {}) {
|
|
951
|
-
return runTranscriptIngestSerialized(transcriptPath, () => ingestTranscriptFileImpl(transcriptPath, options))
|
|
952
|
-
}
|
|
953
|
-
|
|
954
|
-
function parseTsToMs(value) {
|
|
955
|
-
if (typeof value === 'number' && Number.isFinite(value)) return value < 1e12 ? value * 1000 : value
|
|
956
|
-
const parsed = Date.parse(String(value))
|
|
957
|
-
return Number.isFinite(parsed) ? parsed : Date.now()
|
|
958
|
-
}
|
|
959
|
-
|
|
960
|
-
// Extract cwd from the transcript file's JSONL rows. Mixdog embeds
|
|
961
|
-
// the session cwd as a top-level `cwd` field on every message row, so
|
|
962
|
-
// scanning the first few lines is reliable on all platforms (Windows/Linux)
|
|
963
|
-
// without slug-decoding ambiguity. Returns undefined when no cwd is found
|
|
964
|
-
// or the extracted path does not exist on disk (falls back to COMMON).
|
|
965
|
-
function cwdFromTranscriptPath(fp) {
|
|
966
|
-
let fd
|
|
967
|
-
try {
|
|
968
|
-
fd = fs.openSync(fp, 'r')
|
|
969
|
-
const buf = Buffer.alloc(Math.min(fs.fstatSync(fd).size, 100 * 1024))
|
|
970
|
-
fs.readSync(fd, buf, 0, buf.length, 0)
|
|
971
|
-
fs.closeSync(fd)
|
|
972
|
-
fd = undefined
|
|
973
|
-
const lines = buf.toString('utf8').split('\n')
|
|
974
|
-
for (let i = 0; i < Math.min(lines.length, 5); i++) {
|
|
975
|
-
const line = lines[i].trim()
|
|
976
|
-
if (!line) continue
|
|
977
|
-
try {
|
|
978
|
-
const obj = JSON.parse(line)
|
|
979
|
-
if (typeof obj.cwd === 'string' && obj.cwd) {
|
|
980
|
-
const candidate = obj.cwd
|
|
981
|
-
try { if (fs.statSync(candidate).isDirectory()) return candidate } catch {}
|
|
982
|
-
}
|
|
983
|
-
} catch {}
|
|
984
|
-
}
|
|
985
|
-
} catch {} finally {
|
|
986
|
-
if (fd != null) { try { fs.closeSync(fd) } catch {} }
|
|
987
|
-
}
|
|
988
|
-
return undefined
|
|
989
|
-
}
|
|
990
|
-
|
|
991
|
-
function _initTranscriptWatcher() {
|
|
992
|
-
const projectsRoot = path.join(mixdogHome(), 'projects')
|
|
993
|
-
const SAFETY_POLL_MS = 5 * 60_000
|
|
994
|
-
const DEBOUNCE_MS = 500
|
|
995
|
-
const watchedFiles = new Map()
|
|
996
|
-
const pendingByFile = new Map()
|
|
997
|
-
const watchers = []
|
|
998
|
-
const intervals = []
|
|
999
|
-
const polledFiles = new Set()
|
|
1000
|
-
let safetySweepTimeout = null
|
|
1001
|
-
|
|
1002
|
-
function isWatchable(relOrBase) {
|
|
1003
|
-
const base = path.basename(relOrBase)
|
|
1004
|
-
if (!base.endsWith('.jsonl') || base.startsWith('agent-')) return false
|
|
1005
|
-
if (relOrBase.includes('tmp') || relOrBase.includes('cache') || relOrBase.includes('plugins')) return false
|
|
1006
|
-
return true
|
|
1007
|
-
}
|
|
1008
|
-
|
|
1009
|
-
async function ingestOne(fp) {
|
|
1010
|
-
try {
|
|
1011
|
-
if (!fs.existsSync(fp)) return
|
|
1012
|
-
const stat = fs.statSync(fp)
|
|
1013
|
-
const mtime = stat.mtimeMs
|
|
1014
|
-
const prev = watchedFiles.get(fp)
|
|
1015
|
-
if (prev && prev >= mtime) return
|
|
1016
|
-
const n = await ingestTranscriptFile(fp, { cwd: cwdFromTranscriptPath(fp) })
|
|
1017
|
-
// Only mark this mtime as 'consumed' once the persisted offset has
|
|
1018
|
-
// fully advanced past the observed file size. On a transient insert
|
|
1019
|
-
// error (or a malformed trailing line) ingestTranscriptFile leaves
|
|
1020
|
-
// the persisted offset before the failed line for retry; caching
|
|
1021
|
-
// the new mtime unconditionally would suppress the next sweep until
|
|
1022
|
-
// the file mutated again, losing the retry. Leave the cache
|
|
1023
|
-
// untouched on partial advance so the next sweep re-ingests.
|
|
1024
|
-
const off = _transcriptOffsets.get(fp)
|
|
1025
|
-
if (off && off.bytes >= stat.size) {
|
|
1026
|
-
watchedFiles.set(fp, mtime)
|
|
1027
|
-
}
|
|
1028
|
-
if (n > 0) {
|
|
1029
|
-
__mixdogMemoryLog(`[transcript-watch] ingested ${n} entries from ${path.basename(fp)}\n`)
|
|
1030
|
-
}
|
|
1031
|
-
} catch (e) {
|
|
1032
|
-
__mixdogMemoryLog(`[transcript-watch] ingest error: ${e.message}\n`)
|
|
1033
|
-
}
|
|
1034
|
-
}
|
|
1035
|
-
|
|
1036
|
-
function scheduleIngest(fp) {
|
|
1037
|
-
const existing = pendingByFile.get(fp)
|
|
1038
|
-
if (existing) clearTimeout(existing)
|
|
1039
|
-
const timer = setTimeout(() => {
|
|
1040
|
-
pendingByFile.delete(fp)
|
|
1041
|
-
ingestOne(fp)
|
|
1042
|
-
}, DEBOUNCE_MS)
|
|
1043
|
-
pendingByFile.set(fp, timer)
|
|
1044
|
-
}
|
|
1045
|
-
|
|
1046
|
-
async function discoverActiveTranscripts() {
|
|
1047
|
-
let topLevel
|
|
1048
|
-
try { topLevel = await fs.promises.readdir(projectsRoot) }
|
|
1049
|
-
catch { return [] }
|
|
1050
|
-
const files = []
|
|
1051
|
-
for (const d of topLevel) {
|
|
1052
|
-
if (d.includes('tmp') || d.includes('cache') || d.includes('plugins')) continue
|
|
1053
|
-
const full = path.join(projectsRoot, d)
|
|
1054
|
-
let inner
|
|
1055
|
-
try { inner = await fs.promises.readdir(full) } catch { continue }
|
|
1056
|
-
for (const f of inner) {
|
|
1057
|
-
if (!f.endsWith('.jsonl') || f.startsWith('agent-')) continue
|
|
1058
|
-
const fp = path.join(full, f)
|
|
1059
|
-
try {
|
|
1060
|
-
const stat = await fs.promises.stat(fp)
|
|
1061
|
-
files.push({ path: fp, mtime: stat.mtimeMs })
|
|
1062
|
-
} catch {}
|
|
1063
|
-
}
|
|
1064
|
-
}
|
|
1065
|
-
const cutoff = Date.now() - 30 * 60_000
|
|
1066
|
-
return files.filter(f => f.mtime > cutoff)
|
|
1067
|
-
}
|
|
1068
|
-
|
|
1069
|
-
async function safetySweep() {
|
|
1070
|
-
try {
|
|
1071
|
-
const active = await discoverActiveTranscripts()
|
|
1072
|
-
for (const { path: fp } of active) ingestOne(fp)
|
|
1073
|
-
} catch (e) {
|
|
1074
|
-
__mixdogMemoryLog(`[transcript-watch] safety sweep error: ${e.message}\n`)
|
|
1075
|
-
}
|
|
1076
|
-
}
|
|
1077
|
-
|
|
1078
|
-
safetySweepTimeout = setTimeout(safetySweep, 3_000)
|
|
1079
|
-
|
|
1080
|
-
// fs.watch({recursive}) is only reliable on win32.
|
|
1081
|
-
// darwin: recursive option unreliable — use flat watch per-entry (glob dirs at start).
|
|
1082
|
-
// linux/WSL: recursive not supported — use fs.watchFile polling per file found via
|
|
1083
|
-
// the safety sweep, or fall back entirely to safety sweep.
|
|
1084
|
-
if (process.platform === 'win32') {
|
|
1085
|
-
try {
|
|
1086
|
-
const watcher = fs.watch(projectsRoot, { recursive: true, persistent: true }, (_event, filename) => {
|
|
1087
|
-
if (!filename) return
|
|
1088
|
-
if (!isWatchable(filename)) return
|
|
1089
|
-
const fp = path.join(projectsRoot, filename)
|
|
1090
|
-
scheduleIngest(fp)
|
|
1091
|
-
})
|
|
1092
|
-
watcher.on('error', (err) => {
|
|
1093
|
-
__mixdogMemoryLog(`[transcript-watch] fs.watch error: ${err.message}\n`)
|
|
514
|
+
// Always-on post-ingest raw embedding: freshly ingested rows become
|
|
515
|
+
// dense-searchable immediately (autoclear/recall-fasttrack hydration,
|
|
516
|
+
// recall empty-fallback), without waiting for cycle1 chunking or the
|
|
517
|
+
// ~60s background tick. Local ONNX only — no LLM cost, so it runs
|
|
518
|
+
// regardless of the recap toggle. SYNCHRONOUS (bounded): ingest_session
|
|
519
|
+
// resolves only after this session's rows are embedded, so an immediately
|
|
520
|
+
// following recall sees them in the dense leg. Bursts serialize on
|
|
521
|
+
// _rawEmbedFlushChain instead of being skipped; a 15s cap keeps a cold
|
|
522
|
+
// ONNX warmup from wedging ingest (tick fallback sweeps what's left).
|
|
523
|
+
if (inserted > 0) {
|
|
524
|
+
const run = _rawEmbedFlushChain
|
|
525
|
+
.catch(() => {}) // never let a previous flush failure poison the chain
|
|
526
|
+
.then(() => flushRawEmbeddings(db, { limit: 200 }))
|
|
527
|
+
.then((r) => {
|
|
528
|
+
if (r.attempted > 0) __mixdogMemoryLog(`[embed] post-ingest raw flush attempted=${r.attempted} embedded=${r.embedded}\n`)
|
|
529
|
+
return r
|
|
1094
530
|
})
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
// Flat watch: register a non-recursive watcher on each immediate subdirectory.
|
|
1103
|
-
// New subdirs are picked up on the next safety sweep cycle.
|
|
1104
|
-
try {
|
|
1105
|
-
const registerFlat = (dir) => {
|
|
1106
|
-
try {
|
|
1107
|
-
const w = fs.watch(dir, { persistent: true }, (_event, filename) => {
|
|
1108
|
-
if (!filename) return
|
|
1109
|
-
const fp = path.join(dir, filename)
|
|
1110
|
-
if (!isWatchable(fp)) return
|
|
1111
|
-
scheduleIngest(fp)
|
|
1112
|
-
})
|
|
1113
|
-
w.on('error', () => { /* ignore individual dir errors */ })
|
|
1114
|
-
watchers.push(w)
|
|
1115
|
-
} catch { /* dir may not exist yet */ }
|
|
1116
|
-
}
|
|
1117
|
-
registerFlat(projectsRoot)
|
|
1118
|
-
try {
|
|
1119
|
-
for (const entry of fs.readdirSync(projectsRoot, { withFileTypes: true })) {
|
|
1120
|
-
if (entry.isDirectory()) registerFlat(path.join(projectsRoot, entry.name))
|
|
1121
|
-
}
|
|
1122
|
-
} catch { /* best effort */ }
|
|
1123
|
-
__mixdogMemoryLog(`[transcript-watch] flat fs.watch active on ${projectsRoot} (darwin)\n`)
|
|
1124
|
-
} catch (e) {
|
|
1125
|
-
__mixdogMemoryLog(`[transcript-watch] flat watch setup failed: ${e.message} — relying on safety sweep only\n`)
|
|
1126
|
-
}
|
|
1127
|
-
intervals.push(setInterval(safetySweep, SAFETY_POLL_MS))
|
|
1128
|
-
} else {
|
|
1129
|
-
// linux/WSL: fs.watch recursive is unsupported. Use fs.watchFile polling for
|
|
1130
|
-
// individual files surfaced by the safety sweep, in addition to the sweep itself.
|
|
1131
|
-
__mixdogMemoryLog(`[transcript-watch] linux/WSL — using safety sweep + fs.watchFile polling (no recursive watch)\n`)
|
|
1132
|
-
// Wrap by reassigning the closure-captured reference is not possible here;
|
|
1133
|
-
// instead, register watchFile inside the safety sweep callback by intercepting
|
|
1134
|
-
// active file list after each sweep. The interval already calls safetySweep
|
|
1135
|
-
// which calls ingestOne; watchFile additions happen as a side-effect of the sweep.
|
|
1136
|
-
const _patchedSweep = async () => {
|
|
1137
|
-
try {
|
|
1138
|
-
const active = await discoverActiveTranscripts()
|
|
1139
|
-
for (const { path: fp } of active) {
|
|
1140
|
-
if (!polledFiles.has(fp)) {
|
|
1141
|
-
polledFiles.add(fp)
|
|
1142
|
-
fs.watchFile(fp, { persistent: false, interval: 2000 }, () => {
|
|
1143
|
-
if (isWatchable(fp)) scheduleIngest(fp)
|
|
1144
|
-
})
|
|
1145
|
-
}
|
|
1146
|
-
ingestOne(fp)
|
|
1147
|
-
}
|
|
1148
|
-
} catch (e) {
|
|
1149
|
-
__mixdogMemoryLog(`[transcript-watch] linux sweep error: ${e.message}\n`)
|
|
1150
|
-
}
|
|
1151
|
-
}
|
|
1152
|
-
// Replace the safety sweep interval with the patched version.
|
|
1153
|
-
intervals.push(setInterval(_patchedSweep, SAFETY_POLL_MS))
|
|
1154
|
-
}
|
|
1155
|
-
|
|
1156
|
-
return {
|
|
1157
|
-
stop() {
|
|
1158
|
-
if (safetySweepTimeout) { clearTimeout(safetySweepTimeout); safetySweepTimeout = null }
|
|
1159
|
-
for (const t of pendingByFile.values()) { try { clearTimeout(t) } catch {} }
|
|
1160
|
-
pendingByFile.clear()
|
|
1161
|
-
for (const i of intervals) { try { clearInterval(i) } catch {} }
|
|
1162
|
-
intervals.length = 0
|
|
1163
|
-
for (const w of watchers) { try { w.close() } catch {} }
|
|
1164
|
-
watchers.length = 0
|
|
1165
|
-
for (const fp of polledFiles) { try { fs.unwatchFile(fp) } catch {} }
|
|
1166
|
-
polledFiles.clear()
|
|
1167
|
-
},
|
|
1168
|
-
}
|
|
1169
|
-
}
|
|
1170
|
-
|
|
1171
|
-
// Phase B §2.4 — cache-keeper health thresholds.
|
|
1172
|
-
// warning fires when cycle1 is overdue past HEALTH_OVERDUE_MS; an auto-
|
|
1173
|
-
// restart attempt fires when the warning has been emitted AND the most
|
|
1174
|
-
// recent unscheduled restart was more than AUTO_RESTART_COOLDOWN_MS ago.
|
|
1175
|
-
// Both default to 5 min per spec; caller overrides are not exposed yet.
|
|
1176
|
-
const CYCLE1_HEALTH_OVERDUE_MS = 5 * 60_000
|
|
1177
|
-
const CYCLE1_AUTO_RESTART_COOLDOWN_MS = 5 * 60_000
|
|
1178
|
-
|
|
1179
|
-
// In-process cycle1 LLM adapter. The memory daemon runs makeAgentDispatch()
|
|
1180
|
-
// locally (provider registry is initialized in _initStore), so cycle1 never
|
|
1181
|
-
// has to route over the dead IPC agent path (agent-ipc.mjs callAgentDispatch). The
|
|
1182
|
-
// factory is built once (role/taskType are fixed) and the returned function
|
|
1183
|
-
// is reshaped to cycle1's call signature: cycle1 invokes
|
|
1184
|
-
// `callLlm({ role, taskType, mode, preset, timeout, cwd }, userMessage)` and
|
|
1185
|
-
// expects a raw string, while makeAgentDispatch's function takes a single
|
|
1186
|
-
// `{ prompt }` object. The adapter maps the two — preset/cwd resolution is
|
|
1187
|
-
// handled inside makeAgentDispatch via role (cycle1-agent → maint.memory slot).
|
|
1188
|
-
let _cycle1AgentDispatch = null
|
|
1189
|
-
function getCycle1CallLlm() {
|
|
1190
|
-
if (!_cycle1AgentDispatch) {
|
|
1191
|
-
_cycle1AgentDispatch = makeAgentDispatch({
|
|
1192
|
-
agent: 'cycle1-agent',
|
|
1193
|
-
taskType: 'maintenance',
|
|
1194
|
-
sourceType: 'memory-cycle',
|
|
1195
|
-
// cycle1 parses the full raw line-format response; the agent brief cap
|
|
1196
|
-
// (12KB) would truncate a large valid response and append prose, causing
|
|
1197
|
-
// partial parsing / omitted / invalid chunks. Opt out so the cycle1
|
|
1198
|
-
// no-truncation contract is preserved through makeAgentDispatch.
|
|
1199
|
-
brief: false,
|
|
1200
|
-
})
|
|
1201
|
-
}
|
|
1202
|
-
return async (opts = {}, userMessage) => {
|
|
1203
|
-
// Preserve cycle1's timeout contract: cycle1 derives `opts.timeout` from
|
|
1204
|
-
// config / caller deadline and expects it to bound the call. makeAgentDispatch
|
|
1205
|
-
// takes it as a per-call `idleTimeoutMs` (stale watchdog). Map it through;
|
|
1206
|
-
// omit when absent/0 so agent defaults apply.
|
|
1207
|
-
const callTimeout = Number(opts?.timeout)
|
|
1208
|
-
return _cycle1AgentDispatch({
|
|
1209
|
-
prompt: String(userMessage ?? ''),
|
|
1210
|
-
preset: opts?.preset || undefined,
|
|
1211
|
-
...(Number.isFinite(callTimeout) && callTimeout > 0 ? { idleTimeoutMs: callTimeout } : {}),
|
|
1212
|
-
})
|
|
1213
|
-
}
|
|
1214
|
-
}
|
|
1215
|
-
|
|
1216
|
-
let _cycle2AgentDispatch = null
|
|
1217
|
-
function getCycle2CallLlm() {
|
|
1218
|
-
if (!_cycle2AgentDispatch) {
|
|
1219
|
-
_cycle2AgentDispatch = makeAgentDispatch({
|
|
1220
|
-
agent: 'cycle2-agent',
|
|
1221
|
-
taskType: 'maintenance',
|
|
1222
|
-
sourceType: 'memory-cycle',
|
|
1223
|
-
brief: false,
|
|
1224
|
-
})
|
|
1225
|
-
}
|
|
1226
|
-
return async (opts = {}, userMessage) => {
|
|
1227
|
-
const callTimeout = Number(opts?.timeout)
|
|
1228
|
-
return _cycle2AgentDispatch({
|
|
1229
|
-
prompt: String(userMessage ?? ''),
|
|
1230
|
-
preset: opts?.preset || undefined,
|
|
1231
|
-
...(Number.isFinite(callTimeout) && callTimeout > 0 ? { idleTimeoutMs: callTimeout } : {}),
|
|
1232
|
-
})
|
|
1233
|
-
}
|
|
1234
|
-
}
|
|
1235
|
-
|
|
1236
|
-
let _cycle3AgentDispatch = null
|
|
1237
|
-
function getCycle3CallLlm() {
|
|
1238
|
-
if (!_cycle3AgentDispatch) {
|
|
1239
|
-
_cycle3AgentDispatch = makeAgentDispatch({
|
|
1240
|
-
agent: 'cycle3-agent',
|
|
1241
|
-
taskType: 'maintenance',
|
|
1242
|
-
sourceType: 'memory-cycle',
|
|
1243
|
-
brief: false,
|
|
1244
|
-
})
|
|
531
|
+
.catch((err) => __mixdogMemoryLog(`[embed] post-ingest raw flush failed: ${err?.message || err}\n`))
|
|
532
|
+
_rawEmbedFlushChain = run
|
|
533
|
+
let timer
|
|
534
|
+
await Promise.race([
|
|
535
|
+
run,
|
|
536
|
+
new Promise((resolve) => { timer = setTimeout(resolve, INGEST_EMBED_WAIT_MS) }),
|
|
537
|
+
]).finally(() => clearTimeout(timer))
|
|
1245
538
|
}
|
|
1246
|
-
return
|
|
1247
|
-
const callTimeout = Number(opts?.timeout)
|
|
1248
|
-
return _cycle3AgentDispatch({
|
|
1249
|
-
prompt: String(userMessage ?? ''),
|
|
1250
|
-
preset: opts?.preset || undefined,
|
|
1251
|
-
...(Number.isFinite(callTimeout) && callTimeout > 0 ? { idleTimeoutMs: callTimeout } : {}),
|
|
1252
|
-
})
|
|
1253
|
-
}
|
|
1254
|
-
}
|
|
1255
|
-
|
|
1256
|
-
async function recordCycle1Result(result) {
|
|
1257
|
-
const now = Date.now()
|
|
1258
|
-
await setCycleLastRun('cycle1_heartbeat', now)
|
|
1259
|
-
const skipped = result?.skippedInFlight === true
|
|
1260
|
-
const coalescedNoop = result?.coalescedRetryNoop === true
|
|
1261
|
-
const allFailed = !skipped
|
|
1262
|
-
&& Number(result?.chunks ?? 0) === 0
|
|
1263
|
-
&& Number(result?.processed ?? 0) === 0
|
|
1264
|
-
&& Number(result?.skipped ?? 0) > 0
|
|
1265
|
-
if (!skipped && !coalescedNoop && !allFailed) {
|
|
1266
|
-
await setCycleLastRun('cycle1', now)
|
|
1267
|
-
}
|
|
1268
|
-
if (!skipped && !coalescedNoop) markCycleDone('cycle1', !allFailed, allFailed ? 'all rows skipped' : null)
|
|
539
|
+
return { text: `ingest_session: considered=${considered} inserted=${inserted} session=${sessionId}` }
|
|
1269
540
|
}
|
|
1270
541
|
|
|
1271
|
-
// ── Cycle
|
|
1272
|
-
//
|
|
1273
|
-
//
|
|
1274
|
-
//
|
|
1275
|
-
//
|
|
542
|
+
// ── Cycle scheduling cluster (extracted to lib/cycle-scheduler.mjs) ────────
|
|
543
|
+
// The mutually-referential cycle machinery (health ledger, cycle1 outer
|
|
544
|
+
// coalesce layer, scheduled enqueue/retry paths, checkCycles, tick loop) lives
|
|
545
|
+
// in the factory below. index.mjs keeps lifecycle ownership by injecting live
|
|
546
|
+
// getters (getDb/getConfig/setConfig) plus runners and LLM adapters.
|
|
1276
547
|
const CYCLE_STATE_FILE = path.join(DATA_DIR, 'memory-cycle-state.json')
|
|
1277
|
-
const _cycleHealth = {
|
|
1278
|
-
cycle1: { last_success_at: 0, last_error_at: 0, last_error: null, consecutive_failures: 0 },
|
|
1279
|
-
cycle2: { last_success_at: 0, last_error_at: 0, last_error: null, consecutive_failures: 0 },
|
|
1280
|
-
cycle3: { last_success_at: 0, last_error_at: 0, last_error: null, consecutive_failures: 0 },
|
|
1281
|
-
}
|
|
1282
|
-
let _cycleRunning = null // { cycle, started_at }
|
|
1283
|
-
let _cycleBacklogSnapshot = { unchunked: 0, cycle2_pending: 0, at: 0 }
|
|
1284
|
-
let _lastBacklogWarnAt = 0
|
|
1285
|
-
const BACKLOG_WARN_COOLDOWN_MS = 10 * 60_000
|
|
1286
|
-
const BACKLOG_WARN_PENDING = 500
|
|
1287
|
-
const BACKLOG_WARN_FAILURES = 5
|
|
1288
|
-
|
|
1289
|
-
function _writeCycleStateFile() {
|
|
1290
|
-
try {
|
|
1291
|
-
fs.writeFileSync(CYCLE_STATE_FILE, JSON.stringify({
|
|
1292
|
-
running: _cycleRunning,
|
|
1293
|
-
backlog: _cycleBacklogSnapshot,
|
|
1294
|
-
cycles: _cycleHealth,
|
|
1295
|
-
updatedAt: Date.now(),
|
|
1296
|
-
}))
|
|
1297
|
-
} catch { /* best-effort; statusline just shows nothing */ }
|
|
1298
|
-
}
|
|
1299
|
-
|
|
1300
|
-
function markCycleRunning(cycle) {
|
|
1301
|
-
_cycleRunning = { cycle, started_at: Date.now() }
|
|
1302
|
-
_writeCycleStateFile()
|
|
1303
|
-
}
|
|
1304
|
-
|
|
1305
|
-
function markCycleDone(cycle, ok, err = null) {
|
|
1306
|
-
const h = _cycleHealth[cycle]
|
|
1307
|
-
if (h) {
|
|
1308
|
-
const now = Date.now()
|
|
1309
|
-
if (ok) { h.last_success_at = now; h.consecutive_failures = 0; h.last_error = null }
|
|
1310
|
-
else { h.last_error_at = now; h.consecutive_failures += 1; h.last_error = String(err || 'unknown').slice(0, 200) }
|
|
1311
|
-
if (!ok && h.consecutive_failures >= BACKLOG_WARN_FAILURES) {
|
|
1312
|
-
_warnCycleHealth(`${cycle} failing repeatedly (consecutive=${h.consecutive_failures}, last="${h.last_error}")`)
|
|
1313
|
-
}
|
|
1314
|
-
}
|
|
1315
|
-
if (_cycleRunning?.cycle === cycle) _cycleRunning = null
|
|
1316
|
-
_writeCycleStateFile()
|
|
1317
|
-
}
|
|
1318
|
-
|
|
1319
|
-
function _warnCycleHealth(msg) {
|
|
1320
|
-
const now = Date.now()
|
|
1321
|
-
if (now - _lastBacklogWarnAt < BACKLOG_WARN_COOLDOWN_MS) return
|
|
1322
|
-
_lastBacklogWarnAt = now
|
|
1323
|
-
__mixdogMemoryLog(`[cycle-health] WARN ${msg}\n`)
|
|
1324
|
-
}
|
|
1325
|
-
|
|
1326
|
-
function _startCycle1Run(config = {}, options = {}) {
|
|
1327
|
-
// Default to the in-process agent dispatch so every cycle1 path — scheduled,
|
|
1328
|
-
// auto-restart, periodic, manual (action:cycle1), backfill drain, rebuild —
|
|
1329
|
-
// dispatches locally and the dead IPC fallback in memory-cycle1.mjs is never
|
|
1330
|
-
// reached. Explicit options.callLlm (if a caller ever passes one) wins.
|
|
1331
|
-
if (typeof options?.callLlm !== 'function') {
|
|
1332
|
-
options = { ...options, callLlm: getCycle1CallLlm() }
|
|
1333
|
-
}
|
|
1334
|
-
markCycleRunning('cycle1')
|
|
1335
|
-
_cycle1InFlight = (async () => {
|
|
1336
|
-
try {
|
|
1337
|
-
const result = await runCycle1(db, config, options, DATA_DIR)
|
|
1338
|
-
// #13: heartbeat (attempt) is always recorded so the overdue check
|
|
1339
|
-
// can tell the keeper is alive; success timestamp only advances when
|
|
1340
|
-
// the run actually did work. Skipped/in-flight runs do NOT count as
|
|
1341
|
-
// success because the next overdue check would otherwise see a fake
|
|
1342
|
-
// green and stop forcing auto-restarts.
|
|
1343
|
-
if (typeof options?.onCoalescedSuccess !== 'function') {
|
|
1344
|
-
await recordCycle1Result(result)
|
|
1345
|
-
}
|
|
1346
|
-
return result
|
|
1347
|
-
} catch (err) {
|
|
1348
|
-
markCycleDone('cycle1', false, err?.message || err)
|
|
1349
|
-
throw err
|
|
1350
|
-
} finally {
|
|
1351
|
-
if (_cycleRunning?.cycle === 'cycle1') { _cycleRunning = null; _writeCycleStateFile() }
|
|
1352
|
-
if (_cycle1InFlight === promise) _cycle1InFlight = null
|
|
1353
|
-
}
|
|
1354
|
-
})()
|
|
1355
|
-
const promise = _cycle1InFlight
|
|
1356
|
-
return _cycle1InFlight
|
|
1357
|
-
}
|
|
1358
|
-
|
|
1359
|
-
async function _awaitCycle1Run(config = {}, options = {}) {
|
|
1360
|
-
const target = _cycle1InFlight || _startCycle1Run(config, options)
|
|
1361
|
-
const callerDeadlineMs = Number(options.callerDeadlineMs) || 0
|
|
1362
|
-
if (callerDeadlineMs <= 0) return await target
|
|
1363
|
-
// Caller-deadline race. When the channels-side timeout fires, we
|
|
1364
|
-
// (a) graceful-return a skippedInFlight envelope so the calling
|
|
1365
|
-
// SessionStart slot stops blocking with a 200 OK + flags instead of a
|
|
1366
|
-
// 503-class throw, and (b) release the outer in-flight handle. The
|
|
1367
|
-
// underlying LLM run keeps progressing in the background — it still
|
|
1368
|
-
// owns the inner dedup guard (memory-cycle.mjs _runCycle1InFlight).
|
|
1369
|
-
// Releasing the outer handle is what breaks the cascade: any later
|
|
1370
|
-
// _awaitCycle1Run call now re-enters _startCycle1Run, whose inner
|
|
1371
|
-
// runCycle1 short-circuits with skippedInFlight:true the moment it
|
|
1372
|
-
// sees the same db still busy. Returning a graceful object (vs the
|
|
1373
|
-
// pre-0.1.198 throw) keeps the channel route response shape stable
|
|
1374
|
-
// and lets pollers read inFlight=true rather than parse an error.
|
|
1375
|
-
let timer
|
|
1376
|
-
const deadlinePromise = new Promise((resolve) => {
|
|
1377
|
-
timer = setTimeout(() => {
|
|
1378
|
-
// Do NOT clear _cycle1InFlight here — the underlying runCycle1 is still
|
|
1379
|
-
// running in the background and owns that slot; its own `finally`
|
|
1380
|
-
// (_startCycle1Run above) clears it when the real work settles. Clearing
|
|
1381
|
-
// it early made a later unbounded drain (deadlineMs:0, e.g. the query
|
|
1382
|
-
// recall path) re-enter _startCycle1Run and spawn a SECOND concurrent
|
|
1383
|
-
// cycle1 run instead of awaiting the one already in flight, defeating
|
|
1384
|
-
// drain-to-zero silently. The caller here just stops WAITING at the
|
|
1385
|
-
// deadline; state ownership stays with the promise itself.
|
|
1386
|
-
resolve({
|
|
1387
|
-
processed: 0,
|
|
1388
|
-
chunks: 0,
|
|
1389
|
-
skipped: 0,
|
|
1390
|
-
sessions: 0,
|
|
1391
|
-
skippedInFlight: true,
|
|
1392
|
-
timedOutWaiting: true,
|
|
1393
|
-
callerDeadlineMs,
|
|
1394
|
-
})
|
|
1395
|
-
}, callerDeadlineMs)
|
|
1396
|
-
})
|
|
1397
|
-
try {
|
|
1398
|
-
return await Promise.race([target, deadlinePromise])
|
|
1399
|
-
} finally {
|
|
1400
|
-
clearTimeout(timer)
|
|
1401
|
-
}
|
|
1402
|
-
}
|
|
1403
|
-
|
|
1404
|
-
// Periodic cycle1 sizing: only enter when ≥ 20 pending rows have built up,
|
|
1405
|
-
// then process up to 2 sessions × 50 rows per tick. cycle1 itself keeps each
|
|
1406
|
-
// classifier window session-isolated. The on-demand path used by SessionStart
|
|
1407
|
-
// hooks runs with a 1-row threshold and 5×20 windows instead — see
|
|
1408
|
-
// hooks/session-start.cjs ON_DEMAND_CYCLE1_ARGS.
|
|
1409
|
-
// mainConfig.cycle1 values still win, so users can override any of these in
|
|
1410
|
-
// config.json.
|
|
1411
|
-
function periodicCycle1Config() {
|
|
1412
|
-
return {
|
|
1413
|
-
min_batch: 20,
|
|
1414
|
-
session_cap: 2,
|
|
1415
|
-
batch_size: 50,
|
|
1416
|
-
concurrency: 2,
|
|
1417
|
-
...(mainConfig?.cycle1 || {}),
|
|
1418
|
-
}
|
|
1419
|
-
}
|
|
1420
|
-
|
|
1421
|
-
async function enqueueScheduledCycle(kind, intervalMs, signature) {
|
|
1422
|
-
const claim = await claimAndMarkScheduledCycle(db, kind, intervalMs, signature, { reason: 'scheduled' })
|
|
1423
|
-
return claim.claimed === true
|
|
1424
|
-
}
|
|
1425
548
|
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
}
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
}
|
|
1461
|
-
const result = await _awaitCycle1Run(config, {
|
|
1462
|
-
coalescedRetry: true,
|
|
1463
|
-
onCoalescedSuccess: recordCycle1Result,
|
|
1464
|
-
})
|
|
1465
|
-
if (result?.skippedInFlight) scheduleScheduledCycle1(config, signature, attempt + 1)
|
|
1466
|
-
}, config, signature)
|
|
1467
|
-
}
|
|
1468
|
-
|
|
1469
|
-
function scheduleScheduledCycle2(config, signature, attempt = 0) {
|
|
1470
|
-
const maxRetries = resolveCoalesceMaxRetries(config, 3)
|
|
1471
|
-
if (attempt > maxRetries) {
|
|
1472
|
-
__mixdogMemoryLog('[cycle2] scheduled queue retry cap reached\n')
|
|
1473
|
-
return
|
|
1474
|
-
}
|
|
1475
|
-
scheduleCoalescedCycleRetry(db, 'cycle2', async () => {
|
|
1476
|
-
if (_cycle2InFlight) {
|
|
1477
|
-
scheduleScheduledCycle2(config, signature, attempt + 1)
|
|
1478
|
-
return
|
|
1479
|
-
}
|
|
1480
|
-
_cycle2InFlight = true
|
|
1481
|
-
markCycleRunning('cycle2')
|
|
1482
|
-
try {
|
|
1483
|
-
let c2Options = {
|
|
1484
|
-
coalescedRetry: true,
|
|
1485
|
-
onCoalescedSuccess: _finalizeCycle2Run,
|
|
1486
|
-
}
|
|
1487
|
-
if (typeof c2Options?.callLlm !== 'function') {
|
|
1488
|
-
c2Options = { ...c2Options, callLlm: getCycle2CallLlm() }
|
|
1489
|
-
}
|
|
1490
|
-
const result = await runCycle2(db, config, c2Options, DATA_DIR)
|
|
1491
|
-
if (result?.skippedInFlight) {
|
|
1492
|
-
scheduleScheduledCycle2(config, signature, attempt + 1)
|
|
1493
|
-
} else if (result?.coalescedRetryNoop) {
|
|
1494
|
-
__mixdogMemoryLog('[cycle2] scheduled queue noop\n')
|
|
1495
|
-
} else if (result?.ok === false) {
|
|
1496
|
-
await _finalizeCycle2Run(result)
|
|
1497
|
-
}
|
|
1498
|
-
} catch (err) {
|
|
1499
|
-
__mixdogMemoryLog(`[cycle2] scheduled queue failed: ${err?.message || err}\n`)
|
|
1500
|
-
markCycleDone('cycle2', false, err?.message || err)
|
|
1501
|
-
} finally {
|
|
1502
|
-
_cycle2InFlight = false
|
|
1503
|
-
if (_cycleRunning?.cycle === 'cycle2') { _cycleRunning = null; _writeCycleStateFile() }
|
|
1504
|
-
}
|
|
1505
|
-
}, config, signature)
|
|
1506
|
-
}
|
|
1507
|
-
|
|
1508
|
-
function scheduleScheduledCycle3(config, signature, attempt = 0) {
|
|
1509
|
-
const retryConfig = config?.cycle3 || config
|
|
1510
|
-
const maxRetries = resolveCoalesceMaxRetries(retryConfig, 3)
|
|
1511
|
-
if (attempt > maxRetries) {
|
|
1512
|
-
__mixdogMemoryLog('[cycle3] scheduled queue retry cap reached\n')
|
|
1513
|
-
return
|
|
1514
|
-
}
|
|
1515
|
-
scheduleCoalescedCycleRetry(db, 'cycle3', async () => {
|
|
1516
|
-
if (_cycle3InFlight) {
|
|
1517
|
-
scheduleScheduledCycle3(config, signature, attempt + 1)
|
|
1518
|
-
return
|
|
1519
|
-
}
|
|
1520
|
-
_cycle3InFlight = true
|
|
1521
|
-
markCycleRunning('cycle3')
|
|
1522
|
-
try {
|
|
1523
|
-
let c3Options = {
|
|
1524
|
-
coalescedRetry: true,
|
|
1525
|
-
onCoalescedSuccess: async () => { await setCycleLastRun('cycle3', Date.now()); markCycleDone('cycle3', true) },
|
|
1526
|
-
}
|
|
1527
|
-
if (typeof c3Options?.callLlm !== 'function') {
|
|
1528
|
-
c3Options = { ...c3Options, callLlm: getCycle3CallLlm() }
|
|
1529
|
-
}
|
|
1530
|
-
const result = await runCycle3(db, config, DATA_DIR, c3Options)
|
|
1531
|
-
if (result?.skippedInFlight) {
|
|
1532
|
-
scheduleScheduledCycle3(config, signature, attempt + 1)
|
|
1533
|
-
} else if (result?.coalescedRetryNoop) {
|
|
1534
|
-
__mixdogMemoryLog('[cycle3] scheduled queue noop\n')
|
|
1535
|
-
}
|
|
1536
|
-
} catch (err) {
|
|
1537
|
-
__mixdogMemoryLog(`[cycle3] scheduled queue failed: ${err?.message || err}\n`)
|
|
1538
|
-
markCycleDone('cycle3', false, err?.message || err)
|
|
1539
|
-
} finally {
|
|
1540
|
-
_cycle3InFlight = false
|
|
1541
|
-
if (_cycleRunning?.cycle === 'cycle3') { _cycleRunning = null; _writeCycleStateFile() }
|
|
1542
|
-
}
|
|
1543
|
-
}, retryConfig, signature)
|
|
1544
|
-
}
|
|
1545
|
-
|
|
1546
|
-
async function _finalizeCycle2Run(result) {
|
|
1547
|
-
if (result?.skippedInFlight) {
|
|
1548
|
-
__mixdogMemoryLog('[cycle2] skipped: in flight\n')
|
|
1549
|
-
return
|
|
1550
|
-
}
|
|
1551
|
-
if (result.ok) {
|
|
1552
|
-
await setCycleLastRun('cycle2', Date.now())
|
|
1553
|
-
await setCycleLastRun('cycle2_last_error', '')
|
|
1554
|
-
__mixdogMemoryLog('[cycle2] completed\n')
|
|
1555
|
-
markCycleDone('cycle2', true)
|
|
1556
|
-
} else {
|
|
1557
|
-
await setCycleLastRun('cycle2_last_error', result.error || 'unknown error')
|
|
1558
|
-
__mixdogMemoryLog(`[cycle2] failed: ${result.error}\n`)
|
|
1559
|
-
markCycleDone('cycle2', false, result.error || 'unknown error')
|
|
1560
|
-
}
|
|
1561
|
-
}
|
|
1562
|
-
|
|
1563
|
-
async function checkCycles() {
|
|
1564
|
-
// Poll-on-use: re-read memory config each tick so changed enabled/interval
|
|
1565
|
-
// values apply without a restart (mirrors search/cwd poll-on-use). The fixed
|
|
1566
|
-
// 60s poll bounds latency; manual `memory` tool calls already re-read per-call.
|
|
1567
|
-
mainConfig = readMainConfig();
|
|
1568
|
-
if (mainConfig?.enabled === false) return
|
|
1569
|
-
|
|
1570
|
-
const cycle1Ms = parseInterval(mainConfig?.cycle1?.interval || '10m')
|
|
1571
|
-
const cycle2Ms = parseInterval(mainConfig?.cycle2?.interval || '1h')
|
|
1572
|
-
const cycle3Ms = parseInterval(mainConfig?.cycle3?.interval || '24h')
|
|
1573
|
-
|
|
1574
|
-
const now = Date.now()
|
|
1575
|
-
const last = await getCycleLastRun()
|
|
1576
|
-
|
|
1577
|
-
// Phase B §2.4 — cache-keeper health check + auto-restart.
|
|
1578
|
-
//
|
|
1579
|
-
// `last.cycle1 + cycle1Ms` is the next scheduled run time; anything beyond
|
|
1580
|
-
// that by > HEALTH_OVERDUE_MS means the keeper missed its window and the
|
|
1581
|
-
// Anthropic shard is drifting cold. Emit a warning, and — if we haven't
|
|
1582
|
-
// retried in the last cooldown window — force an unscheduled run so the
|
|
1583
|
-
// shard gets re-touched before the next Worker / Sub call pays the 2×
|
|
1584
|
-
// write premium. Cooldown prevents a tight retry loop when the underlying
|
|
1585
|
-
// cause (network, provider outage) is still broken.
|
|
1586
|
-
//
|
|
1587
|
-
// Cold-start guard: a fresh DB has last.cycle1 = 0, which would make
|
|
1588
|
-
// (now - 0 - cycle1Ms) blow past HEALTH_OVERDUE_MS on every first boot
|
|
1589
|
-
// and force-trigger the auto-restart branch even though the shard never
|
|
1590
|
-
// existed in the first place. The "drifting cold" concept doesn't apply
|
|
1591
|
-
// until at least one successful run has anchored a baseline.
|
|
1592
|
-
const cycle1OverdueMs = last.cycle1 > 0
|
|
1593
|
-
? Math.max(0, now - last.cycle1 - cycle1Ms)
|
|
1594
|
-
: 0
|
|
1595
|
-
if (cycle1OverdueMs > CYCLE1_HEALTH_OVERDUE_MS) {
|
|
1596
|
-
const lastSeen = last.cycle1 ? new Date(last.cycle1).toISOString() : 'never'
|
|
1597
|
-
__mixdogMemoryLog(
|
|
1598
|
-
`[cycle1] overdue by ${Math.floor(cycle1OverdueMs / 60_000)}min `
|
|
1599
|
-
+ `(last=${lastSeen}). Pool B Anthropic shard may be cold.\n`
|
|
1600
|
-
)
|
|
1601
|
-
const lastAutoRestart = last.cycle1_autoRestart || 0
|
|
1602
|
-
if (now - lastAutoRestart >= CYCLE1_AUTO_RESTART_COOLDOWN_MS) {
|
|
1603
|
-
// #14: record the attempt timestamp BEFORE the call (so a hung run
|
|
1604
|
-
// cannot tight-loop) and the result timestamp only on success. On
|
|
1605
|
-
// failure we return immediately instead of falling through into the
|
|
1606
|
-
// due branch — falling through would silently re-enter the same
|
|
1607
|
-
// failing path within the same tick.
|
|
1608
|
-
await setCycleLastRun('cycle1_autoRestart_attempt', now)
|
|
1609
|
-
try {
|
|
1610
|
-
const result = await _awaitCycle1Run(periodicCycle1Config())
|
|
1611
|
-
await setCycleLastRun('cycle1_autoRestart', Date.now())
|
|
1612
|
-
__mixdogMemoryLog(
|
|
1613
|
-
`[cycle1] auto-restart completed chunks=${result?.chunks ?? 0} processed=${result?.processed ?? 0}\n`
|
|
1614
|
-
)
|
|
1615
|
-
return
|
|
1616
|
-
} catch (e) {
|
|
1617
|
-
__mixdogMemoryLog(`[cycle1] auto-restart error: ${e.message}\n`)
|
|
1618
|
-
// Cooldown attempt timestamp is committed; do NOT fall through
|
|
1619
|
-
// to the due branch — next tick will retry after cooldown.
|
|
1620
|
-
return
|
|
1621
|
-
}
|
|
1622
|
-
}
|
|
1623
|
-
}
|
|
1624
|
-
|
|
1625
|
-
if (now - last.cycle1 >= cycle1Ms) {
|
|
1626
|
-
await enqueueScheduledCycle1(cycle1Ms, 'scheduled')
|
|
1627
|
-
}
|
|
1628
|
-
|
|
1629
|
-
if (now - last.cycle2 >= cycle2Ms) {
|
|
1630
|
-
await enqueueScheduledCycle2(cycle2Ms, 'scheduled')
|
|
1631
|
-
}
|
|
1632
|
-
|
|
1633
|
-
if (now - last.cycle3 >= cycle3Ms) {
|
|
1634
|
-
await enqueueScheduledCycle3(cycle3Ms, 'scheduled')
|
|
1635
|
-
}
|
|
549
|
+
const _cycleLlmAdapters = createCycleLlmAdapters({ makeAgentDispatch })
|
|
550
|
+
const { getCycle1CallLlm, getCycle2CallLlm, getCycle3CallLlm } = _cycleLlmAdapters
|
|
551
|
+
|
|
552
|
+
const _cycleScheduler = createCycleScheduler({
|
|
553
|
+
getDb: () => db,
|
|
554
|
+
getConfig: () => mainConfig,
|
|
555
|
+
setConfig: (cfg) => { mainConfig = cfg },
|
|
556
|
+
dataDir: DATA_DIR,
|
|
557
|
+
log: __mixdogMemoryLog,
|
|
558
|
+
getCycleLastRun,
|
|
559
|
+
setCycleLastRun,
|
|
560
|
+
readMainConfig,
|
|
561
|
+
memoryCyclesEnabled,
|
|
562
|
+
getCycle1CallLlm,
|
|
563
|
+
getCycle2CallLlm,
|
|
564
|
+
getCycle3CallLlm,
|
|
565
|
+
runCycle1,
|
|
566
|
+
runCycle2,
|
|
567
|
+
runCycle3,
|
|
568
|
+
parseInterval,
|
|
569
|
+
flushRawEmbeddings,
|
|
570
|
+
getInFlightCycle1,
|
|
571
|
+
claimAndMarkScheduledCycle,
|
|
572
|
+
resolveCoalesceMaxRetries,
|
|
573
|
+
scheduleCoalescedCycleRetry,
|
|
574
|
+
scheduledCycle1Signature,
|
|
575
|
+
scheduledCycle2Signature,
|
|
576
|
+
scheduledCycle3Signature,
|
|
577
|
+
cycleStateFile: CYCLE_STATE_FILE,
|
|
578
|
+
})
|
|
579
|
+
// Cycle1 run primitives + cycle2 finalize used by MCP action handlers below.
|
|
580
|
+
const _startCycle1Run = _cycleScheduler.startCycle1Run
|
|
581
|
+
const _awaitCycle1Run = _cycleScheduler.awaitCycle1Run
|
|
582
|
+
const _finalizeCycle2Run = _cycleScheduler.finalizeCycle2Run
|
|
1636
583
|
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
// building a backlog, and if a cycle has not SUCCEEDED for 2h+ while its
|
|
1640
|
-
// backlog is over threshold, kick one unscheduled run now instead of
|
|
1641
|
-
// waiting for the interval — a stalled cycle otherwise hides behind
|
|
1642
|
-
// "the schedule ran it" (attempt != success; see 2026-07 cycle2 stall).
|
|
1643
|
-
try {
|
|
1644
|
-
const unchunked = Number((await db.query(
|
|
1645
|
-
`SELECT COUNT(*) c FROM entries WHERE chunk_root IS NULL AND NULLIF(btrim(session_id), '') IS NOT NULL`,
|
|
1646
|
-
)).rows[0]?.c ?? 0)
|
|
1647
|
-
const cycle2Pending = Number((await db.query(
|
|
1648
|
-
`SELECT COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'pending'`,
|
|
1649
|
-
)).rows[0]?.c ?? 0)
|
|
1650
|
-
_cycleBacklogSnapshot = { unchunked, cycle2_pending: cycle2Pending, at: now }
|
|
1651
|
-
_writeCycleStateFile()
|
|
1652
|
-
if (unchunked > BACKLOG_WARN_PENDING || cycle2Pending > BACKLOG_WARN_PENDING) {
|
|
1653
|
-
_warnCycleHealth(`backlog unchunked=${unchunked} cycle2_pending=${cycle2Pending}`)
|
|
1654
|
-
}
|
|
1655
|
-
const SELF_KICK_STALE_MS = 2 * 3600_000
|
|
1656
|
-
const c1Stale = (_cycleHealth.cycle1.last_success_at || last.cycle1 || 0) < now - SELF_KICK_STALE_MS
|
|
1657
|
-
const c2Stale = (_cycleHealth.cycle2.last_success_at || last.cycle2 || 0) < now - SELF_KICK_STALE_MS
|
|
1658
|
-
if (c1Stale && unchunked > BACKLOG_WARN_PENDING) {
|
|
1659
|
-
_warnCycleHealth(`cycle1 self-kick: no success 2h+, unchunked=${unchunked}`)
|
|
1660
|
-
await enqueueScheduledCycle1(0, 'self-kick')
|
|
1661
|
-
} else if (c2Stale && cycle2Pending > BACKLOG_WARN_PENDING) {
|
|
1662
|
-
_warnCycleHealth(`cycle2 self-kick: no success 2h+, pending=${cycle2Pending}`)
|
|
1663
|
-
await enqueueScheduledCycle2(0, 'self-kick')
|
|
1664
|
-
}
|
|
1665
|
-
} catch { /* counts are best-effort; never fail the tick */ }
|
|
1666
|
-
}
|
|
1667
|
-
let _cycle2InFlight = false
|
|
1668
|
-
let _cycle3InFlight = false
|
|
1669
|
-
|
|
1670
|
-
// #12: self-rescheduling timer. setInterval would fire ticks regardless of
|
|
1671
|
-
// whether the previous checkCycles() call had finished; with cycle1/cycle2
|
|
1672
|
-
// each potentially taking minutes, that races. Use setTimeout that re-arms
|
|
1673
|
-
// itself only after the prior tick resolves, plus an in-flight guard so a
|
|
1674
|
-
// stray manual call cannot stack ticks.
|
|
1675
|
-
let _checkCyclesInFlight = false
|
|
1676
|
-
async function _runCheckCyclesGuarded() {
|
|
1677
|
-
if (_checkCyclesInFlight) return
|
|
1678
|
-
_checkCyclesInFlight = true
|
|
1679
|
-
try { await checkCycles() }
|
|
1680
|
-
catch (e) { __mixdogMemoryLog(`[cycle-tick] error: ${e.message}\n`) }
|
|
1681
|
-
finally { _checkCyclesInFlight = false }
|
|
1682
|
-
}
|
|
1683
|
-
function _scheduleNextCheck() {
|
|
1684
|
-
_cycleInterval = setTimeout(async () => {
|
|
1685
|
-
_cycleInterval = null
|
|
1686
|
-
try {
|
|
1687
|
-
await _runCheckCyclesGuarded()
|
|
1688
|
-
} catch (e) {
|
|
1689
|
-
__mixdogMemoryLog(`[cycle-tick] re-arm guard caught: ${e?.message || e}\n`)
|
|
1690
|
-
} finally {
|
|
1691
|
-
// Re-arm regardless of inner outcome — _runCheckCyclesGuarded already
|
|
1692
|
-
// swallows its own errors, but defensive try/finally guarantees the
|
|
1693
|
-
// periodic tick continues even if a synchronous throw escapes.
|
|
1694
|
-
if (_cyclesActive) _scheduleNextCheck()
|
|
1695
|
-
}
|
|
1696
|
-
}, 60_000)
|
|
1697
|
-
}
|
|
1698
|
-
let _cyclesActive = false
|
|
584
|
+
// Transcript watcher lifecycle stays in the facade (owns _transcriptIngest);
|
|
585
|
+
// the cycle tick loop start/stop is delegated to the scheduler.
|
|
1699
586
|
let _transcriptWatcher = null
|
|
1700
587
|
function _startCycles() {
|
|
1701
|
-
|
|
1702
|
-
_cyclesActive = true
|
|
1703
|
-
_scheduleNextCheck()
|
|
1704
|
-
_startupTimeout = setTimeout(() => { void _runCheckCyclesGuarded() }, 30_000)
|
|
588
|
+
_cycleScheduler.startCycles()
|
|
1705
589
|
}
|
|
1706
590
|
|
|
1707
591
|
function _stopCycles() {
|
|
1708
|
-
|
|
1709
|
-
if (_cycleInterval) { clearTimeout(_cycleInterval); _cycleInterval = null }
|
|
1710
|
-
if (_startupTimeout) { clearTimeout(_startupTimeout); _startupTimeout = null }
|
|
592
|
+
_cycleScheduler.stopCycles()
|
|
1711
593
|
if (_transcriptWatcher) { try { _transcriptWatcher.stop() } catch {} _transcriptWatcher = null }
|
|
1712
594
|
}
|
|
1713
595
|
|
|
@@ -1718,17 +600,36 @@ async function _initRuntime() {
|
|
|
1718
600
|
// increments, so deleted rows leave permanent gaps. Fast no-op when already
|
|
1719
601
|
// contiguous (or empty). Runs only here — never in cycle2/addCore/deleteCore.
|
|
1720
602
|
await compactCoreIds(DATA_DIR)
|
|
1721
|
-
|
|
1722
|
-
|
|
603
|
+
// Memory module is always-on: the transcript watcher/ingest runs
|
|
604
|
+
// unconditionally except in secondary mode (secondary attaches to a primary's
|
|
605
|
+
// PG and must not double-ingest). The recap toggle only gates whether the
|
|
606
|
+
// background cycles actually schedule work — but we still start the tick loop
|
|
607
|
+
// so recap can be toggled ON at runtime (checkCycles polls recap each tick and
|
|
608
|
+
// no-ops while recap is off). The env hard-override / secondary mode skip the
|
|
609
|
+
// tick loop entirely.
|
|
610
|
+
if (!memorySecondaryMode()) {
|
|
611
|
+
_transcriptWatcher = _transcriptIngest.initTranscriptWatcher()
|
|
612
|
+
} else {
|
|
613
|
+
__mixdogMemoryLog('[memory-service] secondary mode; skipping transcript watcher\n')
|
|
614
|
+
}
|
|
615
|
+
if (!memorySecondaryMode() && !envFlagEnabled('MIXDOG_MEMORY_DISABLE_CYCLES')) {
|
|
1723
616
|
_startCycles()
|
|
1724
617
|
} else {
|
|
1725
|
-
__mixdogMemoryLog('[memory-service]
|
|
618
|
+
__mixdogMemoryLog('[memory-service] background cycle tick loop not started (secondary/env-disabled)\n')
|
|
1726
619
|
}
|
|
1727
620
|
_initialized = true
|
|
1728
621
|
// Boot complete — continue straight into the deferred embedding warmup.
|
|
1729
622
|
// Fire-and-forget on the embedding worker thread; never awaited so it does
|
|
1730
623
|
// not delay init() returning or the memory-ready signal.
|
|
1731
|
-
|
|
624
|
+
_embeddingWarmup.fireDeferred()
|
|
625
|
+
// Boot-edge Korean morph warmup. Same fire-and-forget style: lazy async init
|
|
626
|
+
// downloads+caches the Kiwi model once under DATA_DIR/kiwi-model/<version>,
|
|
627
|
+
// then builds the WASM analyzer. On any failure ko-morph stays null and
|
|
628
|
+
// buildFtsQuery keeps the websearch_to_tsquery fallback (never throws here).
|
|
629
|
+
// Skipped in secondary mode (query path lives on the primary).
|
|
630
|
+
if (!memorySecondaryMode()) {
|
|
631
|
+
initKoMorph(DATA_DIR, __mixdogMemoryLog).catch(() => {})
|
|
632
|
+
}
|
|
1732
633
|
}
|
|
1733
634
|
|
|
1734
635
|
function _beginRuntimeInit() {
|
|
@@ -1743,661 +644,22 @@ function _beginRuntimeInit() {
|
|
|
1743
644
|
return _initPromise
|
|
1744
645
|
}
|
|
1745
646
|
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
.filter(Boolean)
|
|
1763
|
-
if (cats.length > 0) {
|
|
1764
|
-
const placeholders = cats.map((cat) => {
|
|
1765
|
-
params.push(cat)
|
|
1766
|
-
return `$${params.length}`
|
|
1767
|
-
})
|
|
1768
|
-
where.push(`category IN (${placeholders.join(', ')})`)
|
|
1769
|
-
}
|
|
1770
|
-
}
|
|
1771
|
-
|
|
1772
|
-
const textExpr = `lower(coalesce(element, '') || ' ' || coalesce(summary, ''))`
|
|
1773
|
-
const termClauses = terms.map((term) => {
|
|
1774
|
-
params.push(`%${term}%`)
|
|
1775
|
-
return `${textExpr} LIKE $${params.length}`
|
|
1776
|
-
})
|
|
1777
|
-
where.push(`(${termClauses.join(' OR ')})`)
|
|
1778
|
-
const hitExpr = termClauses.map((clause) => `CASE WHEN ${clause} THEN 1 ELSE 0 END`).join(' + ')
|
|
1779
|
-
const rowLimit = Math.max(1, Math.min(10, Number(limit) || 5))
|
|
1780
|
-
params.push(rowLimit)
|
|
1781
|
-
|
|
1782
|
-
const rows = (await db.query(`
|
|
1783
|
-
SELECT id, element, summary, category, project_id, created_at, updated_at,
|
|
1784
|
-
(${hitExpr}) AS hit_count
|
|
1785
|
-
FROM core_entries
|
|
1786
|
-
WHERE ${where.join(' AND ')}
|
|
1787
|
-
ORDER BY hit_count DESC, updated_at DESC, id ASC
|
|
1788
|
-
LIMIT $${params.length}
|
|
1789
|
-
`, params)).rows
|
|
1790
|
-
|
|
1791
|
-
return rows.map((row) => ({
|
|
1792
|
-
...row,
|
|
1793
|
-
id: `core:${row.id}`,
|
|
1794
|
-
ts: row.updated_at || row.created_at || Date.now(),
|
|
1795
|
-
is_root: 1,
|
|
1796
|
-
}))
|
|
1797
|
-
}
|
|
1798
|
-
|
|
1799
|
-
// Session-scoped full drain before a recall(query) search. Chunking is
|
|
1800
|
-
// partitioned by session_id (memory-cycle1.mjs:409-431 selected_sessions
|
|
1801
|
-
// GROUP BY session_id + ROW_NUMBER PARTITION BY session_id), so this drains
|
|
1802
|
-
// ONLY the calling session's backlog via the same session_id override
|
|
1803
|
-
// (memory-cycle1.mjs:394) — other sessions' pending rows are left for the
|
|
1804
|
-
// periodic cycle1 sweep. Reuses recall-fasttrack's drainSessionCycle1
|
|
1805
|
-
// (session/compact.mjs:504) which loops cycle1 until raw rows hit 0 or a
|
|
1806
|
-
// no-progress pass. Safety limits: a 500-row hard cap (both in the pending
|
|
1807
|
-
// pre-check and as the per-call cycle1 batch ceiling) and cooperative
|
|
1808
|
-
// abort-signal propagation through handleMemoryAction — no time budget.
|
|
1809
|
-
// Window 20 rows per classifier prompt × concurrency 5 (haiku — cost is
|
|
1810
|
-
// negligible; 500 rows ≈ 25 prompts ≈ 5 waves).
|
|
1811
|
-
//
|
|
1812
|
-
// Multi-terminal safety: the memory daemon is one shared instance across
|
|
1813
|
-
// terminals (active-instance.json memory_port). Concurrent recall(query)
|
|
1814
|
-
// calls for the SAME session share one in-flight drain promise (dedup);
|
|
1815
|
-
// different sessions drain independently in parallel (cycle1's own
|
|
1816
|
-
// advisory-lock/coalesce guard in memory-cycle1.mjs still serializes actual
|
|
1817
|
-
// DB writes). A drain only gates recall for its OWN session — every other
|
|
1818
|
-
// terminal's recall (different session, or no session) proceeds unblocked.
|
|
1819
|
-
const RECALL_DRAIN_HARD_CAP_ROWS = 500
|
|
1820
|
-
async function _drainSessionForRecallQuery(sessionId, signal, deadlineMs = 0) {
|
|
1821
|
-
if (!sessionId || signal?.aborted) return
|
|
1822
|
-
const existing = _recallDrainInFlight.get(sessionId)
|
|
1823
|
-
if (existing) { try { await existing } catch {} return }
|
|
1824
|
-
let pendingCount = 0
|
|
1825
|
-
try {
|
|
1826
|
-
const r = await db.query(
|
|
1827
|
-
`SELECT COUNT(*) c FROM entries WHERE chunk_root IS NULL AND session_id = $1`,
|
|
1828
|
-
[sessionId],
|
|
1829
|
-
)
|
|
1830
|
-
pendingCount = Number(r.rows?.[0]?.c ?? 0)
|
|
1831
|
-
} catch { return }
|
|
1832
|
-
if (pendingCount <= 0) return
|
|
1833
|
-
const runTool = async (_name, toolArgs) => {
|
|
1834
|
-
const result = await handleMemoryAction(toolArgs, signal)
|
|
1835
|
-
return result?.text ?? ''
|
|
1836
|
-
}
|
|
1837
|
-
const dumpArgs = {
|
|
1838
|
-
action: 'dump_session_roots',
|
|
1839
|
-
sessionId,
|
|
1840
|
-
includeRaw: true,
|
|
1841
|
-
limit: Math.min(RECALL_DRAIN_HARD_CAP_ROWS, Math.max(1, pendingCount)),
|
|
1842
|
-
}
|
|
1843
|
-
const drainPromise = (async () => {
|
|
1844
|
-
try {
|
|
1845
|
-
return await drainSessionCycle1(runTool, {
|
|
1846
|
-
sessionId,
|
|
1847
|
-
dumpArgs,
|
|
1848
|
-
deadlineMs, // 0 = no time budget (query path); bare browse passes a
|
|
1849
|
-
// short bound so a browse call never blocks minutes on LLM chunking.
|
|
1850
|
-
maxPasses: 20, // safety stopper; drainSessionCycle1 also self-stops on no-progress
|
|
1851
|
-
cycleArgs: {
|
|
1852
|
-
min_batch: 1,
|
|
1853
|
-
session_cap: 1,
|
|
1854
|
-
batch_size: RECALL_DRAIN_HARD_CAP_ROWS,
|
|
1855
|
-
rows_per_session: RECALL_DRAIN_HARD_CAP_ROWS,
|
|
1856
|
-
window_size: 20,
|
|
1857
|
-
concurrency: 5,
|
|
1858
|
-
},
|
|
1859
|
-
})
|
|
1860
|
-
} catch (err) {
|
|
1861
|
-
__mixdogMemoryLog(`[recall] session drain aborted/failed (sess=${sessionId}): ${err?.message || err}\n`)
|
|
1862
|
-
return null
|
|
1863
|
-
}
|
|
1864
|
-
})()
|
|
1865
|
-
_recallDrainInFlight.set(sessionId, drainPromise)
|
|
1866
|
-
try {
|
|
1867
|
-
await drainPromise
|
|
1868
|
-
} finally {
|
|
1869
|
-
if (_recallDrainInFlight.get(sessionId) === drainPromise) _recallDrainInFlight.delete(sessionId)
|
|
1870
|
-
}
|
|
1871
|
-
}
|
|
1872
|
-
|
|
1873
|
-
// Bare-browse drain deadline: best-effort only. A browse call ("show me
|
|
1874
|
-
// recent context") must never block on LLM chunking for minutes the way a
|
|
1875
|
-
// query recall is allowed to — the raw-row merge already covers unchunked
|
|
1876
|
-
// content, so the drain here is a nice-to-have upgrade to chunked/scored
|
|
1877
|
-
// rows, not a correctness requirement. Failures/timeouts are silently
|
|
1878
|
-
// tolerated; the raw fallback in the caller applies regardless.
|
|
1879
|
-
const RECALL_BROWSE_DRAIN_DEADLINE_MS = 1800
|
|
1880
|
-
|
|
1881
|
-
async function handleSearch(args, signal) {
|
|
1882
|
-
// Cooperative abort check: throw early if the caller already aborted
|
|
1883
|
-
// (IPC cancel handler signals the AbortController before re-entry).
|
|
1884
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
1885
|
-
// Drain this session's unchunked backlog to zero BEFORE searching, so a
|
|
1886
|
-
// recall never returns raw/unclassified transcript tail instead of the
|
|
1887
|
-
// chunked summary. Runs whenever a session is scoped, query or not — a
|
|
1888
|
-
// bare "show recent" browse hits raw unchunked rows just as often as a
|
|
1889
|
-
// query recall does. Query path keeps the unbounded (deadlineMs:0) drain
|
|
1890
|
-
// since chunked accuracy matters more than latency there; bare browse
|
|
1891
|
-
// gets a short best-effort deadline — it must never block on LLM
|
|
1892
|
-
// chunking for minutes, the raw-row merge covers the fallback either way.
|
|
1893
|
-
{
|
|
1894
|
-
const _drainSessionId = String(args?.sessionId || args?.session_id || '').trim()
|
|
1895
|
-
const _drainHasQuery = Array.isArray(args?.query)
|
|
1896
|
-
? args.query.some((v) => String(v || '').trim())
|
|
1897
|
-
: String(args?.query ?? '').trim() !== ''
|
|
1898
|
-
if (_drainSessionId) {
|
|
1899
|
-
await _drainSessionForRecallQuery(
|
|
1900
|
-
_drainSessionId,
|
|
1901
|
-
signal,
|
|
1902
|
-
_drainHasQuery ? 0 : RECALL_BROWSE_DRAIN_DEADLINE_MS,
|
|
1903
|
-
)
|
|
1904
|
-
}
|
|
1905
|
-
}
|
|
1906
|
-
// #id lookup normalization: search_memories and memory action:'search'
|
|
1907
|
-
// callers pass a single `id` (or an id array under that same key), not
|
|
1908
|
-
// the `ids` array below. Normalize once here so every dispatch path gets
|
|
1909
|
-
// exact-id lookup, not just callers who already knew to use `ids`.
|
|
1910
|
-
if (!Array.isArray(args.ids) && args.id != null) {
|
|
1911
|
-
args = { ...args, ids: Array.isArray(args.id) ? args.id : [args.id] }
|
|
1912
|
-
}
|
|
1913
|
-
if (args?.currentSession === true || args?.sessionId || args?.session_id) {
|
|
1914
|
-
return await recallSessionRows(args)
|
|
1915
|
-
}
|
|
1916
|
-
// id mode (follow-up lookup): caller passed `#N` markers from a prior
|
|
1917
|
-
// recall result. Fetch those rows directly + their chunk members,
|
|
1918
|
-
// bypassing hybrid search entirely. Output reuses renderEntryLines so
|
|
1919
|
-
// the shape stays identical to the search path (chunk members first,
|
|
1920
|
-
// root summary fallback).
|
|
1921
|
-
if (Array.isArray(args.ids) && args.ids.length > 0) {
|
|
1922
|
-
const ids = args.ids
|
|
1923
|
-
.map(v => Number(v))
|
|
1924
|
-
.filter(v => Number.isInteger(v) && v > 0)
|
|
1925
|
-
if (ids.length === 0) return { text: '(no valid ids)' }
|
|
1926
|
-
const includeArchived = args.includeArchived !== false
|
|
1927
|
-
const category = args.category
|
|
1928
|
-
const period = String(args.period ?? '').trim() || undefined
|
|
1929
|
-
const temporal = parsePeriod(period, false)
|
|
1930
|
-
let projectScope
|
|
1931
|
-
if (typeof args.projectScope === 'string' && args.projectScope) {
|
|
1932
|
-
projectScope = args.projectScope
|
|
1933
|
-
} else {
|
|
1934
|
-
const projectId = resolveProjectScope(typeof args.cwd === 'string' && args.cwd ? args.cwd : null)
|
|
1935
|
-
projectScope = projectId !== null ? projectId : 'common'
|
|
1936
|
-
}
|
|
1937
|
-
const excludeStatuses = includeArchived ? [] : ['archived']
|
|
1938
|
-
const rows = await fetchEntriesByIdsScoped(db, ids, {
|
|
1939
|
-
ts_from: temporal?.startMs,
|
|
1940
|
-
ts_to: temporal?.endMs,
|
|
1941
|
-
excludeStatuses,
|
|
1942
|
-
category,
|
|
1943
|
-
projectScope,
|
|
1944
|
-
})
|
|
1945
|
-
if (rows.length === 0) return { text: '(no results)' }
|
|
1946
|
-
// Members for any root rows in the result set.
|
|
1947
|
-
const rootIds = rows.filter(r => r.is_root === 1).map(r => Number(r.id))
|
|
1948
|
-
const memberLeafIds = new Set()
|
|
1949
|
-
if (rootIds.length > 0) {
|
|
1950
|
-
const { rows: memberRows } = await db.query(
|
|
1951
|
-
`SELECT id, ts, role, content, chunk_root
|
|
1952
|
-
FROM entries WHERE chunk_root = ANY($1::bigint[]) AND is_root = 0
|
|
1953
|
-
ORDER BY ts ASC, id ASC`,
|
|
1954
|
-
[rootIds],
|
|
1955
|
-
)
|
|
1956
|
-
const membersByRoot = new Map()
|
|
1957
|
-
for (const m of memberRows) {
|
|
1958
|
-
const k = Number(m.chunk_root)
|
|
1959
|
-
if (!membersByRoot.has(k)) membersByRoot.set(k, [])
|
|
1960
|
-
membersByRoot.get(k).push(m)
|
|
1961
|
-
memberLeafIds.add(Number(m.id))
|
|
1962
|
-
}
|
|
1963
|
-
for (const r of rows) {
|
|
1964
|
-
if (r.is_root === 1) r.members = membersByRoot.get(Number(r.id)) ?? []
|
|
1965
|
-
}
|
|
1966
|
-
}
|
|
1967
|
-
// Preserve caller-supplied id order; drop leaves already inlined as a
|
|
1968
|
-
// root's chunk member to prevent double emission when the caller names
|
|
1969
|
-
// a root and one of its leaves in the same batch.
|
|
1970
|
-
const byId = new Map(rows.map(r => [Number(r.id), r]))
|
|
1971
|
-
const ordered = ids
|
|
1972
|
-
.map(id => byId.get(id))
|
|
1973
|
-
.filter(Boolean)
|
|
1974
|
-
.filter(r => !(r.is_root === 0 && memberLeafIds.has(Number(r.id))))
|
|
1975
|
-
return { text: renderEntryLines(ordered) }
|
|
1976
|
-
}
|
|
1977
|
-
// Array query — fan out in parallel, each query runs its own hybrid search
|
|
1978
|
-
// path, and results are grouped in the response so the caller sees one
|
|
1979
|
-
// ranked list per angle. Collapses what would otherwise be N sequential
|
|
1980
|
-
// tool calls into a single invocation.
|
|
1981
|
-
if (Array.isArray(args.query)) {
|
|
1982
|
-
// Dedup + fan-out cap. The cap protects the result envelope from
|
|
1983
|
-
// over-eager callers (20+ near-duplicate queries N× the IO) without
|
|
1984
|
-
// silently swallowing the caller's intent: when the input exceeds
|
|
1985
|
-
// QUERIES_CAP, prepend a one-line note so the caller can see the
|
|
1986
|
-
// truncation and re-shape their query list.
|
|
1987
|
-
const QUERIES_CAP = 5
|
|
1988
|
-
const dedup = [...new Set(args.query.map(q => String(q || '').trim()).filter(Boolean))]
|
|
1989
|
-
if (dedup.length === 0) return { text: '' }
|
|
1990
|
-
const queries = dedup.slice(0, QUERIES_CAP)
|
|
1991
|
-
const dropped = dedup.length - queries.length
|
|
1992
|
-
const rest = { ...args }
|
|
1993
|
-
delete rest.query
|
|
1994
|
-
const deadlineSec = Math.max(1, Number(process.env.MEMORY_FANOUT_DEADLINE_S) || 180)
|
|
1995
|
-
const deadlineMs = deadlineSec * 1000
|
|
1996
|
-
const fanOutAbort = new AbortController()
|
|
1997
|
-
let deadlineTimer
|
|
1998
|
-
const deadlineRace = new Promise((_res, rej) => {
|
|
1999
|
-
deadlineTimer = setTimeout(() => {
|
|
2000
|
-
fanOutAbort.abort(new Error(`memory fan-out deadline exceeded (${deadlineSec}s)`))
|
|
2001
|
-
rej(Object.assign(new Error(`memory fan-out deadline exceeded (${deadlineSec}s)`), { _deadline: true }))
|
|
2002
|
-
}, deadlineMs)
|
|
2003
|
-
})
|
|
2004
|
-
let settled
|
|
2005
|
-
try {
|
|
2006
|
-
// Pre-warm only when the embedding model is already resident. If the
|
|
2007
|
-
// process is still cold, keep recall responsive and let the background
|
|
2008
|
-
// warmup finish independently instead of making the first query pay the
|
|
2009
|
-
// ONNX session-create cost.
|
|
2010
|
-
if (isEmbeddingModelReady()) {
|
|
2011
|
-
// Race against the same deadline as the fan-out itself: a stuck
|
|
2012
|
-
// embedding worker would previously park here indefinitely because
|
|
2013
|
-
// the timer hadn't been started yet from the fan-out's perspective.
|
|
2014
|
-
await Promise.race([embedTexts(queries), deadlineRace])
|
|
2015
|
-
} else if (embeddingWarmupCanStart()) {
|
|
2016
|
-
void warmupEmbeddingProvider().catch((err) => {
|
|
2017
|
-
__mixdogMemoryLog(`[memory-service] embedding warmup after cold fan-out skipped dense search: ${err?.message || err}\n`)
|
|
2018
|
-
})
|
|
2019
|
-
}
|
|
2020
|
-
settled = await Promise.race([
|
|
2021
|
-
Promise.all(queries.map(async (q) => {
|
|
2022
|
-
if (fanOutAbort.signal.aborted) throw fanOutAbort.signal.reason
|
|
2023
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
2024
|
-
const sub = await handleSearch({ ...rest, query: q }, signal)
|
|
2025
|
-
return `[${q}]\n${sub.text || '(no results)'}`
|
|
2026
|
-
})),
|
|
2027
|
-
deadlineRace,
|
|
2028
|
-
])
|
|
2029
|
-
} finally {
|
|
2030
|
-
clearTimeout(deadlineTimer)
|
|
2031
|
-
}
|
|
2032
|
-
const parts = settled
|
|
2033
|
-
const header = dropped > 0
|
|
2034
|
-
? `note: ${dedup.length} queries received, ${queries.length} processed, ${dropped} dropped (cap ${QUERIES_CAP})\n\n`
|
|
2035
|
-
: ''
|
|
2036
|
-
return { text: header + parts.join('\n\n') }
|
|
2037
|
-
}
|
|
2038
|
-
const query = String(args.query ?? '').trim()
|
|
2039
|
-
let period = String(args.period ?? '').trim() || undefined
|
|
2040
|
-
// Period and sort are caller-supplied only. Lead is responsible for
|
|
2041
|
-
// mapping vague time phrases / chronological intent into the period
|
|
2042
|
-
// argument before calling; the engine does not infer them from query
|
|
2043
|
-
// text.
|
|
2044
|
-
const RECALL_LIMIT_CAP = 100
|
|
2045
|
-
const RECALL_OFFSET_CAP = 500
|
|
2046
|
-
const requestedLimit = Number(args.limit)
|
|
2047
|
-
const requestedOffset = Number(args.offset)
|
|
2048
|
-
let limit = Math.max(1, Number.isFinite(requestedLimit) ? requestedLimit : 10)
|
|
2049
|
-
let offset = Math.max(0, Number.isFinite(requestedOffset) ? requestedOffset : 0)
|
|
2050
|
-
const recallCapNotes = []
|
|
2051
|
-
if (Number.isFinite(requestedLimit) && requestedLimit > RECALL_LIMIT_CAP) {
|
|
2052
|
-
limit = RECALL_LIMIT_CAP
|
|
2053
|
-
recallCapNotes.push(`limit capped to ${RECALL_LIMIT_CAP} (requested ${requestedLimit})`)
|
|
2054
|
-
} else {
|
|
2055
|
-
limit = Math.min(RECALL_LIMIT_CAP, limit)
|
|
2056
|
-
}
|
|
2057
|
-
if (Number.isFinite(requestedOffset) && requestedOffset > RECALL_OFFSET_CAP) {
|
|
2058
|
-
offset = RECALL_OFFSET_CAP
|
|
2059
|
-
recallCapNotes.push(`offset capped to ${RECALL_OFFSET_CAP} (requested ${requestedOffset})`)
|
|
2060
|
-
} else {
|
|
2061
|
-
offset = Math.min(RECALL_OFFSET_CAP, offset)
|
|
2062
|
-
}
|
|
2063
|
-
const recallCapPrefix = recallCapNotes.length ? `${recallCapNotes.join('; ')}\n` : ''
|
|
2064
|
-
// Recent-browsing default: a query-less recall is a "show me the latest
|
|
2065
|
-
// messages" browse, not a relevance search — chronological order is the
|
|
2066
|
-
// only ordering that makes sense there, so sort defaults to 'date' when
|
|
2067
|
-
// no query is present (explicit args.sort still wins). Query recalls keep
|
|
2068
|
-
// the importance default.
|
|
2069
|
-
const hasQueryForSort = Array.isArray(args.query)
|
|
2070
|
-
? args.query.some((v) => String(v || '').trim())
|
|
2071
|
-
: String(args.query ?? '').trim() !== ''
|
|
2072
|
-
const sort = args.sort != null ? String(args.sort) : (hasQueryForSort ? 'importance' : 'date')
|
|
2073
|
-
// Chunk content is the primary recall output. Members default to true so
|
|
2074
|
-
// callers receive the raw chunk leaves (the cycle1-produced semantic
|
|
2075
|
-
// chunks) rather than just the root's cycle2-compressed summary line.
|
|
2076
|
-
// Explicit `includeMembers:false` keeps the legacy summary-only mode.
|
|
2077
|
-
const includeMembers = args.includeMembers !== false
|
|
2078
|
-
const includeRaw = Boolean(args.includeRaw)
|
|
2079
|
-
const includeArchived = args.includeArchived !== false
|
|
2080
|
-
const category = args.category
|
|
2081
|
-
const temporal = parsePeriod(period, Boolean(query))
|
|
2082
|
-
|
|
2083
|
-
// Derive projectScope from caller cwd (falls back to process.cwd()).
|
|
2084
|
-
// Explicit args.projectScope (string) takes priority so callers can
|
|
2085
|
-
// override to 'all', 'common', or a specific slug.
|
|
2086
|
-
let projectScope
|
|
2087
|
-
if (typeof args.projectScope === 'string' && args.projectScope) {
|
|
2088
|
-
projectScope = args.projectScope
|
|
2089
|
-
} else {
|
|
2090
|
-
const projectId = resolveProjectScope(typeof args.cwd === 'string' && args.cwd ? args.cwd : null)
|
|
2091
|
-
projectScope = projectId !== null ? projectId : 'common'
|
|
2092
|
-
}
|
|
2093
|
-
|
|
2094
|
-
// R11 reviewer M4: calendar-bounded periods disable freshness decay
|
|
2095
|
-
// so within-period ranking doesn't downgrade Mon entries vs Sun.
|
|
2096
|
-
const CALENDAR_PERIODS = new Set(['yesterday', 'today', 'this_week', 'last_week'])
|
|
2097
|
-
const isCalendarPeriod = period != null
|
|
2098
|
-
&& (CALENDAR_PERIODS.has(period) || /^\d{4}-\d{2}-\d{2}/.test(period))
|
|
2099
|
-
const applyFreshness = !isCalendarPeriod
|
|
2100
|
-
|
|
2101
|
-
if (query) {
|
|
2102
|
-
const _t0 = Date.now()
|
|
2103
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
2104
|
-
let queryVector = null
|
|
2105
|
-
if (isEmbeddingModelReady()) {
|
|
2106
|
-
queryVector = await embedText(query)
|
|
2107
|
-
} else {
|
|
2108
|
-
const now = Date.now()
|
|
2109
|
-
if (now - _embeddingColdRecallLogAt > 10_000) {
|
|
2110
|
-
_embeddingColdRecallLogAt = now
|
|
2111
|
-
__mixdogMemoryLog('[recall] embedding model cold; returning lexical results while background warmup continues\n')
|
|
2112
|
-
}
|
|
2113
|
-
if (embeddingWarmupCanStart()) {
|
|
2114
|
-
void warmupEmbeddingProvider().catch((err) => {
|
|
2115
|
-
__mixdogMemoryLog(`[memory-service] embedding warmup after cold recall failed: ${err?.message || err}\n`)
|
|
2116
|
-
})
|
|
2117
|
-
}
|
|
2118
|
-
}
|
|
2119
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
2120
|
-
const _t1 = Date.now()
|
|
2121
|
-
if (process.env.MIXDOG_DEBUG_MEMORY) {
|
|
2122
|
-
__mixdogMemoryLog(`[search-time] embed=${_t1 - _t0}ms query="${query.slice(0, 60)}"\n`)
|
|
2123
|
-
}
|
|
2124
|
-
// Push ts and status filters into the hybrid candidate query so FTS / vec
|
|
2125
|
-
// rank inside the requested window, not the whole tree. The previous post-
|
|
2126
|
-
// filter approach silently emptied results when relevant matches sat
|
|
2127
|
-
// outside `period` (default 30d) and could not bubble through.
|
|
2128
|
-
// Recall is history-first: archived roots hold most prior work. Callers
|
|
2129
|
-
// that need only live invariants can pass includeArchived:false.
|
|
2130
|
-
const excludeStatuses = includeArchived ? [] : ['archived']
|
|
2131
|
-
const results = await searchRelevantHybrid(db, query, {
|
|
2132
|
-
limit: limit + offset,
|
|
2133
|
-
queryVector: Array.isArray(queryVector) ? queryVector : null,
|
|
2134
|
-
includeMembers,
|
|
2135
|
-
ts_from: temporal?.startMs,
|
|
2136
|
-
ts_to: temporal?.endMs,
|
|
2137
|
-
applyFreshness,
|
|
2138
|
-
projectScope,
|
|
2139
|
-
category,
|
|
2140
|
-
excludeStatuses,
|
|
2141
|
-
// useHotActive was set to true here so default (no-period) calls
|
|
2142
|
-
// routed through the mv_hot_active materialized view — a narrow
|
|
2143
|
-
// active-roots-only pool. Live usage is dominated by vague-time
|
|
2144
|
-
// queries ("recent / lately") where Lead callers omit the period
|
|
2145
|
-
// filter, leaving the MV as the sole source. That hid every
|
|
2146
|
-
// orphan leaf and every pending root — fresh work from the last 1-60
|
|
2147
|
-
// minutes never surfaced. Now that the entries-table CTE legs run
|
|
2148
|
-
// against broaden HNSW + GIN trgm partial indexes (the
|
|
2149
|
-
// is_root=1 predicate was dropped in the same revision), the
|
|
2150
|
-
// entries path is fast enough (1-2 ms ANN on ~10K rows, O(log N)
|
|
2151
|
-
// through 1M+) to be the single source of truth. The MV is left in
|
|
2152
|
-
// place for now but no longer routed to from search; cycle2 may stop
|
|
2153
|
-
// refreshing it in a follow-up commit once nothing else reads it.
|
|
2154
|
-
useHotActive: false,
|
|
2155
|
-
})
|
|
2156
|
-
let filtered = results
|
|
2157
|
-
if (sort === 'date') {
|
|
2158
|
-
// R11 reviewer L5: NaN guard — entries with null/undefined ts default
|
|
2159
|
-
// to 0 so the comparator stays numeric and stable.
|
|
2160
|
-
filtered.sort((a, b) => (Number(b.ts) || 0) - (Number(a.ts) || 0))
|
|
2161
|
-
} else {
|
|
2162
|
-
filtered.sort((a, b) => {
|
|
2163
|
-
const sa = (v) => { const n = Number(v); return Number.isFinite(n) ? n : 0 }
|
|
2164
|
-
return (sa(b.retrievalScore ?? b.rrf ?? 0) - sa(a.retrievalScore ?? a.rrf ?? 0))
|
|
2165
|
-
|| (sa(b.score ?? 0) - sa(a.score ?? 0))
|
|
2166
|
-
|| (sa(b.ts ?? 0) - sa(a.ts ?? 0))
|
|
2167
|
-
|| (Number(a.id ?? 0) - Number(b.id ?? 0))
|
|
2168
|
-
})
|
|
2169
|
-
}
|
|
2170
|
-
if (includeRaw) {
|
|
2171
|
-
// Raw rows (chunk_root IS NULL) carry no retrievalScore, so a naive
|
|
2172
|
-
// append-after-hybrid under sort=importance always lands them past
|
|
2173
|
-
// slice(offset, offset+limit) once the hybrid pool exceeds one page —
|
|
2174
|
-
// every page beyond the first silently drops them. Fetch a wider raw
|
|
2175
|
-
// window (bounded like the hybrid candidate pool) and spread the
|
|
2176
|
-
// fetched raw rows evenly across the WHOLE hybrid list before slicing,
|
|
2177
|
-
// so every offset page gets its proportional share instead of only
|
|
2178
|
-
// page 0. Same projectScope/ts window as the hybrid leg — filter
|
|
2179
|
-
// parity (item 3) is deliberate, not accidental.
|
|
2180
|
-
const RAW_FETCH = Math.min(500, Math.max(20, limit + offset))
|
|
2181
|
-
const rawRows = await readRawRowsInWindow(
|
|
2182
|
-
db,
|
|
2183
|
-
temporal?.startMs ?? null,
|
|
2184
|
-
temporal?.endMs ?? Date.now(),
|
|
2185
|
-
RAW_FETCH,
|
|
2186
|
-
{ projectScope },
|
|
2187
|
-
)
|
|
2188
|
-
const seenIds = new Set(filtered.map(r => r.id))
|
|
2189
|
-
const newRaw = rawRows.filter(r => !seenIds.has(r.id))
|
|
2190
|
-
if (sort === 'date') {
|
|
2191
|
-
for (const r of newRaw) filtered.push(r)
|
|
2192
|
-
filtered.sort((a, b) => (Number(b.ts) || 0) - (Number(a.ts) || 0))
|
|
2193
|
-
} else {
|
|
2194
|
-
// sort=importance: interleave raw rows at a fixed stride through the
|
|
2195
|
-
// full (pre-slice) hybrid list instead of appending at the tail, so
|
|
2196
|
-
// offset > 0 pages also draw from the raw pool proportionally.
|
|
2197
|
-
filtered = interleaveRawRows(filtered, newRaw)
|
|
2198
|
-
}
|
|
2199
|
-
}
|
|
2200
|
-
const coreRows = await recallCoreRows(query, { projectScope, category, limit })
|
|
2201
|
-
if (coreRows.length > 0) {
|
|
2202
|
-
filtered = [...coreRows, ...filtered]
|
|
2203
|
-
}
|
|
2204
|
-
const sliced = filtered.slice(offset, offset + limit)
|
|
2205
|
-
const _t2 = Date.now()
|
|
2206
|
-
if (process.env.MIXDOG_DEBUG_MEMORY) {
|
|
2207
|
-
__mixdogMemoryLog(`[search-time] hybrid+sort+raw=${_t2 - _t1}ms rows=${filtered.length} sliced=${sliced.length}\n`)
|
|
2208
|
-
}
|
|
2209
|
-
// Emit a recall trace event so getTraceWithEntries() can correlate
|
|
2210
|
-
// this search with the top-ranked memory entry. One event per
|
|
2211
|
-
// handleSearch call (not per returned row) — cheapest meaningful link.
|
|
2212
|
-
// parent_span_id left null: the agent-side span id is only known after
|
|
2213
|
-
// the DB insert of the loop/tool events, which happens async on the
|
|
2214
|
-
// client side and is not available here.
|
|
2215
|
-
if (_traceDb && filtered.length > 0) {
|
|
2216
|
-
const topHit = filtered[0]
|
|
2217
|
-
const topId = topHit?.id != null ? Number(topHit.id) : null
|
|
2218
|
-
if (topId !== null && Number.isFinite(topId)) {
|
|
2219
|
-
insertTraceEvents(_traceDb, [{
|
|
2220
|
-
ts: Date.now(),
|
|
2221
|
-
kind: 'recall',
|
|
2222
|
-
entry_id: topId,
|
|
2223
|
-
payload: { query: query.slice(0, 200), hit_count: filtered.length },
|
|
2224
|
-
}]).catch(e => __mixdogMemoryLog(`[trace] insertTraceEvents error: ${e?.message}\n`))
|
|
2225
|
-
}
|
|
2226
|
-
}
|
|
2227
|
-
const out = { text: recallCapPrefix + renderEntryLines(sliced) }
|
|
2228
|
-
if (process.env.MIXDOG_DEBUG_MEMORY) {
|
|
2229
|
-
__mixdogMemoryLog(`[search-time] render+trace=${Date.now() - _t2}ms total=${Date.now() - _t0}ms textLen=${out.text.length}\n`)
|
|
2230
|
-
}
|
|
2231
|
-
return out
|
|
2232
|
-
}
|
|
2233
|
-
|
|
2234
|
-
const filters = { limit: limit + offset }
|
|
2235
|
-
if (temporal?.startMs != null) { filters.ts_from = temporal.startMs; filters.ts_to = temporal.endMs }
|
|
2236
|
-
// period='last' used to hard-cap ts_to at _bootTimestamp-1 unconditionally,
|
|
2237
|
-
// which silently hid the CURRENT session's own content from a query-less
|
|
2238
|
-
// recent browse (the whole point of "what did we just talk about"). Keep
|
|
2239
|
-
// the boot cap only for session-less global browsing, where 'last' means
|
|
2240
|
-
// "the previously completed session" and there's no session-scoped drain
|
|
2241
|
-
// above to have already surfaced the in-flight rows. When a sessionId is
|
|
2242
|
-
// present the drain above already pulled that session's raw backlog to
|
|
2243
|
-
// zero, so the current session's rows are safe (and wanted) to include.
|
|
2244
|
-
const _hasScopedSession = Boolean(String(args?.sessionId || args?.session_id || '').trim())
|
|
2245
|
-
if (temporal?.mode === 'last' && _bootTimestamp && !_hasScopedSession) {
|
|
2246
|
-
filters.ts_to = _bootTimestamp - 1
|
|
2247
|
-
}
|
|
2248
|
-
filters.projectScope = projectScope
|
|
2249
|
-
if (category != null) filters.category = category
|
|
2250
|
-
filters.sort = sort
|
|
2251
|
-
if (!includeArchived) filters.excludeStatuses = ['archived']
|
|
2252
|
-
if (includeMembers) filters.includeMembers = true
|
|
2253
|
-
const rows = await retrieveEntries(db, filters)
|
|
2254
|
-
// Recent-browsing raw merge: a query-less recall must show the freshest
|
|
2255
|
-
// turns even when cycle1 hasn't chunked them yet. Roots lag ingest by up
|
|
2256
|
-
// to a cycle interval, so on sort=date pull the raw (unchunked) window
|
|
2257
|
-
// too and merge chronologically — original text first, no summaries.
|
|
2258
|
-
// Query-less + includeRaw:false callers keep the roots-only view.
|
|
2259
|
-
let merged = rows
|
|
2260
|
-
if (sort === 'date' && args.includeRaw !== false) {
|
|
2261
|
-
const rawRows = await readRawRowsInWindow(
|
|
2262
|
-
db,
|
|
2263
|
-
temporal?.startMs ?? null,
|
|
2264
|
-
temporal?.endMs ?? (filters.ts_to ?? Date.now()),
|
|
2265
|
-
Math.min(500, Math.max(20, limit + offset)),
|
|
2266
|
-
{ projectScope },
|
|
2267
|
-
)
|
|
2268
|
-
const seenIds = new Set(rows.map(r => Number(r.id)))
|
|
2269
|
-
// Drop raw leaves already inlined as some returned root's member.
|
|
2270
|
-
for (const r of rows) {
|
|
2271
|
-
if (Array.isArray(r.members)) for (const m of r.members) seenIds.add(Number(m.id))
|
|
2272
|
-
}
|
|
2273
|
-
const newRaw = rawRows.filter(r => !seenIds.has(Number(r.id)))
|
|
2274
|
-
if (newRaw.length > 0) {
|
|
2275
|
-
merged = [...rows, ...newRaw]
|
|
2276
|
-
merged.sort((a, b) => (Number(b.ts) || 0) - (Number(a.ts) || 0) || (Number(b.id) || 0) - (Number(a.id) || 0))
|
|
2277
|
-
}
|
|
2278
|
-
}
|
|
2279
|
-
const sliced = merged.slice(offset, offset + limit)
|
|
2280
|
-
return { text: recallCapPrefix + renderEntryLines(sliced) }
|
|
2281
|
-
}
|
|
2282
|
-
|
|
2283
|
-
async function dumpSessionRootChunks(args = {}) {
|
|
2284
|
-
const sessionId = String(args.sessionId || args.session_id || '').trim()
|
|
2285
|
-
if (!sessionId) return { text: '(no current session)', rows: [], chunks: [], isError: true }
|
|
2286
|
-
const includeRaw = args.includeRaw !== false
|
|
2287
|
-
const limit = Math.max(1, Math.min(1000, Number(args.limit) || 1000))
|
|
2288
|
-
const rootRows = (await db.query(`
|
|
2289
|
-
SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root,
|
|
2290
|
-
element, category, summary, status, score, last_seen_at, project_id
|
|
2291
|
-
FROM entries
|
|
2292
|
-
WHERE session_id = $1 AND is_root = 1
|
|
2293
|
-
ORDER BY COALESCE(source_turn, 2147483647) ASC, ts ASC, id ASC
|
|
2294
|
-
LIMIT $2
|
|
2295
|
-
`, [sessionId, limit])).rows
|
|
2296
|
-
const roots = rootRows.map((r) => ({ ...r, members: [] }))
|
|
2297
|
-
const rootIds = roots.map((r) => Number(r.id)).filter((id) => Number.isFinite(id))
|
|
2298
|
-
const memberRows = rootIds.length > 0
|
|
2299
|
-
? (await db.query(`
|
|
2300
|
-
SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root, project_id
|
|
2301
|
-
FROM entries
|
|
2302
|
-
WHERE chunk_root = ANY($1::bigint[]) AND is_root = 0
|
|
2303
|
-
ORDER BY chunk_root ASC, COALESCE(source_turn, 2147483647) ASC, ts ASC, id ASC
|
|
2304
|
-
`, [rootIds])).rows
|
|
2305
|
-
: []
|
|
2306
|
-
const byRoot = new Map(roots.map((r) => [Number(r.id), r]))
|
|
2307
|
-
for (const m of memberRows) {
|
|
2308
|
-
const root = byRoot.get(Number(m.chunk_root))
|
|
2309
|
-
if (root) root.members.push(m)
|
|
2310
|
-
}
|
|
2311
|
-
let rawRows = []
|
|
2312
|
-
if (includeRaw) {
|
|
2313
|
-
rawRows = (await db.query(`
|
|
2314
|
-
SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root, project_id
|
|
2315
|
-
FROM entries
|
|
2316
|
-
WHERE session_id = $1
|
|
2317
|
-
AND is_root = 0
|
|
2318
|
-
AND (chunk_root IS NULL OR chunk_root = id)
|
|
2319
|
-
ORDER BY COALESCE(source_turn, 2147483647) ASC, ts ASC, id ASC
|
|
2320
|
-
LIMIT $2
|
|
2321
|
-
`, [sessionId, limit])).rows
|
|
2322
|
-
}
|
|
2323
|
-
const chunks = []
|
|
2324
|
-
for (const root of roots) {
|
|
2325
|
-
const memberText = root.members
|
|
2326
|
-
.map((m) => `${m.role === 'assistant' ? 'assistant' : m.role === 'user' ? 'user' : m.role}: ${cleanMemoryText(String(m.content ?? ''))}`)
|
|
2327
|
-
.filter(Boolean)
|
|
2328
|
-
.join('\n')
|
|
2329
|
-
const summary = [root.element, root.summary].map((v) => String(v || '').trim()).filter(Boolean).join(' — ')
|
|
2330
|
-
chunks.push({
|
|
2331
|
-
id: Number(root.id),
|
|
2332
|
-
kind: 'root',
|
|
2333
|
-
ts: Number(root.ts) || 0,
|
|
2334
|
-
sourceTurn: root.source_turn ?? null,
|
|
2335
|
-
category: root.category || null,
|
|
2336
|
-
summary,
|
|
2337
|
-
text: memberText || cleanMemoryText(String(root.content ?? '')),
|
|
2338
|
-
members: root.members,
|
|
2339
|
-
})
|
|
2340
|
-
}
|
|
2341
|
-
for (const raw of rawRows) {
|
|
2342
|
-
chunks.push({
|
|
2343
|
-
id: Number(raw.id),
|
|
2344
|
-
kind: 'raw',
|
|
2345
|
-
chunkRoot: raw.chunk_root ?? null,
|
|
2346
|
-
ts: Number(raw.ts) || 0,
|
|
2347
|
-
sourceTurn: raw.source_turn ?? null,
|
|
2348
|
-
category: null,
|
|
2349
|
-
summary: '',
|
|
2350
|
-
text: `${raw.role === 'assistant' ? 'assistant' : raw.role === 'user' ? 'user' : raw.role}: ${cleanMemoryText(String(raw.content ?? ''))}`,
|
|
2351
|
-
members: [],
|
|
2352
|
-
})
|
|
2353
|
-
}
|
|
2354
|
-
chunks.sort((a, b) => {
|
|
2355
|
-
const at = Number.isFinite(Number(a.sourceTurn)) ? Number(a.sourceTurn) : 2147483647
|
|
2356
|
-
const bt = Number.isFinite(Number(b.sourceTurn)) ? Number(b.sourceTurn) : 2147483647
|
|
2357
|
-
return (at - bt) || ((a.ts || 0) - (b.ts || 0)) || ((a.id || 0) - (b.id || 0))
|
|
2358
|
-
})
|
|
2359
|
-
const text = chunks.length
|
|
2360
|
-
? chunks.map((chunk, idx) => {
|
|
2361
|
-
const label = chunk.kind === 'root'
|
|
2362
|
-
? `# chunk ${idx + 1} root=${chunk.id}${chunk.category ? ` category=${chunk.category}` : ''}`
|
|
2363
|
-
: `${chunk.chunkRoot == null ? '# raw_pending' : '# raw_terminal'} ${idx + 1} id=${chunk.id}`
|
|
2364
|
-
const summary = chunk.summary ? `summary: ${chunk.summary}\n` : ''
|
|
2365
|
-
return `${label}\n${summary}${chunk.text}`.trim()
|
|
2366
|
-
}).join('\n\n')
|
|
2367
|
-
: '(no results)'
|
|
2368
|
-
return { text, rows: [...roots, ...rawRows], chunks }
|
|
2369
|
-
}
|
|
2370
|
-
|
|
2371
|
-
async function entryStats() {
|
|
2372
|
-
return await db.transaction(async (tx) => {
|
|
2373
|
-
const total = (await tx.query(`SELECT COUNT(*) c FROM entries`)).rows[0].c
|
|
2374
|
-
const roots = (await tx.query(`SELECT COUNT(*) c FROM entries WHERE is_root = 1`)).rows[0].c
|
|
2375
|
-
const active_roots = (await tx.query(`SELECT COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'active'`)).rows[0].c
|
|
2376
|
-
const archived_roots = (await tx.query(`SELECT COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'archived'`)).rows[0].c
|
|
2377
|
-
const unchunked_leaves = (await tx.query(`SELECT COUNT(*) c FROM entries WHERE chunk_root IS NULL`)).rows[0].c
|
|
2378
|
-
const cycle2_pending_roots = (await tx.query(`SELECT COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'pending'`)).rows[0].c
|
|
2379
|
-
const core_entries = (await tx.query(`SELECT COUNT(*) c FROM core_entries`)).rows[0].c
|
|
2380
|
-
const core_embed_null = (await tx.query(`SELECT COUNT(*) c FROM core_entries WHERE embedding IS NULL`)).rows[0].c
|
|
2381
|
-
const active_core_summaries = (await tx.query(`SELECT COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'active' AND core_summary IS NOT NULL`)).rows[0].c
|
|
2382
|
-
const active_core_summary_missing = (await tx.query(`
|
|
2383
|
-
SELECT COUNT(*) c
|
|
2384
|
-
FROM entries
|
|
2385
|
-
WHERE is_root = 1
|
|
2386
|
-
AND status = 'active'
|
|
2387
|
-
AND (core_summary IS NULL OR btrim(core_summary) = '')
|
|
2388
|
-
`)).rows[0].c
|
|
2389
|
-
const byStatus = (await tx.query(`SELECT status, COUNT(*) c FROM entries WHERE is_root = 1 GROUP BY status`)).rows
|
|
2390
|
-
const byCategory = (await tx.query(`SELECT category, COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'active' GROUP BY category ORDER BY c DESC`)).rows
|
|
2391
|
-
const mvRows = (await tx.query(`SELECT relispopulated FROM pg_class WHERE relname = 'mv_hot_active' LIMIT 1`)).rows
|
|
2392
|
-
const mv_hot_active_populated = mvRows.length ? Boolean(mvRows[0].relispopulated) : null
|
|
2393
|
-
return {
|
|
2394
|
-
total, roots, active_roots, archived_roots, unchunked_leaves, cycle2_pending_roots,
|
|
2395
|
-
core_entries, core_embed_null, active_core_summaries, active_core_summary_missing,
|
|
2396
|
-
mv_hot_active_populated,
|
|
2397
|
-
byStatus, byCategory,
|
|
2398
|
-
}
|
|
2399
|
-
})
|
|
2400
|
-
}
|
|
647
|
+
const __queryHandlers = createQueryHandlers({
|
|
648
|
+
getDb: () => db,
|
|
649
|
+
log: __mixdogMemoryLog,
|
|
650
|
+
resolveProjectScope,
|
|
651
|
+
embeddingWarmupCanStart,
|
|
652
|
+
getBootTimestamp: () => _bootTimestamp,
|
|
653
|
+
getTraceDb: () => _traceDb,
|
|
654
|
+
})
|
|
655
|
+
const {
|
|
656
|
+
readRawRowsInWindow,
|
|
657
|
+
recallSessionRows,
|
|
658
|
+
recallCoreRows,
|
|
659
|
+
handleSearch,
|
|
660
|
+
dumpSessionRootChunks,
|
|
661
|
+
entryStats,
|
|
662
|
+
} = __queryHandlers
|
|
2401
663
|
|
|
2402
664
|
async function _handleMemCycle1(args, config, signal) {
|
|
2403
665
|
const minBatchOverride = Number(args?.min_batch)
|
|
@@ -2591,7 +853,7 @@ async function _handleMemRebuild(args, config, signal) {
|
|
|
2591
853
|
// then loop once more if one layer exposed another promise while awaiting.
|
|
2592
854
|
const drainedCycle1Promises = new Set()
|
|
2593
855
|
for (;;) {
|
|
2594
|
-
const pendingCycle1Promises = [
|
|
856
|
+
const pendingCycle1Promises = [_cycleScheduler.getCycle1InFlight(), getInFlightCycle1(db)]
|
|
2595
857
|
.filter(p => p && !drainedCycle1Promises.has(p))
|
|
2596
858
|
if (pendingCycle1Promises.length === 0) break
|
|
2597
859
|
for (const pendingCycle1 of pendingCycle1Promises) {
|
|
@@ -3196,6 +1458,9 @@ async function handleToolCall(name, args, signal) {
|
|
|
3196
1458
|
...(a.sessionId ? { sessionId: a.sessionId } : {}),
|
|
3197
1459
|
...(a.session_id ? { session_id: a.session_id } : {}),
|
|
3198
1460
|
...(a.currentSession !== undefined ? { currentSession: a.currentSession } : {}),
|
|
1461
|
+
// Hint only — never a filter. Marks the caller's own session as
|
|
1462
|
+
// "(current)" in the multi-session grouped browse output.
|
|
1463
|
+
...(a.currentSessionId ? { currentSessionId: a.currentSessionId } : {}),
|
|
3199
1464
|
}
|
|
3200
1465
|
const result = await handleSearch(searchArgs, signal)
|
|
3201
1466
|
return { ...result, content: [{ type: 'text', text: result.text }], isError: result.isError || false }
|
|
@@ -3325,9 +1590,9 @@ const httpServer = http.createServer(async (req, res) => {
|
|
|
3325
1590
|
active_core_summaries: stats.active_core_summaries,
|
|
3326
1591
|
active_core_summary_missing: stats.active_core_summary_missing,
|
|
3327
1592
|
mv_hot_active_populated: stats.mv_hot_active_populated,
|
|
3328
|
-
cycle_running:
|
|
3329
|
-
cycle_health:
|
|
3330
|
-
cycle_backlog:
|
|
1593
|
+
cycle_running: _cycleScheduler.getCycleRunning(),
|
|
1594
|
+
cycle_health: _cycleScheduler.getCycleHealth(),
|
|
1595
|
+
cycle_backlog: _cycleScheduler.getCycleBacklogSnapshot(),
|
|
3331
1596
|
})
|
|
3332
1597
|
} catch (e) { sendError(res, e.message) }
|
|
3333
1598
|
return
|
|
@@ -3911,7 +2176,7 @@ const httpServer = http.createServer(async (req, res) => {
|
|
|
3911
2176
|
}
|
|
3912
2177
|
const fileSize = stat.size
|
|
3913
2178
|
await ingestTranscriptFile(filePath, { cwd: body.cwd })
|
|
3914
|
-
const off =
|
|
2179
|
+
const off = _transcriptIngest.getOffset(filePath)
|
|
3915
2180
|
const offsetBytes = off && Number.isFinite(off.bytes) ? off.bytes : 0
|
|
3916
2181
|
const complete = offsetBytes >= fileSize
|
|
3917
2182
|
sendJson(res, { ok: true, offsetBytes, fileSize, complete })
|
|
@@ -3973,7 +2238,7 @@ export async function stop() {
|
|
|
3973
2238
|
}
|
|
3974
2239
|
_periodicAdvertiseInstalled = false
|
|
3975
2240
|
_currentAdvertisedPort = null
|
|
3976
|
-
|
|
2241
|
+
_embeddingWarmup.reset()
|
|
3977
2242
|
if (_idleShutdownTimer) {
|
|
3978
2243
|
try { clearTimeout(_idleShutdownTimer) } catch {}
|
|
3979
2244
|
_idleShutdownTimer = null
|
|
@@ -4051,11 +2316,8 @@ export async function stop() {
|
|
|
4051
2316
|
_initialized = false
|
|
4052
2317
|
_initPromise = null
|
|
4053
2318
|
_bootTimestamp = null
|
|
4054
|
-
|
|
4055
|
-
|
|
4056
|
-
_cycle2InFlight = false
|
|
4057
|
-
_cycle3InFlight = false
|
|
4058
|
-
_checkCyclesInFlight = false
|
|
2319
|
+
_transcriptIngest.resetOffsets()
|
|
2320
|
+
_cycleScheduler.resetInFlight()
|
|
4059
2321
|
releaseLock()
|
|
4060
2322
|
})().finally(() => {
|
|
4061
2323
|
_stopPromise = null
|