clawmem 0.7.2 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/AGENTS.md CHANGED
@@ -96,6 +96,14 @@ curl http://host:8090/v1/models
96
96
  | `CLAWMEM_ENABLE_AMEM` | enabled | A-MEM note construction + link generation during indexing. |
97
97
  | `CLAWMEM_ENABLE_CONSOLIDATION` | disabled | Background worker backfills unenriched docs. Needs long-lived MCP process. |
98
98
  | `CLAWMEM_CONSOLIDATION_INTERVAL` | 300000 | Worker interval in ms (min 15000). |
99
+ | `CLAWMEM_HEAVY_LANE` | disabled | **v0.8.0.** Enable the quiet-window heavy maintenance worker — a second, longer-interval consolidation lane with DB-backed `worker_leases` exclusivity, stale-first batching, and `maintenance_runs` journaling. Runs alongside the light lane; off by default. |
100
+ | `CLAWMEM_HEAVY_LANE_INTERVAL` | 1800000 | **v0.8.0.** Heavy-lane tick interval in ms (min 30000, default 30 min). |
101
+ | `CLAWMEM_HEAVY_LANE_WINDOW_START` | (none) | **v0.8.0.** Start hour (0-23) of the quiet window. Unset → no window. |
102
+ | `CLAWMEM_HEAVY_LANE_WINDOW_END` | (none) | **v0.8.0.** End hour (0-23, exclusive) of the quiet window. Supports midnight wrap (22→6). |
103
+ | `CLAWMEM_HEAVY_LANE_MAX_USAGES` | 30 | **v0.8.0.** Max `context_usage` rows in the last 10 min before the heavy lane skips with `reason='query_rate_high'`. |
104
+ | `CLAWMEM_HEAVY_LANE_OBS_LIMIT` | 100 | **v0.8.0.** Phase 2 stale-first observation batch size. |
105
+ | `CLAWMEM_HEAVY_LANE_DED_LIMIT` | 40 | **v0.8.0.** Phase 3 stale-first deductive candidate batch size. |
106
+ | `CLAWMEM_HEAVY_LANE_SURPRISAL` | `false` | **v0.8.0.** When `true`, the heavy lane seeds Phase 2 with k-NN anomaly-ranked doc ids from `computeSurprisalScores` instead of stale-first ordering. Degrades to stale-first (`surprisal-fallback-stale` metric) on vaults without embeddings. |
99
107
  | `CLAWMEM_NUDGE_INTERVAL` | `15` | Prompts between lifecycle tool use before `<vault-nudge>` injection. 0 to disable. |
100
108
  | `CLAWMEM_MERGE_SCORE_NORMAL` | `0.93` | **v0.7.1.** Phase 2 merge-safety score threshold when candidate and existing anchors align. Merges above this normalized 3-gram cosine similarity are allowed. |
101
109
  | `CLAWMEM_MERGE_SCORE_STRICT` | `0.98` | **v0.7.1.** Strictest merge-safety score threshold (fallback when anchors are ambiguous). |
@@ -503,8 +511,9 @@ The `memory_relations` table is populated by multiple independent sources:
503
511
  | `buildTemporalBackbone()` | temporal | `build_graphs` MCP tool (manual) | Creation-order edges between all active docs. |
504
512
  | `buildSemanticGraph()` | semantic | `build_graphs` MCP tool (manual) | Pure cosine similarity. PK collision: `INSERT OR IGNORE` means A-MEM semantic edges take precedence if they exist first. |
505
513
  | Entity co-occurrence graph | entity | A-MEM enrichment (indexing) | LLM entity extraction → quality filters (title/length/blocklist/location validation) → type-agnostic canonical resolution within compatibility buckets (person, org, location, tech=project/service/tool/concept) → `entity_mentions` + `entity_cooccurrences` tables. Entity edges use IDF-based specificity scoring. Feeds ENTITY intent queries and MPFP `[entity, semantic]` patterns. |
506
- | `consolidated_observations` | supporting, contradicts | Consolidation worker (background) | 3-tier consolidation: facts → observations → mental models. Observations track `proof_count`, `trend` (STABLE/STRENGTHENING/WEAKENING/STALE), and source links. **v0.7.1 safety gates:** name-aware merge gate uses entity-anchor comparison + 3-gram cosine similarity (dual-threshold `CLAWMEM_MERGE_SCORE_NORMAL`=0.93 / `_STRICT`=0.98) to prevent cross-entity merges ("Alice decided X" merging into "Bob decided X"). Merge-time contradiction gate runs deterministic heuristic + LLM check; blocked merges route to `CLAWMEM_CONTRADICTION_POLICY`=`link` (new row + `contradicts` edge, default) or `supersede` (old row `status='inactive'`, new row replaces). |
507
- | Deductive synthesis | supporting, contradicts | Consolidation worker Phase 3 (background, every ~15 min) | Combines 2-3 related recent observations (decision/preference/milestone/problem, last 7 days) into `content_type='deductive'` documents with `source_doc_ids` provenance. First-class searchable docs with ∞ half-life. **v0.7.1 anti-contamination wrapper:** every draft passes through deterministic pre-checks (empty conclusion, invalid source_indices, pool-only entity contamination via `entity_mentions` or lexical fallback) + LLM validator (fail-open with `validatorFallbackAccepts` counter) + dedupe. Per-reason rejection stats exposed via `DeductiveSynthesisStats` (contaminationRejects, invalidIndexRejects, unsupportedRejects, emptyRejects, dedupSkipped, validatorFallbackAccepts). Contradictory dedupe matches are linked via `contradicts` edges. |
514
+ | `consolidated_observations` | supporting, contradicts | Consolidation worker (background, light + heavy lanes) | 3-tier consolidation: facts → observations → mental models. Observations track `proof_count`, `trend` (STABLE/STRENGTHENING/WEAKENING/STALE), and source links. **v0.7.1 safety gates:** name-aware merge gate uses entity-anchor comparison + 3-gram cosine similarity (dual-threshold `CLAWMEM_MERGE_SCORE_NORMAL`=0.93 / `_STRICT`=0.98) to prevent cross-entity merges ("Alice decided X" merging into "Bob decided X"). Merge-time contradiction gate runs deterministic heuristic + LLM check; blocked merges route to `CLAWMEM_CONTRADICTION_POLICY`=`link` (new row + `contradicts` edge, default) or `supersede` (old row `status='inactive'`, new row replaces). **v0.8.0:** the heavy lane calls `consolidateObservations(store, llm, { maxDocs, guarded: true, staleOnly: true })` — `guarded: true` forces merge-safety enforcement regardless of `CLAWMEM_MERGE_GUARD_DRY_RUN`, and `staleOnly: true` reorders candidates by `recall_stats.last_recalled_at ASC` so long-unseen docs bubble up first. Optional `candidateIds` filter plumbs k-NN anomaly ids from `computeSurprisalScores`. |
515
+ | Deductive synthesis | supporting, contradicts | Consolidation worker Phase 3 (background, every ~15 min in the light lane; batched in the heavy lane) | Combines 2-3 related recent observations (decision/preference/milestone/problem, last 7 days) into `content_type='deductive'` documents with `source_doc_ids` provenance. First-class searchable docs with ∞ half-life. **v0.7.1 anti-contamination wrapper:** every draft passes through deterministic pre-checks (empty conclusion, invalid source_indices, pool-only entity contamination via `entity_mentions` or lexical fallback) + LLM validator (fail-open with `validatorFallbackAccepts` counter) + dedupe. Per-reason rejection stats exposed via `DeductiveSynthesisStats` (contaminationRejects, invalidIndexRejects, unsupportedRejects, emptyRejects, dedupSkipped, validatorFallbackAccepts). Contradictory dedupe matches are linked via `contradicts` edges. **v0.8.0:** heavy lane calls `generateDeductiveObservations(store, llm, { maxRecent, guarded: true, staleOnly: true })` for stale-first batching on large vaults. |
516
+ | Heavy maintenance lane journal | — | `clawmem` process with `CLAWMEM_HEAVY_LANE=true` (v0.8.0) | Writes a row to the `maintenance_runs` table for every scheduled heavy-lane attempt (including skips). Columns: `lane` (`heavy`), `phase` (`gate`/`consolidate`/`deductive`), `status` (`started`/`completed`/`failed`/`skipped`), `reason` (`outside_window`/`query_rate_high`/`lease_unavailable`), per-phase `selected`/`processed`/`created`/`null_call` counts, and `metrics_json` with selector type (`stale-first`/`surprisal`/`surprisal-fallback-stale`) + full `DeductiveSynthesisStats` on completed Phase 3 runs. Exclusivity is enforced via the new `worker_leases` table with atomic `INSERT ... ON CONFLICT DO UPDATE ... WHERE expires_at <= ?` acquisition, random 16-byte fencing tokens, and TTL reclaim. Gate uses the existing `context_usage` table (no `query_activity` table) and `recall_stats.last_recalled_at` for stale-first ordering. Off by default — requires explicit opt-in. |
508
517
  | Conversation synthesis (`runConversationSynthesis`) | semantic, supporting, contradicts, causal, temporal, entity | `clawmem mine <dir> --synthesize` (opt-in, post-index) | **v0.7.2.** Two-pass LLM pipeline over freshly imported `content_type='conversation'` docs. Pass 1 extracts structured facts (decision/preference/milestone/problem) via `extractFactsFromConversation`, saves each via dedup-aware `saveMemory`, populates a local Set-based alias map. Pass 2 resolves cross-fact links against the local map first (fails closed on ambiguity — multi-candidate titles return unresolved), falls back to collection-scoped SQL lookup with LIMIT 2 ambiguity detection. Relations upsert via `ON CONFLICT DO UPDATE SET weight = MAX(weight, excluded.weight)` — idempotent on equal-weight reruns but monotonically accepts stronger later evidence. Synthesized fact paths are a pure function of `(sourceDocId, slug(title), short sha256(normalizedTitle))`, so reruns update in place instead of creating parallel rows. Counters split into `llmFailures` (null/thrown/invalid-JSON) vs `docsWithNoFacts` (valid-empty extraction). All failures non-fatal — never rolls back the mine import. |
509
518
 
510
519
  **Edge collision:** Both `generateMemoryLinks()` and `buildSemanticGraph()` insert `relation_type='semantic'`. PK is `(source_id, target_id, relation_type)` — first writer wins.
package/CLAUDE.md CHANGED
@@ -96,6 +96,14 @@ curl http://host:8090/v1/models
96
96
  | `CLAWMEM_ENABLE_AMEM` | enabled | A-MEM note construction + link generation during indexing. |
97
97
  | `CLAWMEM_ENABLE_CONSOLIDATION` | disabled | Background worker backfills unenriched docs. Needs long-lived MCP process. |
98
98
  | `CLAWMEM_CONSOLIDATION_INTERVAL` | 300000 | Worker interval in ms (min 15000). |
99
+ | `CLAWMEM_HEAVY_LANE` | disabled | **v0.8.0.** Enable the quiet-window heavy maintenance worker — a second, longer-interval consolidation lane with DB-backed `worker_leases` exclusivity, stale-first batching, and `maintenance_runs` journaling. Runs alongside the light lane; off by default. |
100
+ | `CLAWMEM_HEAVY_LANE_INTERVAL` | 1800000 | **v0.8.0.** Heavy-lane tick interval in ms (min 30000, default 30 min). |
101
+ | `CLAWMEM_HEAVY_LANE_WINDOW_START` | (none) | **v0.8.0.** Start hour (0-23) of the quiet window. Unset → no window. |
102
+ | `CLAWMEM_HEAVY_LANE_WINDOW_END` | (none) | **v0.8.0.** End hour (0-23, exclusive) of the quiet window. Supports midnight wrap (22→6). |
103
+ | `CLAWMEM_HEAVY_LANE_MAX_USAGES` | 30 | **v0.8.0.** Max `context_usage` rows in the last 10 min before the heavy lane skips with `reason='query_rate_high'`. |
104
+ | `CLAWMEM_HEAVY_LANE_OBS_LIMIT` | 100 | **v0.8.0.** Phase 2 stale-first observation batch size. |
105
+ | `CLAWMEM_HEAVY_LANE_DED_LIMIT` | 40 | **v0.8.0.** Phase 3 stale-first deductive candidate batch size. |
106
+ | `CLAWMEM_HEAVY_LANE_SURPRISAL` | `false` | **v0.8.0.** When `true`, the heavy lane seeds Phase 2 with k-NN anomaly-ranked doc ids from `computeSurprisalScores` instead of stale-first ordering. Degrades to stale-first (`surprisal-fallback-stale` metric) on vaults without embeddings. |
99
107
  | `CLAWMEM_NUDGE_INTERVAL` | `15` | Prompts between lifecycle tool use before `<vault-nudge>` injection. 0 to disable. |
100
108
  | `CLAWMEM_MERGE_SCORE_NORMAL` | `0.93` | **v0.7.1.** Phase 2 merge-safety score threshold when candidate and existing anchors align. Merges above this normalized 3-gram cosine similarity are allowed. |
101
109
  | `CLAWMEM_MERGE_SCORE_STRICT` | `0.98` | **v0.7.1.** Strictest merge-safety score threshold (fallback when anchors are ambiguous). |
@@ -503,8 +511,9 @@ The `memory_relations` table is populated by multiple independent sources:
503
511
  | `buildTemporalBackbone()` | temporal | `build_graphs` MCP tool (manual) | Creation-order edges between all active docs. |
504
512
  | `buildSemanticGraph()` | semantic | `build_graphs` MCP tool (manual) | Pure cosine similarity. PK collision: `INSERT OR IGNORE` means A-MEM semantic edges take precedence if they exist first. |
505
513
  | Entity co-occurrence graph | entity | A-MEM enrichment (indexing) | LLM entity extraction → quality filters (title/length/blocklist/location validation) → type-agnostic canonical resolution within compatibility buckets (person, org, location, tech=project/service/tool/concept) → `entity_mentions` + `entity_cooccurrences` tables. Entity edges use IDF-based specificity scoring. Feeds ENTITY intent queries and MPFP `[entity, semantic]` patterns. |
506
- | `consolidated_observations` | supporting, contradicts | Consolidation worker (background) | 3-tier consolidation: facts → observations → mental models. Observations track `proof_count`, `trend` (STABLE/STRENGTHENING/WEAKENING/STALE), and source links. **v0.7.1 safety gates:** name-aware merge gate uses entity-anchor comparison + 3-gram cosine similarity (dual-threshold `CLAWMEM_MERGE_SCORE_NORMAL`=0.93 / `_STRICT`=0.98) to prevent cross-entity merges ("Alice decided X" merging into "Bob decided X"). Merge-time contradiction gate runs deterministic heuristic + LLM check; blocked merges route to `CLAWMEM_CONTRADICTION_POLICY`=`link` (new row + `contradicts` edge, default) or `supersede` (old row `status='inactive'`, new row replaces). |
507
- | Deductive synthesis | supporting, contradicts | Consolidation worker Phase 3 (background, every ~15 min) | Combines 2-3 related recent observations (decision/preference/milestone/problem, last 7 days) into `content_type='deductive'` documents with `source_doc_ids` provenance. First-class searchable docs with ∞ half-life. **v0.7.1 anti-contamination wrapper:** every draft passes through deterministic pre-checks (empty conclusion, invalid source_indices, pool-only entity contamination via `entity_mentions` or lexical fallback) + LLM validator (fail-open with `validatorFallbackAccepts` counter) + dedupe. Per-reason rejection stats exposed via `DeductiveSynthesisStats` (contaminationRejects, invalidIndexRejects, unsupportedRejects, emptyRejects, dedupSkipped, validatorFallbackAccepts). Contradictory dedupe matches are linked via `contradicts` edges. |
514
+ | `consolidated_observations` | supporting, contradicts | Consolidation worker (background, light + heavy lanes) | 3-tier consolidation: facts → observations → mental models. Observations track `proof_count`, `trend` (STABLE/STRENGTHENING/WEAKENING/STALE), and source links. **v0.7.1 safety gates:** name-aware merge gate uses entity-anchor comparison + 3-gram cosine similarity (dual-threshold `CLAWMEM_MERGE_SCORE_NORMAL`=0.93 / `_STRICT`=0.98) to prevent cross-entity merges ("Alice decided X" merging into "Bob decided X"). Merge-time contradiction gate runs deterministic heuristic + LLM check; blocked merges route to `CLAWMEM_CONTRADICTION_POLICY`=`link` (new row + `contradicts` edge, default) or `supersede` (old row `status='inactive'`, new row replaces). **v0.8.0:** the heavy lane calls `consolidateObservations(store, llm, { maxDocs, guarded: true, staleOnly: true })` — `guarded: true` forces merge-safety enforcement regardless of `CLAWMEM_MERGE_GUARD_DRY_RUN`, and `staleOnly: true` reorders candidates by `recall_stats.last_recalled_at ASC` so long-unseen docs bubble up first. Optional `candidateIds` filter plumbs k-NN anomaly ids from `computeSurprisalScores`. |
515
+ | Deductive synthesis | supporting, contradicts | Consolidation worker Phase 3 (background, every ~15 min in the light lane; batched in the heavy lane) | Combines 2-3 related recent observations (decision/preference/milestone/problem, last 7 days) into `content_type='deductive'` documents with `source_doc_ids` provenance. First-class searchable docs with ∞ half-life. **v0.7.1 anti-contamination wrapper:** every draft passes through deterministic pre-checks (empty conclusion, invalid source_indices, pool-only entity contamination via `entity_mentions` or lexical fallback) + LLM validator (fail-open with `validatorFallbackAccepts` counter) + dedupe. Per-reason rejection stats exposed via `DeductiveSynthesisStats` (contaminationRejects, invalidIndexRejects, unsupportedRejects, emptyRejects, dedupSkipped, validatorFallbackAccepts). Contradictory dedupe matches are linked via `contradicts` edges. **v0.8.0:** heavy lane calls `generateDeductiveObservations(store, llm, { maxRecent, guarded: true, staleOnly: true })` for stale-first batching on large vaults. |
516
+ | Heavy maintenance lane journal | — | `clawmem` process with `CLAWMEM_HEAVY_LANE=true` (v0.8.0) | Writes a row to the `maintenance_runs` table for every scheduled heavy-lane attempt (including skips). Columns: `lane` (`heavy`), `phase` (`gate`/`consolidate`/`deductive`), `status` (`started`/`completed`/`failed`/`skipped`), `reason` (`outside_window`/`query_rate_high`/`lease_unavailable`), per-phase `selected`/`processed`/`created`/`null_call` counts, and `metrics_json` with selector type (`stale-first`/`surprisal`/`surprisal-fallback-stale`) + full `DeductiveSynthesisStats` on completed Phase 3 runs. Exclusivity is enforced via the new `worker_leases` table with atomic `INSERT ... ON CONFLICT DO UPDATE ... WHERE expires_at <= ?` acquisition, random 16-byte fencing tokens, and TTL reclaim. Gate uses the existing `context_usage` table (no `query_activity` table) and `recall_stats.last_recalled_at` for stale-first ordering. Off by default — requires explicit opt-in. |
508
517
  | Conversation synthesis (`runConversationSynthesis`) | semantic, supporting, contradicts, causal, temporal, entity | `clawmem mine <dir> --synthesize` (opt-in, post-index) | **v0.7.2.** Two-pass LLM pipeline over freshly imported `content_type='conversation'` docs. Pass 1 extracts structured facts (decision/preference/milestone/problem) via `extractFactsFromConversation`, saves each via dedup-aware `saveMemory`, populates a local Set-based alias map. Pass 2 resolves cross-fact links against the local map first (fails closed on ambiguity — multi-candidate titles return unresolved), falls back to collection-scoped SQL lookup with LIMIT 2 ambiguity detection. Relations upsert via `ON CONFLICT DO UPDATE SET weight = MAX(weight, excluded.weight)` — idempotent on equal-weight reruns but monotonically accepts stronger later evidence. Synthesized fact paths are a pure function of `(sourceDocId, slug(title), short sha256(normalizedTitle))`, so reruns update in place instead of creating parallel rows. Counters split into `llmFailures` (null/thrown/invalid-JSON) vs `docsWithNoFacts` (valid-empty extraction). All failures non-fatal — never rolls back the mine import. |
509
518
 
510
519
  **Edge collision:** Both `generateMemoryLinks()` and `buildSemanticGraph()` insert `relation_type='semantic'`. PK is `(source_id, target_id, relation_type)` — first writer wins.
package/README.md CHANGED
@@ -43,6 +43,7 @@ ClawMem turns your markdown notes, project docs, and research dumps into persist
43
43
  - **Manages document lifecycle** — policy-driven archival sweeps with restore capability
44
44
  - **Auto-routes queries** via `memory_retrieve` — classifies intent and dispatches to the optimal search backend
45
45
  - **Syncs project issues** from Beads issue trackers into searchable memory
46
+ - **Runs a quiet-window heavy maintenance lane** — a second consolidation worker, off by default behind `CLAWMEM_HEAVY_LANE=true`, that runs on a longer interval only inside a configurable hour window. Gated by `context_usage` query-rate so it never competes for CPU/GPU with interactive sessions, scoped exclusively via DB-backed `worker_leases`, stale-first by default with an optional surprisal selector, and journals every attempt in `maintenance_runs` for operator visibility (v0.8.0)
46
47
 
47
48
  Runs fully local with no API keys and no cloud services. Integrates via Claude Code hooks and MCP tools, as an OpenClaw ContextEngine plugin, or as a Hermes Agent MemoryProvider plugin. All modes share the same vault for cross-runtime memory. Works with any MCP-compatible client.
48
49
 
@@ -80,6 +81,20 @@ Opt-in LLM pass that runs **after** `clawmem mine` finishes indexing an imported
80
81
 
81
82
  Adds +63 tests (46 unit + 5 integration + 12 regression) on top of the v0.7.1 baseline.
82
83
 
84
+ ### v0.8.0 Quiet-Window Heavy Maintenance Lane
85
+
86
+ A second, longer-interval consolidation worker that keeps Phase 2 + Phase 3 running on large vaults without starving interactive sessions. Off by default — set `CLAWMEM_HEAVY_LANE=true` to enable. The existing 5-minute light-lane worker is unchanged. See [heavy maintenance lane](docs/concepts/architecture.md#heavy-maintenance-lane-v080) for the architectural walkthrough.
87
+
88
+ - **Quiet-window gating** — the heavy lane only runs inside the hours set by `CLAWMEM_HEAVY_LANE_WINDOW_START` / `CLAWMEM_HEAVY_LANE_WINDOW_END` (0-23). Supports midnight wraparound (e.g., 22→6). Null on either bound means "always in window".
89
+ - **Query-rate gating via `context_usage`** — counts hook injections in the last 10 minutes and skips the tick when the rate exceeds `CLAWMEM_HEAVY_LANE_MAX_USAGES` (default 30). No new `query_activity` table; reuses v0.7.0 telemetry.
90
+ - **DB-backed worker leases** — exclusivity enforced via a new `worker_leases` table with atomic `INSERT ... ON CONFLICT DO UPDATE ... WHERE expires_at <= ?` acquisition, random 16-byte fencing tokens, and TTL reclaim. Safe under multi-process contention; any SQLite error translates to a `lease_unavailable` skip rather than a thrown exception.
91
+ - **Stale-first selection** — Phase 2 and Phase 3 reorder their candidate sets by `COALESCE(recall_stats.last_recalled_at, documents.last_accessed_at, documents.modified_at) ASC` so long-unseen docs bubble up first. Empty `recall_stats` falls through to access-time without erroring.
92
+ - **Optional surprisal selector** — `CLAWMEM_HEAVY_LANE_SURPRISAL=true` plumbs k-NN anomaly-ranked doc ids (via the existing `computeSurprisalScores`) into Phase 2 as an explicit `candidateIds` filter. Degrades to stale-first on vaults without embeddings and logs `selector: 'surprisal-fallback-stale'` in the journal.
93
+ - **`maintenance_runs` journal** — every scheduled attempt writes a row: `status` (`started`/`completed`/`failed`/`skipped`), `reason` for skips, selected/processed/created/null_call counts, and a `metrics_json` payload with selector type and full `DeductiveSynthesisStats` breakdown. Operators can reconstruct any lane decision without reading worker logs.
94
+ - **Force-enforce merge gate** — the heavy lane passes `guarded: true` to `consolidateObservations`, which overrides `CLAWMEM_MERGE_GUARD_DRY_RUN` inside `findSimilarConsolidation` so experimenting operators cannot weaken heavy-lane enforcement via env flag.
95
+
96
+ Adds +56 tests (13 worker-lease + 35 maintenance unit + 8 maintenance integration) on top of the v0.7.2 baseline.
97
+
83
98
  ## Architecture
84
99
 
85
100
  <p align="center">
package/SKILL.md CHANGED
@@ -92,6 +92,12 @@ curl http://host:8090/v1/models
92
92
  | `CLAWMEM_MERGE_GUARD_DRY_RUN` | `false` | **v0.7.1.** When `true`, Phase 2 merge-safety rejections are logged but not enforced — use for calibration. |
93
93
  | `CLAWMEM_CONTRADICTION_POLICY` | `link` | **v0.7.1.** How the merge-time contradiction gate handles a blocked merge. `link` (default) keeps both rows + inserts `contradicts` edge. `supersede` marks the old row `status='inactive'`. |
94
94
  | `CLAWMEM_CONTRADICTION_MIN_CONFIDENCE` | `0.5` | **v0.7.1.** Minimum combined heuristic+LLM confidence required before the contradiction gate blocks a merge. |
95
+ | `CLAWMEM_HEAVY_LANE` | disabled | **v0.8.0.** Enable the quiet-window heavy maintenance worker — a second, longer-interval consolidation lane with DB-backed `worker_leases` exclusivity, stale-first batching, and `maintenance_runs` journaling. Runs alongside the light lane. |
96
+ | `CLAWMEM_HEAVY_LANE_INTERVAL` | 1800000 | **v0.8.0.** Heavy-lane tick interval in ms (min 30000, default 30 min). |
97
+ | `CLAWMEM_HEAVY_LANE_WINDOW_START` / `_END` | (none) | **v0.8.0.** Start/end hours (0-23) of the quiet window. Supports midnight wrap (22→6). Null on either bound = always in window. |
98
+ | `CLAWMEM_HEAVY_LANE_MAX_USAGES` | 30 | **v0.8.0.** Max `context_usage` rows in the last 10 min before the heavy lane skips with `reason='query_rate_high'`. |
99
+ | `CLAWMEM_HEAVY_LANE_OBS_LIMIT` / `_DED_LIMIT` | 100 / 40 | **v0.8.0.** Phase 2 / Phase 3 stale-first batch sizes. |
100
+ | `CLAWMEM_HEAVY_LANE_SURPRISAL` | `false` | **v0.8.0.** When `true`, the heavy lane seeds Phase 2 with k-NN anomaly-ranked doc ids from `computeSurprisalScores` instead of stale-first ordering. Degrades to stale-first on vaults without embeddings. |
95
101
 
96
102
  **Note:** The `bin/clawmem` wrapper sets all endpoint defaults. Always use the wrapper — never `bun run src/clawmem.ts` directly.
97
103
 
@@ -526,9 +532,10 @@ mcp__clawmem__vsearch(query, collection="name", compact=true) # vector
526
532
  | Beads `syncBeadsIssues()` | causal, supporting, semantic | `beads_sync` MCP or watcher | Queries `bd` CLI (Dolt backend). |
527
533
  | `buildTemporalBackbone()` | temporal | `build_graphs` MCP (manual) | Creation-order edges. |
528
534
  | `buildSemanticGraph()` | semantic | `build_graphs` MCP (manual) | Pure cosine similarity. A-MEM edges take precedence (first-writer wins). |
529
- | `consolidated_observations` | supporting, contradicts | Consolidation worker (background) | **v0.7.1 safety gates:** Phase 2 name-aware merge gate (entity anchors + 3-gram cosine, dual-threshold `CLAWMEM_MERGE_SCORE_NORMAL`=0.93 / `_STRICT`=0.98) blocks cross-entity merges. Merge-time contradiction gate (heuristic + LLM) routes blocked merges to `link` (default, inserts `contradicts` edge) or `supersede` (old row `status='inactive'`) via `CLAWMEM_CONTRADICTION_POLICY`. |
530
- | Deductive synthesis | supporting, contradicts | Consolidation worker Phase 3 (every ~15 min) | Combines 2-3 related observations (decision/preference/milestone/problem, last 7 days) into `content_type='deductive'` docs. **v0.7.1 anti-contamination:** deterministic pre-checks (empty/invalid_indices/pool-only entity contamination) + LLM validator (fail-open, `validatorFallbackAccepts` counter) + dedupe. Per-reason rejection stats via `DeductiveSynthesisStats`. Contradictory dedupe matches linked via `contradicts` edges. |
535
+ | `consolidated_observations` | supporting, contradicts | Consolidation worker (background, light + heavy lanes) | **v0.7.1 safety gates:** Phase 2 name-aware merge gate (entity anchors + 3-gram cosine, dual-threshold `CLAWMEM_MERGE_SCORE_NORMAL`=0.93 / `_STRICT`=0.98) blocks cross-entity merges. Merge-time contradiction gate (heuristic + LLM) routes blocked merges to `link` (default, inserts `contradicts` edge) or `supersede` (old row `status='inactive'`) via `CLAWMEM_CONTRADICTION_POLICY`. **v0.8.0:** heavy lane calls `consolidateObservations(store, llm, { maxDocs, guarded: true, staleOnly: true, candidateIds? })` — `guarded: true` forces enforcement regardless of `CLAWMEM_MERGE_GUARD_DRY_RUN`, `staleOnly: true` reorders by `recall_stats.last_recalled_at ASC`, and optional `candidateIds` plumbs surprisal-selector output. |
536
+ | Deductive synthesis | supporting, contradicts | Consolidation worker Phase 3 (every ~15 min in light lane; batched in heavy lane) | Combines 2-3 related observations (decision/preference/milestone/problem, last 7 days) into `content_type='deductive'` docs. **v0.7.1 anti-contamination:** deterministic pre-checks (empty/invalid_indices/pool-only entity contamination) + LLM validator (fail-open, `validatorFallbackAccepts` counter) + dedupe. Per-reason rejection stats via `DeductiveSynthesisStats`. Contradictory dedupe matches linked via `contradicts` edges. **v0.8.0:** heavy lane calls `generateDeductiveObservations(store, llm, { maxRecent, guarded: true, staleOnly: true })` for stale-first batching. |
531
537
  | Conversation synthesis | semantic, supporting, contradicts, causal, temporal, entity | `clawmem mine <dir> --synthesize` (opt-in, post-index) | **v0.7.2.** Two-pass LLM pipeline over freshly imported `content_type='conversation'` docs. Pass 1 extracts structured decision/preference/milestone/problem facts + aliases + cross-fact links, saves via dedup-aware `saveMemory`, populates ambiguity-aware local Set map. Pass 2 resolves links (local first, SQL fallback with `LIMIT 2` ambiguity detection), upserts relations via `ON CONFLICT DO UPDATE SET weight = MAX(weight, excluded.weight)`. Synthesized paths are a pure function of `(sourceDocId, slug(title), short sha256(normalizedTitle))` so reruns update in place. All failures non-fatal. Counters split: `llmFailures` (LLM/parse error) vs `docsWithNoFacts` (valid empty extraction). |
538
+ | Heavy maintenance lane journal | — | `CLAWMEM_HEAVY_LANE=true` (v0.8.0) | Writes a row to `maintenance_runs` for every scheduled heavy-lane attempt (including skips). Columns: `lane` (`heavy`), `phase` (`gate`/`consolidate`/`deductive`), `status` (`started`/`completed`/`failed`/`skipped`), `reason` (`outside_window`/`query_rate_high`/`lease_unavailable`), per-phase counts, and `metrics_json` with selector type (`stale-first`/`surprisal`/`surprisal-fallback-stale`) + full `DeductiveSynthesisStats`. Exclusivity via new `worker_leases` table (atomic `INSERT ... ON CONFLICT DO UPDATE ... WHERE expires_at <= ?`, 16-byte fencing tokens, TTL reclaim). Gate reuses `context_usage` (no `query_activity` table). |
532
539
 
533
540
  **Graph traversal asymmetry:** `adaptiveTraversal()` traverses all edge types outbound (source->target) but only `semantic` and `entity` inbound.
534
541
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "clawmem",
3
- "version": "0.7.2",
3
+ "version": "0.8.0",
4
4
  "description": "On-device context engine and memory for AI agents. Claude Code and OpenClaw. Hooks + MCP server + hybrid RAG search.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -114,6 +114,50 @@ interface ObservationCluster {
114
114
  collection: string;
115
115
  }
116
116
 
117
+ /**
118
+ * Phase 2 consolidation option bag (v0.8.0 Ext 5). All fields optional;
119
+ * omitting the bag reproduces pre-Ext-5 behavior exactly.
120
+ *
121
+ * - `maxDocs` override for the observation batch size (default 50)
122
+ * - `guarded` when true, force merge-safety enforcement regardless of
123
+ * the `CLAWMEM_MERGE_GUARD_DRY_RUN` env var. Heavy-lane
124
+ * callers pass this so experimenting operators cannot
125
+ * weaken the heavy-lane gate by toggling an env flag.
126
+ * - `staleOnly` when true, order observations by
127
+ * `recall_stats.last_recalled_at ASC` (with
128
+ * `documents.last_accessed_at` fallback) so least-recalled
129
+ * documents are processed first instead of most-recent.
130
+ * - `candidateIds` when non-empty, restricts the Phase 2 candidate set to
131
+ * exactly these document ids. Heavy lane uses this to
132
+ * plumb surprisal-selector output into consolidation so
133
+ * `useSurprisalSelector: true` actually feeds anomaly-first
134
+ * candidates instead of just relabeling the default batch.
135
+ */
136
+ export interface ConsolidateOptions {
137
+ maxDocs?: number;
138
+ guarded?: boolean;
139
+ staleOnly?: boolean;
140
+ candidateIds?: number[];
141
+ }
142
+
143
+ /**
144
+ * Phase 3 deductive synthesis option bag (v0.8.0 Ext 5). All fields optional;
145
+ * omitting the bag reproduces pre-Ext-5 behavior exactly.
146
+ *
147
+ * - `maxRecent` override for the recent-observation window size (default 20)
148
+ * - `guarded` forward-compat marker; currently a no-op because the Phase 3
149
+ * deductive guardrails always enforce their deterministic
150
+ * pre-checks + LLM validator regardless of this flag
151
+ * - `staleOnly` when true, order candidate observations by
152
+ * `recall_stats.last_recalled_at ASC` with
153
+ * `documents.last_accessed_at` fallback
154
+ */
155
+ export interface DeductiveOptions {
156
+ maxRecent?: number;
157
+ guarded?: boolean;
158
+ staleOnly?: boolean;
159
+ }
160
+
117
161
  // =============================================================================
118
162
  // Worker State
119
163
  // =============================================================================
@@ -247,17 +291,71 @@ async function backfillAmem(store: Store, llm: LlamaCpp): Promise<void> {
247
291
  /**
248
292
  * Find clusters of related observations and synthesize into consolidated observations.
249
293
  * Runs per-collection to prevent cross-vault false merges.
294
+ *
295
+ * `opts` (v0.8.0 Ext 5) lets the heavy-maintenance lane override batch size,
296
+ * force merge-safety enforcement, and switch to stale-first ordering. The
297
+ * zero-arg call path (internal light-lane tick) preserves pre-Ext-5 behavior.
250
298
  */
251
- async function consolidateObservations(store: Store, llm: LlamaCpp): Promise<void> {
252
- console.log("[consolidation] Starting observation consolidation");
299
+ export async function consolidateObservations(
300
+ store: Store,
301
+ llm: LlamaCpp,
302
+ opts: ConsolidateOptions = {},
303
+ ): Promise<void> {
304
+ const maxDocs = opts.maxDocs && opts.maxDocs > 0 ? opts.maxDocs : 50;
305
+ const staleOnly = opts.staleOnly === true;
306
+ const guarded = opts.guarded === true;
307
+ const candidateIds = opts.candidateIds && opts.candidateIds.length > 0
308
+ ? opts.candidateIds
309
+ : null;
310
+
311
+ // Early exit when caller passed candidateIds: [] explicitly. An empty
312
+ // array means "the selector found nothing" — not "select everything".
313
+ if (opts.candidateIds && opts.candidateIds.length === 0) {
314
+ console.log("[consolidation] Empty candidateIds array — nothing to consolidate");
315
+ return;
316
+ }
317
+
318
+ console.log(
319
+ `[consolidation] Starting observation consolidation (maxDocs=${maxDocs}, ` +
320
+ `staleOnly=${staleOnly}, guarded=${guarded}, ` +
321
+ `candidateIds=${candidateIds ? candidateIds.length : "null"})`,
322
+ );
323
+
324
+ // Base SELECT — observation-type docs not yet consolidated.
325
+ // Stale-first ordering joins recall_stats.last_recalled_at ASC with a
326
+ // documents.last_accessed_at fallback so long-unseen docs bubble up first.
327
+ // Default ordering (modified_at DESC) preserves pre-Ext-5 light-lane semantics.
328
+ const orderBy = staleOnly
329
+ ? `ORDER BY
330
+ d.collection,
331
+ COALESCE(rs.last_recalled_at, d.last_accessed_at, d.modified_at) ASC,
332
+ d.modified_at ASC`
333
+ : `ORDER BY d.collection, d.modified_at DESC`;
334
+
335
+ const joinClause = staleOnly
336
+ ? `LEFT JOIN recall_stats rs ON rs.doc_id = d.id`
337
+ : ``;
338
+
339
+ // When candidateIds is provided, restrict the SELECT to those docs. This
340
+ // is how the heavy lane plumbs surprisal-selector output into Phase 2:
341
+ // select anomaly-first IDs via computeSurprisalScores, then ask
342
+ // consolidateObservations to limit its pattern detection to that subset.
343
+ const candidateFilter = candidateIds
344
+ ? `AND d.id IN (${candidateIds.map(() => "?").join(",")})`
345
+ : ``;
346
+
347
+ const sqlParams: (number | string)[] = candidateIds
348
+ ? [...candidateIds, maxDocs]
349
+ : [maxDocs];
253
350
 
254
- // Find observation-type documents not yet consolidated
255
351
  const observations = store.db.prepare(`
256
352
  SELECT d.id, d.title, d.facts, d.amem_context as context, d.modified_at, d.collection
257
353
  FROM documents d
354
+ ${joinClause}
258
355
  WHERE d.active = 1
259
356
  AND d.content_type = 'observation'
260
357
  AND d.facts IS NOT NULL
358
+ ${candidateFilter}
261
359
  AND d.id NOT IN (
262
360
  SELECT value FROM (
263
361
  SELECT json_each.value as value
@@ -265,9 +363,9 @@ async function consolidateObservations(store: Store, llm: LlamaCpp): Promise<voi
265
363
  WHERE co.status = 'active'
266
364
  )
267
365
  )
268
- ORDER BY d.collection, d.modified_at DESC
269
- LIMIT 50
270
- `).all() as { id: number; title: string; facts: string; context: string; modified_at: string; collection: string }[];
366
+ ${orderBy}
367
+ LIMIT ?
368
+ `).all(...sqlParams) as { id: number; title: string; facts: string; context: string; modified_at: string; collection: string }[];
271
369
 
272
370
  if (observations.length === 0) {
273
371
  console.log("[consolidation] No unconsolidated observations found");
@@ -288,7 +386,7 @@ async function consolidateObservations(store: Store, llm: LlamaCpp): Promise<voi
288
386
  if (cluster.docs.length < 2) continue; // Need at least 2 observations to consolidate
289
387
 
290
388
  try {
291
- await synthesizeCluster(store, llm, cluster);
389
+ await synthesizeCluster(store, llm, cluster, { guarded });
292
390
  } catch (err) {
293
391
  console.error(`[consolidation] Failed to consolidate cluster for ${collection}:`, err);
294
392
  }
@@ -302,11 +400,15 @@ async function consolidateObservations(store: Store, llm: LlamaCpp): Promise<voi
302
400
 
303
401
  /**
304
402
  * Synthesize a cluster of observations into consolidated observations using LLM.
403
+ *
404
+ * `opts.guarded` (v0.8.0 Ext 5) forces merge-safety enforcement inside
405
+ * `findSimilarConsolidation` regardless of `CLAWMEM_MERGE_GUARD_DRY_RUN`.
305
406
  */
306
407
  async function synthesizeCluster(
307
408
  store: Store,
308
409
  llm: LlamaCpp,
309
- cluster: ObservationCluster
410
+ cluster: ObservationCluster,
411
+ opts: { guarded?: boolean } = {},
310
412
  ): Promise<void> {
311
413
  const docsText = cluster.docs.map((d, i) =>
312
414
  `${i + 1}. [${d.modified_at}] "${d.title}"\n Facts: ${d.facts?.slice(0, 300) || 'none'}\n Context: ${d.context?.slice(0, 200) || 'none'}`
@@ -364,11 +466,13 @@ Return ONLY the JSON array. /no_think`;
364
466
 
365
467
  // Check for existing similar consolidated observation (avoid duplicates).
366
468
  // Two-stage gate: Jaccard shortlist + name-aware merge safety (Ext 3).
469
+ // `guarded` forces gate enforcement when the heavy lane calls us.
367
470
  const existing = findSimilarConsolidation(
368
471
  store,
369
472
  pattern.observation,
370
473
  cluster.collection,
371
- sourceDocIds
474
+ sourceDocIds,
475
+ opts.guarded === true,
372
476
  );
373
477
  if (existing) {
374
478
  // Ext 2: contradiction gate. Before merging into an existing
@@ -524,7 +628,8 @@ export function findSimilarConsolidation(
524
628
  store: Store,
525
629
  observation: string,
526
630
  collection: string,
527
- candidateSourceDocIds: number[]
631
+ candidateSourceDocIds: number[],
632
+ forceEnforce: boolean = false,
528
633
  ): { id: number; observation: string; source_doc_ids: string } | null {
529
634
  // ORDER BY id ASC makes "first shortlist hit" deterministic across
530
635
  // SQLite plan changes — the dry-run legacy parity case relies on
@@ -553,7 +658,12 @@ export function findSimilarConsolidation(
553
658
 
554
659
  if (shortlist.length === 0) return null;
555
660
 
556
- const dryRun = process.env.CLAWMEM_MERGE_GUARD_DRY_RUN === "true";
661
+ // `forceEnforce` (v0.8.0 Ext 5) lets the heavy lane override the env
662
+ // `CLAWMEM_MERGE_GUARD_DRY_RUN` so operators can experiment with dry-run
663
+ // in the light lane without weakening heavy-lane guarantees.
664
+ const dryRun = forceEnforce
665
+ ? false
666
+ : process.env.CLAWMEM_MERGE_GUARD_DRY_RUN === "true";
557
667
 
558
668
  // Dry-run: preserve EXACT legacy behavior — return the first shortlist
559
669
  // hit (the pre-Ext-3 code iterated the SELECT rows in order and returned
@@ -723,18 +833,38 @@ function updateTrends(store: Store): void {
723
833
  *
724
834
  * Only considers decision/preference/milestone/problem observations from the
725
835
  * last 7 days that haven't already been used as sources for deductions.
836
+ *
837
+ * `opts` (v0.8.0 Ext 5):
838
+ * - `maxRecent` override batch size (default 20)
839
+ * - `staleOnly` order by recall_stats.last_recalled_at ASC with
840
+ * documents.last_accessed_at fallback instead of
841
+ * modified_at DESC
842
+ * - `guarded` forward-compat marker — no-op in v0.8.0 because the Phase 3
843
+ * guardrails always enforce their gates regardless
726
844
  */
727
- async function generateDeductiveObservations(
845
+ export async function generateDeductiveObservations(
728
846
  store: Store,
729
- llm: LlamaCpp
847
+ llm: LlamaCpp,
848
+ opts: DeductiveOptions = {},
730
849
  ): Promise<DeductiveSynthesisStats> {
850
+ const maxRecent = opts.maxRecent && opts.maxRecent > 0 ? opts.maxRecent : 20;
851
+ const staleOnly = opts.staleOnly === true;
731
852
  const stats = emptyDeductiveStats();
732
853
  // Find recent high-value observations not yet used in deductions
733
854
  const DEDUCTIVE_TYPES = ['decision', 'preference', 'milestone', 'problem'];
855
+
856
+ const orderBy = staleOnly
857
+ ? `ORDER BY COALESCE(rs.last_recalled_at, d.last_accessed_at, d.modified_at) ASC, d.modified_at ASC`
858
+ : `ORDER BY d.modified_at DESC`;
859
+ const joinClause = staleOnly
860
+ ? `LEFT JOIN recall_stats rs ON rs.doc_id = d.id`
861
+ : ``;
862
+
734
863
  const recentObs = store.db.prepare(`
735
864
  SELECT d.id, d.title, d.facts, d.narrative, d.observation_type, d.content_type,
736
865
  d.collection, d.path, d.modified_at
737
866
  FROM documents d
867
+ ${joinClause}
738
868
  WHERE d.active = 1
739
869
  AND d.content_type IN (${DEDUCTIVE_TYPES.map(() => '?').join(',')})
740
870
  AND d.observation_type IS NOT NULL
@@ -747,9 +877,9 @@ async function generateDeductiveObservations(
747
877
  WHERE dd.content_type = 'deductive' AND dd.active = 1
748
878
  )
749
879
  )
750
- ORDER BY d.modified_at DESC
751
- LIMIT 20
752
- `).all(...DEDUCTIVE_TYPES) as {
880
+ ${orderBy}
881
+ LIMIT ?
882
+ `).all(...DEDUCTIVE_TYPES, maxRecent) as {
753
883
  id: number; title: string; facts: string; narrative: string;
754
884
  observation_type: string; content_type: string; collection: string;
755
885
  path: string; modified_at: string;
@@ -0,0 +1,540 @@
1
+ /**
2
+ * ClawMem Heavy Maintenance Lane (v0.8.0 Ext 5)
3
+ *
4
+ * A second, longer-interval consolidation worker that runs during configured
5
+ * quiet windows with stale-first batching, DB-backed exclusivity via
6
+ * worker_leases, and journal rows in `maintenance_runs` for every attempt.
7
+ *
8
+ * Keeps Phase 2/3 consolidation + deductive synthesis running on large vaults
9
+ * without competing for CPU/GPU against the interactive light lane that
10
+ * ticks every 5 minutes. Off by default — enabled via `CLAWMEM_HEAVY_LANE=true`
11
+ * next to the existing light-lane `CLAWMEM_ENABLE_CONSOLIDATION` flag.
12
+ *
13
+ * Design notes:
14
+ * - Uses existing `context_usage` telemetry for query-rate gating. No new
15
+ * `query_activity` table — we count rows where `timestamp > -10 minutes`.
16
+ * - Stale-first selection prefers docs whose `recall_stats.last_recalled_at`
17
+ * is oldest/null, falling back to `documents.last_accessed_at` / `modified_at`.
18
+ * - Optional surprisal selector reuses `computeSurprisalScores` to bubble up
19
+ * high-anomaly observations for curator-style runs.
20
+ * - Every scheduled attempt writes a `maintenance_runs` row so operators can
21
+ * reconstruct the decision without reading worker logs.
22
+ */
23
+
24
+ import type { Store } from "./store.ts";
25
+ import type { LlamaCpp } from "./llm.ts";
26
+ import {
27
+ consolidateObservations,
28
+ generateDeductiveObservations,
29
+ computeSurprisalScores,
30
+ type DeductiveSynthesisStats,
31
+ } from "./consolidation.ts";
32
+ import { withWorkerLease } from "./worker-lease.ts";
33
+
34
+ // =============================================================================
35
+ // Config
36
+ // =============================================================================
37
+
38
+ export interface HeavyMaintenanceConfig {
39
+ /** Interval between heavy-lane ticks in milliseconds (default 30 min). */
40
+ intervalMs?: number;
41
+ /** Start hour (0-23) of the quiet window; null/undefined = no window. */
42
+ windowStartHour?: number | null;
43
+ /** End hour (0-23, exclusive) of the quiet window; null/undefined = no window. */
44
+ windowEndHour?: number | null;
45
+ /** Max context_usage rows in the last 10 min before the lane skips (default 30). */
46
+ maxContextUsagesPer10m?: number;
47
+ /** Batch size for Phase 2 consolidation (default 100). */
48
+ staleObservationLimit?: number;
49
+ /** Batch size for Phase 3 deductive synthesis (default 40). */
50
+ staleDeductiveLimit?: number;
51
+ /** When true, use computeSurprisalScores to select batches for Phase 2. */
52
+ useSurprisalSelector?: boolean;
53
+ /** Worker lease TTL in ms (default 10 min — covers worst-case run time). */
54
+ leaseTtlMs?: number;
55
+ /** Worker lease name. Override only in tests (default "heavy-maintenance"). */
56
+ workerName?: string;
57
+ /** Clock injection for unit tests — defaults to `() => new Date()`. */
58
+ clock?: () => Date;
59
+ }
60
+
61
+ // Vault scoping note: the heavy lane operates on whatever Store it is handed
62
+ // — createStore(path) maps 1:1 to a single SQLite vault, so context_usage
63
+ // and recall_stats reads are implicitly vault-scoped via `store.db`. Multi-
64
+ // vault mode is out of scope for v0.8.0 and would require extending this
65
+ // config with an explicit vault list plus a per-vault lease name.
66
+
67
+ const DEFAULT_CONFIG: Required<Omit<HeavyMaintenanceConfig, "workerName" | "clock">> = {
68
+ intervalMs: 30 * 60 * 1000,
69
+ windowStartHour: null,
70
+ windowEndHour: null,
71
+ maxContextUsagesPer10m: 30,
72
+ staleObservationLimit: 100,
73
+ staleDeductiveLimit: 40,
74
+ useSurprisalSelector: false,
75
+ leaseTtlMs: 10 * 60 * 1000,
76
+ };
77
+
78
+ const DEFAULT_WORKER_NAME = "heavy-maintenance";
79
+
80
+ // =============================================================================
81
+ // Journal helpers
82
+ // =============================================================================
83
+
84
+ export type MaintenanceStatus = "started" | "skipped" | "completed" | "failed";
85
+
86
+ export interface MaintenanceRunSummary {
87
+ id: number;
88
+ lane: string;
89
+ phase: string;
90
+ status: MaintenanceStatus;
91
+ reason: string | null;
92
+ selected_count: number;
93
+ processed_count: number;
94
+ created_count: number;
95
+ updated_count: number;
96
+ rejected_count: number;
97
+ null_call_count: number;
98
+ started_at: string;
99
+ finished_at: string | null;
100
+ metrics_json: string | null;
101
+ }
102
+
103
+ export function insertMaintenanceRun(
104
+ store: Store,
105
+ row: {
106
+ lane: string;
107
+ phase: string;
108
+ status: MaintenanceStatus;
109
+ reason?: string | null;
110
+ selectedCount?: number;
111
+ processedCount?: number;
112
+ createdCount?: number;
113
+ updatedCount?: number;
114
+ rejectedCount?: number;
115
+ nullCallCount?: number;
116
+ startedAt?: string;
117
+ finishedAt?: string | null;
118
+ metrics?: Record<string, unknown> | null;
119
+ },
120
+ ): number {
121
+ const startedAt = row.startedAt ?? new Date().toISOString();
122
+ const result = store.db.prepare(
123
+ `INSERT INTO maintenance_runs
124
+ (lane, phase, status, reason, selected_count, processed_count,
125
+ created_count, updated_count, rejected_count, null_call_count,
126
+ started_at, finished_at, metrics_json)
127
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
128
+ ).run(
129
+ row.lane,
130
+ row.phase,
131
+ row.status,
132
+ row.reason ?? null,
133
+ row.selectedCount ?? 0,
134
+ row.processedCount ?? 0,
135
+ row.createdCount ?? 0,
136
+ row.updatedCount ?? 0,
137
+ row.rejectedCount ?? 0,
138
+ row.nullCallCount ?? 0,
139
+ startedAt,
140
+ row.finishedAt ?? null,
141
+ row.metrics ? JSON.stringify(row.metrics) : null,
142
+ );
143
+ return Number(result.lastInsertRowid);
144
+ }
145
+
146
+ function finalizeMaintenanceRun(
147
+ store: Store,
148
+ id: number,
149
+ patch: {
150
+ status: MaintenanceStatus;
151
+ reason?: string | null;
152
+ selectedCount?: number;
153
+ processedCount?: number;
154
+ createdCount?: number;
155
+ updatedCount?: number;
156
+ rejectedCount?: number;
157
+ nullCallCount?: number;
158
+ finishedAt?: string;
159
+ metrics?: Record<string, unknown> | null;
160
+ },
161
+ ): void {
162
+ store.db.prepare(
163
+ `UPDATE maintenance_runs
164
+ SET status = ?, reason = ?,
165
+ selected_count = ?, processed_count = ?,
166
+ created_count = ?, updated_count = ?,
167
+ rejected_count = ?, null_call_count = ?,
168
+ finished_at = ?, metrics_json = ?
169
+ WHERE id = ?`,
170
+ ).run(
171
+ patch.status,
172
+ patch.reason ?? null,
173
+ patch.selectedCount ?? 0,
174
+ patch.processedCount ?? 0,
175
+ patch.createdCount ?? 0,
176
+ patch.updatedCount ?? 0,
177
+ patch.rejectedCount ?? 0,
178
+ patch.nullCallCount ?? 0,
179
+ patch.finishedAt ?? new Date().toISOString(),
180
+ patch.metrics ? JSON.stringify(patch.metrics) : null,
181
+ id,
182
+ );
183
+ }
184
+
185
+ // =============================================================================
186
+ // Gating logic
187
+ // =============================================================================
188
+
189
+ /**
190
+ * True when `now` falls inside [windowStartHour, windowEndHour). Both nulls
191
+ * mean "always in window". Handles midnight wraparound (e.g., 22→6) by
192
+ * accepting either hour >= start OR hour < end.
193
+ */
194
+ export function isInQuietWindow(
195
+ now: Date,
196
+ windowStartHour: number | null | undefined,
197
+ windowEndHour: number | null | undefined,
198
+ ): boolean {
199
+ if (windowStartHour == null || windowEndHour == null) return true;
200
+ if (windowStartHour < 0 || windowStartHour > 23 || windowEndHour < 0 || windowEndHour > 23) {
201
+ throw new Error(
202
+ `isInQuietWindow: hours must be 0-23, got start=${windowStartHour} end=${windowEndHour}`,
203
+ );
204
+ }
205
+ if (windowStartHour === windowEndHour) return false; // empty window
206
+ const hour = now.getHours();
207
+ if (windowStartHour < windowEndHour) {
208
+ return hour >= windowStartHour && hour < windowEndHour;
209
+ }
210
+ // Wraps midnight
211
+ return hour >= windowStartHour || hour < windowEndHour;
212
+ }
213
+
214
+ /**
215
+ * Count context_usage rows in the last `minutes` minutes. Used as a proxy
216
+ * for "how busy is the interactive light lane right now" — replaces the
217
+ * `query_activity` table that Turn 2 of Ext 5 proposed.
218
+ *
219
+ * The cutoff is computed in JS as an ISO 8601 string and bound as a
220
+ * parameter instead of using `datetime('now', '-N minutes')`. SQLite's
221
+ * `datetime()` returns a space-separated format (`YYYY-MM-DD HH:MM:SS`)
222
+ * while `context_usage.timestamp` is written in ISO 8601 with a T
223
+ * separator; lexicographic comparison across those two formats is wrong
224
+ * (space < T sorts ALL ISO rows as "newer" than any datetime() result).
225
+ */
226
+ export function countRecentContextUsages(
227
+ store: Store,
228
+ minutes: number = 10,
229
+ ): number {
230
+ const cutoff = new Date(Date.now() - minutes * 60 * 1000).toISOString();
231
+ const row = store.db.prepare(
232
+ `SELECT COUNT(*) AS cnt FROM context_usage WHERE timestamp > ?`,
233
+ ).get(cutoff) as { cnt: number } | undefined;
234
+ return row?.cnt ?? 0;
235
+ }
236
+
237
+ /**
238
+ * Decide whether the heavy lane should run on this tick. Returns the
239
+ * reason for skipping so the journal row can record it.
240
+ */
241
+ export function shouldRunHeavyMaintenance(
242
+ store: Store,
243
+ now: Date,
244
+ cfg: HeavyMaintenanceConfig = {},
245
+ ): { run: boolean; reason?: string } {
246
+ const merged = { ...DEFAULT_CONFIG, ...cfg };
247
+ if (!isInQuietWindow(now, merged.windowStartHour, merged.windowEndHour)) {
248
+ return { run: false, reason: "outside_window" };
249
+ }
250
+ const usages = countRecentContextUsages(store, 10);
251
+ if (usages > merged.maxContextUsagesPer10m) {
252
+ return { run: false, reason: "query_rate_high" };
253
+ }
254
+ return { run: true };
255
+ }
256
+
257
+ // =============================================================================
258
+ // Stale-first selection helpers
259
+ // =============================================================================
260
+
261
+ /**
262
+ * Select up to `limit` observation doc IDs ordered by stale-first:
263
+ * least-recently-recalled (recall_stats.last_recalled_at ASC, NULL first)
264
+ * with documents.last_accessed_at as a fallback when recall_stats is empty.
265
+ *
266
+ * Used by tests and operators who want to inspect the heavy-lane batch
267
+ * without actually running Phase 2. The real Phase 2 SQL inside
268
+ * `consolidateObservations` applies its own stale-first ordering when
269
+ * `staleOnly: true` is passed.
270
+ */
271
+ export function selectStaleObservationBatch(
272
+ store: Store,
273
+ limit: number,
274
+ ): number[] {
275
+ const rows = store.db.prepare(
276
+ `SELECT d.id FROM documents d
277
+ LEFT JOIN recall_stats rs ON rs.doc_id = d.id
278
+ WHERE d.active = 1
279
+ AND d.content_type = 'observation'
280
+ ORDER BY
281
+ COALESCE(rs.last_recalled_at, d.last_accessed_at, d.modified_at) ASC,
282
+ d.modified_at ASC
283
+ LIMIT ?`,
284
+ ).all(limit) as { id: number }[];
285
+ return rows.map(r => r.id);
286
+ }
287
+
288
+ /**
289
+ * Select up to `limit` decision/preference/milestone/problem doc IDs
290
+ * ordered by stale-first for Phase 3 deductive synthesis.
291
+ */
292
+ export function selectStaleDeductiveBatch(
293
+ store: Store,
294
+ limit: number,
295
+ ): number[] {
296
+ const DEDUCTIVE_TYPES = ["decision", "preference", "milestone", "problem"];
297
+ const placeholders = DEDUCTIVE_TYPES.map(() => "?").join(",");
298
+ const rows = store.db.prepare(
299
+ `SELECT d.id FROM documents d
300
+ LEFT JOIN recall_stats rs ON rs.doc_id = d.id
301
+ WHERE d.active = 1
302
+ AND d.content_type IN (${placeholders})
303
+ ORDER BY
304
+ COALESCE(rs.last_recalled_at, d.last_accessed_at, d.modified_at) ASC,
305
+ d.modified_at ASC
306
+ LIMIT ?`,
307
+ ).all(...DEDUCTIVE_TYPES, limit) as { id: number }[];
308
+ return rows.map(r => r.id);
309
+ }
310
+
311
+ /**
312
+ * Select up to `limit` observation doc IDs ranked by surprisal score
313
+ * (k-NN average neighbor distance — higher = more anomalous). Wraps
314
+ * `computeSurprisalScores` so the heavy lane can swap in anomaly-first
315
+ * selection via `useSurprisalSelector: true`.
316
+ */
317
+ export function selectSurprisingObservationBatch(
318
+ store: Store,
319
+ limit: number,
320
+ ): number[] {
321
+ const results = computeSurprisalScores(store, { limit });
322
+ return results.map(r => r.docId);
323
+ }
324
+
325
+ // =============================================================================
326
+ // Worker topology
327
+ // =============================================================================
328
+
329
+ let heavyTimer: Timer | null = null;
330
+ let heavyRunning = false;
331
+
332
+ /**
333
+ * Run a single heavy-lane tick: gate check → worker lease → Phase 2 → Phase 3
334
+ * → journal row. Exported for tests and for manual invocation via a future
335
+ * `clawmem heavy-lane --once` CLI flag.
336
+ */
337
+ export async function runHeavyMaintenanceTick(
338
+ store: Store,
339
+ llm: LlamaCpp,
340
+ cfg: HeavyMaintenanceConfig = {},
341
+ ): Promise<MaintenanceRunSummary[]> {
342
+ const merged = { ...DEFAULT_CONFIG, ...cfg };
343
+ const workerName = cfg.workerName ?? DEFAULT_WORKER_NAME;
344
+ const clock = cfg.clock ?? (() => new Date());
345
+ const results: MaintenanceRunSummary[] = [];
346
+
347
+ const now = clock();
348
+ const gate = shouldRunHeavyMaintenance(store, now, cfg);
349
+ if (!gate.run) {
350
+ const skippedId = insertMaintenanceRun(store, {
351
+ lane: "heavy",
352
+ phase: "gate",
353
+ status: "skipped",
354
+ reason: gate.reason ?? "unknown",
355
+ finishedAt: new Date().toISOString(),
356
+ });
357
+ results.push(loadMaintenanceRun(store, skippedId));
358
+ return results;
359
+ }
360
+
361
+ const lease = await withWorkerLease(
362
+ store,
363
+ workerName,
364
+ merged.leaseTtlMs,
365
+ async () => {
366
+ // Phase 2 — consolidation
367
+ const phase2Id = insertMaintenanceRun(store, {
368
+ lane: "heavy",
369
+ phase: "consolidate",
370
+ status: "started",
371
+ });
372
+ try {
373
+ // Surprisal selector path: compute anomaly-first candidate ids up
374
+ // front, then plumb them into consolidateObservations via
375
+ // candidateIds. When the surprisal backend returns empty (no
376
+ // embeddings, small vault, k-NN unavailable), fall through to
377
+ // stale-first ordering so the heavy lane still does useful work.
378
+ let candidateIds: number[] | undefined;
379
+ let selectorUsed: "stale-first" | "surprisal" | "surprisal-fallback-stale";
380
+ if (merged.useSurprisalSelector) {
381
+ candidateIds = selectSurprisingObservationBatch(
382
+ store,
383
+ merged.staleObservationLimit,
384
+ );
385
+ if (candidateIds.length === 0) {
386
+ // Nothing surprising — degrade to stale-first so the lane
387
+ // does not become a no-op on vaults without embeddings.
388
+ candidateIds = undefined;
389
+ selectorUsed = "surprisal-fallback-stale";
390
+ } else {
391
+ selectorUsed = "surprisal";
392
+ }
393
+ } else {
394
+ selectorUsed = "stale-first";
395
+ }
396
+
397
+ await consolidateObservations(store, llm, {
398
+ maxDocs: merged.staleObservationLimit,
399
+ guarded: true,
400
+ staleOnly: selectorUsed !== "surprisal",
401
+ candidateIds,
402
+ });
403
+ finalizeMaintenanceRun(store, phase2Id, {
404
+ status: "completed",
405
+ selectedCount: candidateIds
406
+ ? candidateIds.length
407
+ : merged.staleObservationLimit,
408
+ metrics: {
409
+ selector: selectorUsed,
410
+ ...(candidateIds ? { candidateCount: candidateIds.length } : {}),
411
+ },
412
+ });
413
+ } catch (err) {
414
+ finalizeMaintenanceRun(store, phase2Id, {
415
+ status: "failed",
416
+ reason: "phase2_exception",
417
+ metrics: { error: (err as Error).message },
418
+ });
419
+ }
420
+ results.push(loadMaintenanceRun(store, phase2Id));
421
+
422
+ // Phase 3 — deductive synthesis
423
+ const phase3Id = insertMaintenanceRun(store, {
424
+ lane: "heavy",
425
+ phase: "deductive",
426
+ status: "started",
427
+ });
428
+ try {
429
+ const stats: DeductiveSynthesisStats = await generateDeductiveObservations(
430
+ store,
431
+ llm,
432
+ {
433
+ maxRecent: merged.staleDeductiveLimit,
434
+ guarded: true,
435
+ staleOnly: true,
436
+ },
437
+ );
438
+ finalizeMaintenanceRun(store, phase3Id, {
439
+ status: "completed",
440
+ selectedCount: stats.considered,
441
+ processedCount: stats.drafted,
442
+ createdCount: stats.created,
443
+ rejectedCount: stats.rejected,
444
+ nullCallCount: stats.nullCalls,
445
+ metrics: {
446
+ accepted: stats.accepted,
447
+ contaminationRejects: stats.contaminationRejects,
448
+ invalidIndexRejects: stats.invalidIndexRejects,
449
+ unsupportedRejects: stats.unsupportedRejects,
450
+ emptyRejects: stats.emptyRejects,
451
+ dedupSkipped: stats.dedupSkipped,
452
+ validatorFallbackAccepts: stats.validatorFallbackAccepts,
453
+ },
454
+ });
455
+ } catch (err) {
456
+ finalizeMaintenanceRun(store, phase3Id, {
457
+ status: "failed",
458
+ reason: "phase3_exception",
459
+ metrics: { error: (err as Error).message },
460
+ });
461
+ }
462
+ results.push(loadMaintenanceRun(store, phase3Id));
463
+ },
464
+ );
465
+
466
+ if (!lease.acquired) {
467
+ const skippedId = insertMaintenanceRun(store, {
468
+ lane: "heavy",
469
+ phase: "gate",
470
+ status: "skipped",
471
+ reason: "lease_unavailable",
472
+ finishedAt: new Date().toISOString(),
473
+ });
474
+ results.push(loadMaintenanceRun(store, skippedId));
475
+ }
476
+
477
+ return results;
478
+ }
479
+
480
+ function loadMaintenanceRun(store: Store, id: number): MaintenanceRunSummary {
481
+ const row = store.db.prepare(
482
+ `SELECT id, lane, phase, status, reason, selected_count, processed_count,
483
+ created_count, updated_count, rejected_count, null_call_count,
484
+ started_at, finished_at, metrics_json
485
+ FROM maintenance_runs WHERE id = ?`,
486
+ ).get(id) as MaintenanceRunSummary | undefined;
487
+ if (!row) {
488
+ throw new Error(`loadMaintenanceRun: row ${id} not found`);
489
+ }
490
+ return row;
491
+ }
492
+
493
+ /**
494
+ * Start the heavy-maintenance worker loop. Fire-and-forget — returns a stop
495
+ * function the caller can invoke on process shutdown. Off by default — the
496
+ * caller decides whether to start it via `CLAWMEM_HEAVY_LANE=true` or equivalent.
497
+ *
498
+ * A reentrancy guard prevents overlapping ticks if a prior tick is still
499
+ * running when the next interval fires. Separate from the DB-backed lease,
500
+ * which prevents overlap across processes.
501
+ */
502
+ export function startHeavyMaintenanceWorker(
503
+ store: Store,
504
+ llm: LlamaCpp,
505
+ cfg: HeavyMaintenanceConfig = {},
506
+ ): () => void {
507
+ const merged = { ...DEFAULT_CONFIG, ...cfg };
508
+ // Clamp interval to minimum 30 seconds so buggy configs can't pin the CPU.
509
+ const interval = Math.max(30_000, merged.intervalMs);
510
+
511
+ console.log(
512
+ `[heavy-lane] Starting worker (interval=${interval}ms, ` +
513
+ `window=${merged.windowStartHour ?? "always"}-${merged.windowEndHour ?? "always"}, ` +
514
+ `maxUsagesPer10m=${merged.maxContextUsagesPer10m})`,
515
+ );
516
+
517
+ heavyTimer = setInterval(async () => {
518
+ if (heavyRunning) {
519
+ console.log("[heavy-lane] Skipping tick (still running)");
520
+ return;
521
+ }
522
+ heavyRunning = true;
523
+ try {
524
+ await runHeavyMaintenanceTick(store, llm, cfg);
525
+ } catch (err) {
526
+ console.error("[heavy-lane] Tick failed:", err);
527
+ } finally {
528
+ heavyRunning = false;
529
+ }
530
+ }, interval);
531
+ heavyTimer.unref();
532
+
533
+ return () => {
534
+ if (heavyTimer) {
535
+ clearInterval(heavyTimer);
536
+ heavyTimer = null;
537
+ console.log("[heavy-lane] Worker stopped");
538
+ }
539
+ };
540
+ }
package/src/mcp.ts CHANGED
@@ -39,6 +39,7 @@ import { classifyIntent, decomposeQuery, extractTemporalConstraint, type IntentT
39
39
  import { adaptiveTraversal, mergeTraversalResults, mpfpTraversal } from "./graph-traversal.ts";
40
40
  import { getDefaultLlamaCpp } from "./llm.ts";
41
41
  import { startConsolidationWorker, stopConsolidationWorker } from "./consolidation.ts";
42
+ import { startHeavyMaintenanceWorker, type HeavyMaintenanceConfig } from "./maintenance.ts";
42
43
  import { listVaults, loadVaultConfig } from "./config.ts";
43
44
  import { getEntityGraphNeighbors, searchEntities } from "./entity.ts";
44
45
 
@@ -2604,10 +2605,42 @@ This is the recommended entry point for ALL memory queries.`,
2604
2605
  startConsolidationWorker(store, llm, intervalMs);
2605
2606
  }
2606
2607
 
2608
+ // v0.8.0 Ext 5: Start heavy-maintenance worker if enabled. Runs on a
2609
+ // longer interval than the light lane, only inside a configurable quiet
2610
+ // window, and gated by context_usage query-rate so interactive sessions
2611
+ // are never starved. Off by default.
2612
+ let stopHeavyLane: (() => void) | null = null;
2613
+ if (Bun.env.CLAWMEM_HEAVY_LANE === "true") {
2614
+ const llm = getDefaultLlamaCpp();
2615
+ const cfg: HeavyMaintenanceConfig = {
2616
+ intervalMs: Bun.env.CLAWMEM_HEAVY_LANE_INTERVAL
2617
+ ? parseInt(Bun.env.CLAWMEM_HEAVY_LANE_INTERVAL, 10)
2618
+ : undefined,
2619
+ windowStartHour: Bun.env.CLAWMEM_HEAVY_LANE_WINDOW_START
2620
+ ? parseInt(Bun.env.CLAWMEM_HEAVY_LANE_WINDOW_START, 10)
2621
+ : null,
2622
+ windowEndHour: Bun.env.CLAWMEM_HEAVY_LANE_WINDOW_END
2623
+ ? parseInt(Bun.env.CLAWMEM_HEAVY_LANE_WINDOW_END, 10)
2624
+ : null,
2625
+ maxContextUsagesPer10m: Bun.env.CLAWMEM_HEAVY_LANE_MAX_USAGES
2626
+ ? parseInt(Bun.env.CLAWMEM_HEAVY_LANE_MAX_USAGES, 10)
2627
+ : undefined,
2628
+ staleObservationLimit: Bun.env.CLAWMEM_HEAVY_LANE_OBS_LIMIT
2629
+ ? parseInt(Bun.env.CLAWMEM_HEAVY_LANE_OBS_LIMIT, 10)
2630
+ : undefined,
2631
+ staleDeductiveLimit: Bun.env.CLAWMEM_HEAVY_LANE_DED_LIMIT
2632
+ ? parseInt(Bun.env.CLAWMEM_HEAVY_LANE_DED_LIMIT, 10)
2633
+ : undefined,
2634
+ useSurprisalSelector: Bun.env.CLAWMEM_HEAVY_LANE_SURPRISAL === "true",
2635
+ };
2636
+ stopHeavyLane = startHeavyMaintenanceWorker(store, llm, cfg);
2637
+ }
2638
+
2607
2639
  // Signal handlers for graceful shutdown
2608
2640
  process.on("SIGINT", () => {
2609
2641
  console.error("\n[mcp] Received SIGINT, shutting down...");
2610
2642
  stopConsolidationWorker();
2643
+ if (stopHeavyLane) stopHeavyLane();
2611
2644
  closeAllStores();
2612
2645
  process.exit(0);
2613
2646
  });
@@ -2615,6 +2648,7 @@ This is the recommended entry point for ALL memory queries.`,
2615
2648
  process.on("SIGTERM", () => {
2616
2649
  console.error("\n[mcp] Received SIGTERM, shutting down...");
2617
2650
  stopConsolidationWorker();
2651
+ if (stopHeavyLane) stopHeavyLane();
2618
2652
  closeAllStores();
2619
2653
  process.exit(0);
2620
2654
  });
package/src/store.ts CHANGED
@@ -854,6 +854,41 @@ function initializeDatabase(db: Database): void {
854
854
  if (!mrColNames.has("contradict_confidence")) {
855
855
  try { db.exec(`ALTER TABLE memory_relations ADD COLUMN contradict_confidence REAL`); } catch { /* column exists */ }
856
856
  }
857
+
858
+ // v0.8.0 Ext 5: Heavy maintenance lane journal. Every scheduled attempt
859
+ // writes one row — including skips — so operators can reconstruct why a
860
+ // lane did or did not run on any tick.
861
+ db.exec(`
862
+ CREATE TABLE IF NOT EXISTS maintenance_runs (
863
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
864
+ lane TEXT NOT NULL,
865
+ phase TEXT NOT NULL,
866
+ status TEXT NOT NULL,
867
+ reason TEXT,
868
+ selected_count INTEGER NOT NULL DEFAULT 0,
869
+ processed_count INTEGER NOT NULL DEFAULT 0,
870
+ created_count INTEGER NOT NULL DEFAULT 0,
871
+ updated_count INTEGER NOT NULL DEFAULT 0,
872
+ rejected_count INTEGER NOT NULL DEFAULT 0,
873
+ null_call_count INTEGER NOT NULL DEFAULT 0,
874
+ started_at TEXT NOT NULL DEFAULT (datetime('now')),
875
+ finished_at TEXT,
876
+ metrics_json TEXT
877
+ )
878
+ `);
879
+ db.exec(`CREATE INDEX IF NOT EXISTS idx_maintenance_runs_lane_started ON maintenance_runs(lane, started_at DESC)`);
880
+
881
+ // v0.8.0 Ext 5: DB-backed worker lease table for multi-process exclusivity
882
+ // on the heavy lane. Lease holders fence via random token; expired leases
883
+ // are reclaimed via atomic upsert inside a transaction.
884
+ db.exec(`
885
+ CREATE TABLE IF NOT EXISTS worker_leases (
886
+ worker_name TEXT PRIMARY KEY,
887
+ lease_token TEXT NOT NULL,
888
+ acquired_at TEXT NOT NULL,
889
+ expires_at TEXT NOT NULL
890
+ )
891
+ `);
857
892
  }
858
893
 
859
894
 
@@ -0,0 +1,141 @@
1
+ /**
2
+ * ClawMem Worker Lease (v0.8.0 Ext 5)
3
+ *
4
+ * DB-backed exclusive lease for heavy-lane workers. Uses the `worker_leases`
5
+ * table (schema in store.ts) instead of module globals so multiple processes
6
+ * sharing a vault cannot run heavy maintenance concurrently.
7
+ *
8
+ * Lease lifecycle:
9
+ * 1. acquireWorkerLease inserts or reclaims an expired row via transaction
10
+ * and returns a random fencing token on success.
11
+ * 2. The holder runs its work.
12
+ * 3. releaseWorkerLease deletes the row only if the caller's token matches,
13
+ * so a lease reclaimed by another worker after TTL expiry cannot be
14
+ * torn down by the original holder.
15
+ *
16
+ * withWorkerLease wraps acquire/release around a callback; failure to acquire
17
+ * is a silent no-op (returns `{acquired: false}`) — callers should log a
18
+ * `skipped` journal row with reason `lease_unavailable`.
19
+ */
20
+
21
+ import { randomBytes } from "node:crypto";
22
+ import type { Store } from "./store.ts";
23
+
24
+ export interface LeaseAcquireResult {
25
+ acquired: boolean;
26
+ token?: string;
27
+ expiresAt?: string;
28
+ }
29
+
30
+ function nowIso(now: Date = new Date()): string {
31
+ return now.toISOString();
32
+ }
33
+
34
+ function futureIso(now: Date, ttlMs: number): string {
35
+ return new Date(now.getTime() + ttlMs).toISOString();
36
+ }
37
+
38
+ /**
39
+ * Attempt to acquire an exclusive lease on `workerName` for `ttlMs`.
40
+ *
41
+ * Returns `{acquired: true, token, expiresAt}` on success, or
42
+ * `{acquired: false}` if another worker holds a live (non-expired) lease.
43
+ *
44
+ * Race-safe under multi-process contention: uses a single
45
+ * `INSERT ... ON CONFLICT DO UPDATE ... WHERE expires_at <= ?` statement
46
+ * so the "no row → insert" and "expired row → update" paths cannot
47
+ * both fire for two concurrent callers. SQLite's changes() reports 1
48
+ * iff THIS call either inserted a fresh row or reclaimed an expired row;
49
+ * 0 means a live lease was held by someone else.
50
+ *
51
+ * Any SQLITE_BUSY / constraint failure is translated to
52
+ * `{ acquired: false }` so the advertised non-throw contract holds for
53
+ * callers that are layering `shouldRunHeavyMaintenance` above this.
54
+ */
55
+ export function acquireWorkerLease(
56
+ store: Store,
57
+ workerName: string,
58
+ ttlMs: number,
59
+ now: Date = new Date(),
60
+ ): LeaseAcquireResult {
61
+ if (ttlMs <= 0) {
62
+ throw new Error(`acquireWorkerLease: ttlMs must be positive, got ${ttlMs}`);
63
+ }
64
+ const token = randomBytes(16).toString("hex");
65
+ const acquiredAt = nowIso(now);
66
+ const expiresAt = futureIso(now, ttlMs);
67
+
68
+ try {
69
+ // Single-statement atomic acquire. The WHERE on the UPDATE clause
70
+ // only reclaims when the existing lease has expired (its expires_at
71
+ // <= our acquired_at); otherwise the ON CONFLICT DO UPDATE becomes
72
+ // a no-op and SQLite reports changes=0.
73
+ const result = store.db.prepare(
74
+ `INSERT INTO worker_leases
75
+ (worker_name, lease_token, acquired_at, expires_at)
76
+ VALUES (?, ?, ?, ?)
77
+ ON CONFLICT(worker_name) DO UPDATE SET
78
+ lease_token = excluded.lease_token,
79
+ acquired_at = excluded.acquired_at,
80
+ expires_at = excluded.expires_at
81
+ WHERE worker_leases.expires_at <= excluded.acquired_at`,
82
+ ).run(workerName, token, acquiredAt, expiresAt);
83
+
84
+ if (result.changes === 0) {
85
+ return { acquired: false };
86
+ }
87
+ return { acquired: true, token, expiresAt };
88
+ } catch (err) {
89
+ // Defensive fallback: any unexpected DB error (SQLITE_BUSY under
90
+ // extreme contention, constraint error from schema drift, etc.) is
91
+ // translated to a lease-unavailable result instead of bubbling up,
92
+ // so heavy-maintenance callers always get a deterministic
93
+ // "skipped/lease_unavailable" journal row.
94
+ console.error(
95
+ `[worker-lease] acquire error for ${workerName}: ${(err as Error).message}`,
96
+ );
97
+ return { acquired: false };
98
+ }
99
+ }
100
+
101
+ /**
102
+ * Release a lease if the caller's token still matches. Returns `true` if
103
+ * the lease was owned and deleted, `false` if a different token held it
104
+ * (e.g., TTL expired and another worker reclaimed).
105
+ */
106
+ export function releaseWorkerLease(
107
+ store: Store,
108
+ workerName: string,
109
+ token: string,
110
+ ): boolean {
111
+ const result = store.db.prepare(
112
+ `DELETE FROM worker_leases WHERE worker_name = ? AND lease_token = ?`,
113
+ ).run(workerName, token);
114
+ return result.changes > 0;
115
+ }
116
+
117
+ /**
118
+ * Run `fn` under an exclusive lease on `workerName`. If the lease cannot
119
+ * be acquired, returns `{acquired: false}` without invoking `fn`. The
120
+ * lease is always released in a `finally` block, even if `fn` throws.
121
+ *
122
+ * Rethrows any error from `fn` — callers are responsible for translating
123
+ * exceptions into journal rows.
124
+ */
125
+ export async function withWorkerLease<T>(
126
+ store: Store,
127
+ workerName: string,
128
+ ttlMs: number,
129
+ fn: () => Promise<T>,
130
+ ): Promise<{ acquired: boolean; result?: T }> {
131
+ const lease = acquireWorkerLease(store, workerName, ttlMs);
132
+ if (!lease.acquired || !lease.token) {
133
+ return { acquired: false };
134
+ }
135
+ try {
136
+ const result = await fn();
137
+ return { acquired: true, result };
138
+ } finally {
139
+ releaseWorkerLease(store, workerName, lease.token);
140
+ }
141
+ }