@unblocklabs/unblock-memory 0.7.3 → 0.8.0

package/dist/src/training-input.js

@@ -1,6 +1,6 @@
 import { createHash } from "node:crypto";
 import { DatabaseSync } from "node:sqlite";
-import { ACTIVE_EVENT_COUNT_SQL, ACTIVE_EVENT_ROWS_SQL, assertAgentTranscriptSchema } from "./agent-transcript.js";
+import { AgentTranscriptReader } from "./agent-transcript.js";
 import { messageText } from "./whisperer-context.js";
 import { conversationUserText } from "./response-text.js";
 import { prepareQueryConversation, QUERY_CONTRACT, QueryInputBudgetError } from "./query-contract.js";
@@ -133,12 +133,13 @@ export function trainingExamples(rows, renew) {
 /** Only active events; no Markdown projections, archived branches, or tool bodies. */
 export class TrainingTranscriptReader {
     #db;
+    #transcripts;
     #lineage;
     constructor(path, agentId) {
         this.#db = new DatabaseSync(path, { readOnly: true });
         try {
             this.#db.exec("PRAGMA query_only=ON; PRAGMA busy_timeout=1000");
-            assertAgentTranscriptSchema(this.#db, agentId, "Unsupported training transcript schema or agent");
+            this.#transcripts = new AgentTranscriptReader(this.#db, agentId, "Unsupported training transcript schema or agent");
             const columns = this.#db.prepare("PRAGMA table_info(session_windows)").all().map(c => c.name);
             this.#lineage = ["parent_session_key", "spawned_by", "plugin_owner_id", "hook_external_content_source"].every(c => columns.includes(c));
         }
@@ -161,11 +162,12 @@ export class TrainingTranscriptReader {
                 /:(?:cron|subagent|heartbeat|hook)(?::|$)/i.test(String(session.session_key)) ||
                 session.parent_session_key || session.spawned_by || session.plugin_owner_id || session.hook_external_content_source)
                 return null;
-            const size = this.#db.prepare(ACTIVE_EVENT_COUNT_SQL).get(sessionId);
-            if (Number(size.n) > 50_000 || Number(size.bytes) > 32_000_000)
+            const snapshot = this.#transcripts.read(sessionId, { maxEvents: 50_000, maxBytes: 32_000_000 });
+            if (snapshot.kind === "oversized")
                 return { oversized: true };
-            const rows = this.#db.prepare(ACTIVE_EVENT_ROWS_SQL).iterate(sessionId);
-            return trainingExamples(rows, renew);
+            if (snapshot.kind !== "ready")
+                return { unavailable: true };
+            return trainingExamples(snapshot.rows, renew);
         }
         finally {
             this.#db.exec("COMMIT");

package/dist/src/training-queries.d.ts

@@ -34,6 +34,7 @@ export declare function generateTrainingQueries(source: Source, store: TrainingS
         sessions: number;
         excludedSessions: number;
         oversizedSessions: number;
+        unavailableSessions: number;
         eligible: number;
         users: number;
         filtered: number;
@@ -79,6 +80,7 @@ export declare function evaluateTrainingQueries(source: Source, store: TrainingS
         sessions: number;
         excludedSessions: number;
         oversizedSessions: number;
+        unavailableSessions: number;
         eligible: number;
         users: number;
         filtered: number;

package/dist/src/training.d.ts

@@ -13,6 +13,7 @@ export declare function collectTraining(source: Source, store?: TrainingStore, o
     sessions: number;
     excludedSessions: number;
     oversizedSessions: number;
+    unavailableSessions: number;
     eligible: number;
     users: number;
     filtered: number;
@@ -39,6 +40,7 @@ export declare function runTraining(source: Source, store: TrainingStore, config
         sessions: number;
         excludedSessions: number;
         oversizedSessions: number;
+        unavailableSessions: number;
         eligible: number;
         users: number;
         filtered: number;

package/dist/src/training.js

@@ -5,7 +5,7 @@ import { judgeTrainingInput, TRAINING_GATE_QUESTIONS } from "./training-gate.js"
 export function collectTraining(source, store, options = {}) {
     const since = options.since ?? 0, until = options.until ?? Number.MAX_SAFE_INTEGER;
     const reader = new TrainingTranscriptReader(source.databasePath, source.agentId);
-    const result = { sessions: 0, excludedSessions: 0, oversizedSessions: 0, eligible: 0,
+    const result = { sessions: 0, excludedSessions: 0, oversizedSessions: 0, unavailableSessions: 0, eligible: 0,
         users: 0, filtered: 0, oversized: 0, unanswered: 0, added: 0, changed: 0, unchanged: 0, retired: 0,
         review: [] };
     try {
@@ -14,6 +14,11 @@ export function collectTraining(source, store, options = {}) {
             store?.renew();
             const extracted = reader.read(id, () => store?.renew());
             result.sessions++;
+            if (extracted && "unavailable" in extracted) {
+                result.unavailableSessions++;
+                store?.unavailable(id);
+                continue;
+            }
             if (extracted && "oversized" in extracted) {
                 result.oversizedSessions++;
                 store?.unavailable(id);

package/docs/configuration.md

@@ -11,7 +11,8 @@ settings; a rotated credential file is reread without a restart.
 
 | Feature | Required settings/dependencies | TypeSafe disabled / no key | Provider or unreadable-key failure |
 | --- | --- | --- | --- |
-| Ordinary search/get | Installed/enabled memory slot; configured corpora | Unchanged local retrieval | Unchanged; its own indexing/embedding errors still matter |
+| `memory_search` | Installed/enabled memory slot; configured corpora; TypeSafe key | Explicit unavailable error | Successful judged siblings returned with a partial warning; total failure is an error; no vector fallback |
+| `memory_get` | Installed/enabled memory slot; configured corpora | Unchanged local indexed reads | Unchanged; its own indexing errors still matter |
 | Skill Whisperer | `skillWhisperer.enabled`, skills corpus, host hooks | Best local vector candidate meeting `minScore` | Exclude failed candidates; successful siblings may qualify; no vector fallback |
 | Memory Whisperer | `memoryWhisperer.enabled`, explicit approved corpora, host hooks | No hints | Exclude failed candidates; recall/deadline failures still suppress the turn |
 | Complementary hints | Enabled Memory Whisperer + `complementaryHints` | No additional judgment; base hints also require a key | Keep baseline hints unless the total deadline expires |
@@ -202,7 +203,7 @@ contract; these tables explain their effects.
 | `corpora[].kind` | Required for explicit entries | `files`, `sessions`, or `skills` |
 | `corpora[].paths` | Required for files/skills | Nonempty exact-file/directory/glob list; workspace-relative, absolute or `~/`; directory means recursive Markdown; does not grant host write trust |
 | `corpora[sessions].chatTypes` | `['channel','group']` | Nonempty subset of channel/group/direct; DMs require `direct` |
-| `corpora[sessions].maxExpandedTokens` | `500` | 1–10,000; ordinary search uses full turn/message when it fits, otherwise preserves the matched chunk. Whisperer instead shares a fixed 1,200-character renderer with training. |
+| `corpora[sessions].maxExpandedTokens` | `500` | 1–10,000; internal host vector retrieval expands a full turn/message when it fits. Public search and Whisperer share the fixed 1,200-character complete-passage renderer with training. |
 | `corpora[sessions].syncIntervalMinutes` | `60` | 0–1,440; zero manual-only; first scheduled sync after one interval; requires running Gateway |
 | `keepEmbeddingModelWarm` | `true` | Retain embedding model/context after first use; false allows five-minute idle disposal |
 | `analysis.executable` | Unset | Optional absolute local worker path; enables ability to recluster, not automatic scheduling |

package/docs/memory-training.md

@@ -78,7 +78,9 @@ bytes, so changed corpus content still invalidates caches.
 
 ## Input and historical boundaries
 
-Read original agent SQLite active-branch conversations (schema 17–19), including
+Read original agent SQLite active-branch conversations through the shared read-only
+adapter, including compressed payloads, while deferring cold or oversized sessions
+without retiring their previously collected examples. This includes
 direct/group/channel chats, excluding cron, heartbeat, spawned/subagent, hook/plugin
 and untyped diagnostic sessions. Exclude explicit bots, synthetic messages,
 analysis/thinking, errors and tool payloads. Ordinary older users need not have

package/docs/response-audit.md

@@ -202,8 +202,8 @@ factual verification. Week buckets
 may be partial, and several exchanges in one session are not independent. Wilson
 intervals are descriptive, not calibrated confidence about overall agent ability.
 
-Each run selects at most 100 recent sessions for inference, each at most 2,000 active events/2M
-characters; episodes must fit 24K characters and six feedback messages without
+Each run selects at most 100 recent sessions containing approved-sender messages for inference,
+each at most 2,000 active events/2M decoded UTF-8 bytes; episodes must fit 24K characters and six feedback messages without
 truncating the answer. Coverage counts describe the scanned sessions; only episodes
 within `lookbackDays` are judged. Caps, failures and no-feedback cases remain visible.
 Saved sessions in the report window are also reconciled independently of that

package/docs/retrieval.md

@@ -6,26 +6,31 @@
 
 `memory_search` searches every configured **non-skill** corpus for this agent by
 default. Select named corpora, or use `["all"]` alone. An empty list or unknown
-name is an error. Default: **5 results**, vector `minScore: 0.3`; configurable
-`maxResults` is 1–20 and `minScore` is 0–1. No TypeSafe, BM25, query expansion
-or reranker runs in this tool.
+name is an error. Supply `bm25Query` (distinctive keywords/names) and
+`vectorQuery` (a natural-language question describing the information needed).
+Each lane retrieves ten candidates; complete passages are deduplicated and
+independently judged by TypeSafe against `vectorQuery`. No query generation or
+recall gate runs. Default: **5 results**, `minUsefulness: 0.7`;
+`maxResults` is 1–20 and `minUsefulness` is 0–1. The old `query` and vector
+`minScore` parameters are removed, not silently reinterpreted.
 
 Example tool input (the `memory` corpus exists by default):
 
 ```json
-{ "query": "Who approved the staging rollout?", "corpora": ["memory"], "maxResults": 5 }
+{ "bm25Query": "staging rollout approval", "vectorQuery": "Who approved the staging rollout?", "corpora": ["memory"], "maxResults": 5 }
 ```
 
 Results use compact JSON and carry `path`, `startLine`, `endLine`, `snippet`,
-`corpus`, and `score`/`vectorScore` rounded to hundredths. Ranking and threshold
+`corpus`, and a usefulness `score` rounded to hundredths. Ranking and threshold
 filtering still use full precision. The constant `source` and top-level `provider`
 fields are omitted; `path` plus line numbers replace the redundant `citation`.
 Session hits also carry session metadata and, when available,
 `messageTimestamp`: the original timestamp text (including timezone) of the message
 containing the matched chunk. It stays tied to that message even when the excerpt
 expands to the surrounding turn. Missing timestamps are omitted, not replaced by
-session start time. Vector similarity is a retrieval signal, not confidence in
-the truth of a claim.
+session start time. Usefulness is not confidence in the truth of a claim.
+Passages retain complete evidence within 1,200 characters; oversized matches
+are omitted, never trimmed into a misleading prefix.
 
 Session `snippet` values are arrays of messages, in source order:
 
@@ -61,56 +66,40 @@ Search snippets are leads. Inspect attribution, qualifications, dates and adjace
 context before relying on a factual claim. An old plan is not proof it happened,
 and memory does not grant permission to act. No results may mean the wrong corpus,
 a high threshold, an unsynced session or an unavailable index—not absence of the fact.
+Search requires enabled TypeSafe and a configured key. It sends the semantic
+query, candidate excerpts, dates, corpus and source paths to TypeSafe. Missing
+credentials and total judging failure include an `error` alongside empty `results`;
+partial judging failure includes a `warning` and only successfully judged results.
+Oversized queries are rejected before credentials, retrieval or provider calls.
 Search initialization errors include an `error` alongside empty `results`;
 other failures may surface as tool errors. Investigate them instead of reporting
 “nothing is remembered.”
+`memory_diagnostics` exposes content-free manual-search measurements under
+`whisperers.telemetry.operations.memorySearch`: end-to-end latency, retrieval and
+judging time, candidate/result counts and outcomes (including partial failures).
+These counters are separate from automatic `memoryWhisperer` and internal `vector`
+retrieval; no query or evidence text is recorded.
 
 Corpora and session filters select evidence; they are not audience ACLs. Normal
 tools can access this agent's configured non-skill corpora, not just its current
 chat. Only index material suitable for the agent's tool callers. Per-feature
 TypeSafe allowlists do not restrict ordinary retrieval.
 
-## QMD search modes
-
-The following describes QMD's TypeSafe-ranked `query` API (2.10+). Use the exact
-dependency in the installed plugin's package metadata when diagnosing an older
-installation.
-
-| Surface | Retrieval/ranking | Score / useful distinction |
-| --- | --- | --- |
-| Plugin `memory_search` | Direct vector search, expansion off | Vector similarity; ordinary agent recall |
-| QMD CLI `search` / SDK `searchLex` | Local BM25 | Normalized lexical relevance; useful for names, identifiers and exact phrases |
-| QMD CLI/SDK `vsearch` | Local vectors; standalone default includes local query expansion | Vector similarity; `--no-expand` / `expand:false` selects literal retrieval |
-| QMD CLI/MCP `query` / SDK `search` | Literal vectors + BM25, deduplicated source excerpts, independent TypeSafe usefulness ranking | Usefulness divided by 3, not cosine similarity; default limit 10/minScore 0 |
-| QMD `query --no-rerank` / SDK/MCP `rerank:false` | Local retrieval, best reciprocal retrieval-rank ordering | 1 / rank; explicitly skips remote scoring |
-
-Plain `query` takes `ceil(1.5 × limit)` candidates per backend, without local
-expansion or local reranking. Explicit `lex`/`vec`/`hyde` variants share the
-TypeSafe ranking policy by default. A supplied hypothetical `hyde` passage is
-a retrieval input, not evidence. Scores across modes are not interchangeable.
-
-**Migration:** Memory 0.3.22 removed the temporary `memory_xsearch` tool. QMD's
-`query` provides that hybrid retrieval/ranking functionality; there is no plugin
-`memory_query`, and `memory_search` did not become hybrid.
-
-Standalone QMD is a separate entry point. Its project/named/global index is not
-automatically this agent's `unblock-memory/index.sqlite`. Plugin corpus names
-also are not QMD collection IDs: each configured path maps to a `source-<hash>`
-collection, and a corpus can contain several. Establish the intended index,
-collection scope and installed QMD version before using CLI/MCP as an alternative.
-Do not run standalone collection/update/embed maintenance against a live
-plugin-managed index as a casual search fallback.
-
-QMD CLI/MCP read `TYPESAFE_API_KEY` or `TYPESAFE_API_KEY_FILE` from their own
-process; SDK callers can supply credentials. Plugin `typesafe.apiKeyFile` does
-not export a key to those processes. Ranked query sends the query, intent,
-selected excerpts, source paths and evaluation time to TypeSafe. Missing keys or
-scoring failures return errors, not vector fallback or a successful empty answer.
-Use an explicitly local mode when appropriate.
-
-See the [QMD guide](https://github.com/unblocklabs-ai/qmd#readme) for CLI/MCP syntax.
-Private skill frontmatter retrieval and response-audit lexical investigations are
-internal workflows, not extra public plugin search modes.
+## QMD entry points
+
+`memory_search` is the agent-facing ranked search. The QMD CLI `query`,
+`search` and `deep-search` alias, MCP `query` tool, and HTTP `/query` and
+`/search` routes are removed in the companion QMD change. Existing installations
+retain those commands until that QMD change is released and installed.
+
+QMD remains the local indexing/retrieval backend. Its BM25/vector SDK primitives,
+vector-only `vsearch`, indexed reads, collection maintenance and benchmark APIs
+remain available. Private skill retrieval and response-audit lexical investigations
+are internal workflows, not extra public plugin search tools.
+
+Standalone QMD and the plugin need not use the same index. Do not run standalone
+collection/update/embed maintenance against a live plugin-managed index as a
+casual search fallback.
 
 ## Sessions
 
@@ -127,7 +116,8 @@ corpora searchable. Supported fields are `startedFrom` and `startedTo`
 
 ```json
 {
-  "query": "deployment decision",
+  "bm25Query": "deployment decision",
+  "vectorQuery": "What deployment decision was made?",
   "sessionFilter": {
     "startedFrom": "2026-08-01T00:00:00Z",
     "provider": "slack",
@@ -155,13 +145,19 @@ fits. The default is `500`; the original semantic chunk is preserved when
 neither complete context fits, so expansion never clips the matched evidence.
 Run
 `memory_sync_sessions` to start a refresh, then use `memory_sync_status` to
-check its progress or result. The read-only adapter explicitly supports OpenClaw
-agent database schemas 17, 18, and 19 and validates its required columns before
-reading. Projections are private derived Markdown under the
+check its progress or result. The shared read-only transcript adapter validates
+agent identity and the storage capabilities it needs, rather than rejecting every
+new schema number. It reads both text and bounded Zstd payloads (schema 23+),
+counts decoded UTF-8 bytes for budgets, and preserves previous projections when
+history is cold, oversized, or unreadable. Cold history is deferred until restored
+by OpenClaw; the plugin never restores or migrates the source database. The public
+full-transcript SDK reader currently permits restoration, while its read-only
+catalog redacts and truncates text, so neither supplies this full-fidelity read-only
+contract. Extraction retains its supported host cursor API. Projections are private derived Markdown under the
 agent's `unblock-memory/sessions` state directory and can be rebuilt from
 OpenClaw at any time. Their embedded text contains only `# Transcript` and
 role-labeled, timestamped speaker messages; filtering metadata remains in the
-session manifest. Projection v7 also retains message metadata and exact character
+session manifest. Projection v8 retains message metadata and exact character
 boundaries there, without duplicating message bodies. Readers use those boundaries
 only when the projection hash matches the indexed document. Older/mismatched snapshots
 use a conservative heading parser that skips code fences and blockquotes; an unfenced
@@ -294,7 +290,9 @@ through BM25 and vectors respectively, retrieving ten candidates per backend acr
 the complete approved collection scope (not ten per collection). The lexical adapter
 uses literal OR keywords; quotation marks do not enable exact-phrase matching.
 API or generated-output failures skip the hint; there is no direct-vector fallback.
-TypeSafe evaluates one independent
+After query generation Whisperer uses the same discovery, deduplication and
+ranking core as `memory_search`, but judges against the bounded conversation
+rather than the explicit tool's semantic query alone. TypeSafe evaluates one independent
 Noul question per candidate in its own concurrent HTTP request: would a careful
 assistant use a specific factual detail from the excerpt when answering the current
 request? Partial answers and concrete leads count; repeated facts, topic/name matches
@@ -309,16 +307,18 @@ Explicit configured thresholds are preserved. Evaluate it on your own conversati
 **Privacy and budgets:** training and runtime prepare the same visible conversation:
 at most 8,192 pinned-LFM tokens and 24,000 UTF-8 bytes after serialization. The whole
 current request and most recent whole messages are retained; an oversized current
-request skips automatic memory. The judge receives that conversation plus one complete
-1,200-character excerpt, its corpus/source path, all represented message timestamps,
+request skips automatic memory before query generation or the TypeSafe recall gate.
+Diagnostics count `input_too_large`; completed logs report `outcome: "skipped"` and
+`reason: "input_too_large"`, without recording the input. The judge receives that
+conversation plus one complete 1,200-character excerpt, its corpus/source path, all represented message timestamps,
 and the request's evaluation time per HTTP request. A v2 query pair can therefore
 produce up to twenty passage requests before deduplication and eligibility filtering.
 Dates record when source messages were said, without inferring event dates.
 Session excerpts retain a complete turn or message when it fits,
 otherwise the complete matched chunk. Chunks exceeding the excerpt budget are
-skipped, never sliced. This shared Whisperer renderer uses the fixed 1,200-character
-budget independently of `sessions.maxExpandedTokens`; that setting still controls
-ordinary `memory_search` expansion, which is unchanged.
+skipped, never sliced. Public `memory_search` and Whisperer use the same fixed
+1,200-character renderer, independently of `sessions.maxExpandedTokens`.
+That setting controls only internal host-vector retrieval expansion.
 It does not fetch a complete historical transcript; the host may
 already have compacted the available context. System messages, thinking blocks,
 images, and tool-result messages are omitted;

package/openclaw.plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "unblock-memory",
   "name": "Unblock Memory",
-  "version": "0.7.3",
+  "version": "0.8.0",
   "description": "Indexes, retrieves, and analyzes configured workspace memory with existing QMD vectors.",
   "kind": "memory",
   "activation": { "onStartup": true },
@@ -64,7 +64,7 @@
     },
     "typesafe.enabled": {
       "label": "TypeSafe Judgments",
-      "help": "Shared provider gate for opted-in whisperers, people primer/save review, evidence/quality reviews and response audits. Does not enable features or approve corpora by itself. Ordinary search and People Whisperer injection stay local."
+      "help": "Shared provider gate for opted-in whisperers, people primer/save review, evidence/quality reviews and response audits. Does not enable features or approve corpora by itself. Memory search discovers locally and uses TypeSafe to rank usefulness; People Whisperer injection stays local."
     },
     "typesafe.apiKey": { "label": "TypeSafe API Key", "sensitive": true },
     "typesafe.timeoutMs": { "label": "TypeSafe Request Timeout", "help": "Per-request timeout, not the whole operation. People primer/dossier review use peoplePrimer.timeoutMs instead; Memory Whisperer has its own total deadline, and Skill Whisperer has a fixed 3-second total budget." },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@unblocklabs/unblock-memory",
-  "version": "0.7.3",
+  "version": "0.8.0",
   "description": "Workspace-native memory for OpenClaw, powered by QMD",
   "type": "module",
   "license": "MIT",
@@ -48,7 +48,7 @@
   },
   "dependencies": {
     "@huggingface/tokenizers": "0.2.0",
-    "@unblocklabs/qmd": "https://github.com/unblocklabs-ai/qmd/releases/download/v2.11.1/unblocklabs-qmd-2.11.1.tgz",
+    "@unblocklabs/qmd": "https://github.com/unblocklabs-ai/qmd/releases/download/v3.0.0/unblocklabs-qmd-3.0.0.tgz",
     "chokidar": "5.0.0",
     "js-tiktoken": "1.0.21",
     "picomatch": "^4.0.5",