@kisev/memomatic 1.0.0-dev.99.g48fbb8555698 → 12.0.1-dev.115.g8010bdfeea15

package/README.md

@@ -4,8 +4,8 @@
 
 `@kisev/memomatic` is a personal learning memory for agents following the OpenClaw
 memory architecture: a tiered Markdown corpus you can read as plain files, a
-rebuildable SQLite index with FTS5 and optional local embeddings, a `sessions`
-extraction pass, and a nightly `dream` consolidation sweep.
+rebuildable SQLite index with FTS5, optional local embeddings and an optional
+reranker, a `sessions` extraction pass, and a nightly `dream` consolidation sweep.
 
 ## Install
 
@@ -78,7 +78,7 @@ when memomatic is absent.
 
 Every entry can carry a `source` annotation. Visibility derives from it:
 `team-*`, `gitlab`, and `spec-manage` entries may be quoted in team-facing
-artifacts; every other source (`people-journal`, `stopit`,
+artifacts; every other source (`people-journal`, `handoff`,
 `mattermost-triage`, `task-*`, `docs-*`, `user`) is personal-only. The label
 is exposed in search responses.
 

package/README.ru.md

@@ -4,8 +4,9 @@
 
 `@kisev/memomatic` — персональная обучающая память для агентов по архитектуре
 памяти OpenClaw: ярусный Markdown-корпус, который можно читать как обычные файлы,
-пересобираемый SQLite-индекс с FTS5 и опциональными локальными эмбеддингами,
-проход извлечения `sessions` и ночной проход консолидации `dream`.
+пересобираемый SQLite-индекс с FTS5, опциональными локальными эмбеддингами и
+опциональным реранкером, проход извлечения `sessions` и ночной проход
+консолидации `dream`.
 
 ## Установка
 
@@ -78,7 +79,7 @@ inbox, бэклог сессий, кандидатов на продвижени
 
 Каждая запись может нести аннотацию `source`. Из неё выводится видимость:
 записи `team-*`, `gitlab` и `spec-manage` можно цитировать в командных
-артефактах; остальные источники (`people-journal`, `stopit`,
+артефактах; остальные источники (`people-journal`, `handoff`,
 `mattermost-triage`, `task-*`, `docs-*`, `user`) — personal-only. Метка
 возвращается в поиске.
 

package/dist/search.d.ts

@@ -14,7 +14,11 @@ export type SearchHit = {
         exact: boolean;
         recency: number;
         importance: number;
-        reason: "lexical" | "semantic" | "both";
+        reason: "lexical" | "semantic" | "both" | "rerank";
+        rerank?: {
+            candidates: number;
+            threshold: number;
+        };
     };
 };
 export declare function corpusFiles(paths: MemomaticPaths): Promise<string[]>;

package/dist/search.js

@@ -3,7 +3,7 @@ import { join } from "node:path";
 import { cosineSimilarity, stableIdFor } from "./store.js";
 import { entryKind, parseCorpusEntries } from "./corpus.js";
 import { entryImportance, entryObservedAt } from "./entries.js";
-import { embedTexts, embeddingFingerprint } from "./settings.js";
+import { embedTexts, embeddingFingerprint, rerankTexts } from "./settings.js";
 import { check } from "./operations.js";
 export async function corpusFiles(paths) {
     const files = [];
@@ -73,9 +73,10 @@ export async function reindex(paths, settings, store, options = {}) {
         completed: 0,
         cached: entries.length - missing.length,
     });
-    for (let start = 0; start < missing.length; start += 32) {
+    const batchSize = Math.min(32, settings.embedding?.maxBatchTexts ?? 32);
+    for (let start = 0; start < missing.length; start += batchSize) {
         check(options);
-        const batch = missing.slice(start, start + 32);
+        const batch = missing.slice(start, start + batchSize);
         const computed = await embedTexts(settings, batch.map((item) => item.entry.text), "document", options.signal);
         if (vectors && computed)
             batch.forEach((item, index) => {
@@ -85,7 +86,7 @@ export async function reindex(paths, settings, store, options = {}) {
             phase: "index.embeddings",
             message: "Embedding batch complete",
             total: missing.length,
-            completed: Math.min(start + 32, missing.length),
+            completed: Math.min(start + batchSize, missing.length),
         });
     }
     check(options);
@@ -117,40 +118,21 @@ export function queryTokens(query) {
             .match(/[\p{L}\p{N}]+/gu) ?? []),
     ].filter((token) => token.length >= 2);
 }
-export async function search(store, query, settings, options = {}) {
-    if (!query.trim())
-        throw new Error("query is required");
-    const tokens = queryTokens(query);
-    const keyword = store.ftsSearch(query, 200);
-    const vector = new Map();
-    const storedVectors = store.vectors();
-    if (settings.embedding &&
-        store.allEntries().length &&
-        store.getMeta("embeddingFingerprint") !== embeddingFingerprint(settings))
-        throw new Error("embedding index is missing or belongs to another model; run memomatic index");
-    const queryVector = (await embedTexts(settings, [query], "query"))?.[0];
-    if (queryVector?.length) {
-        for (const row of storedVectors) {
-            if (row.data.length !== queryVector.length)
-                throw new Error("embedding dimensions changed; run memomatic index");
-            const similarity = cosineSimilarity(queryVector, row.data);
-            if (similarity > 0.05)
-                vector.set(row.stableId, similarity);
-        }
-    }
-    const candidates = new Set([...keyword.keys(), ...vector.keys()]);
-    const entries = store
-        .allEntries()
-        .filter((item) => options.project === undefined || item.project === options.project || item.kind === "user");
-    const byId = new Map(entries.map((entry) => [entry.stableId, entry]));
+function lexicalCoverage(entry, tokens, keyword) {
+    const words = queryTokens(entry.text);
+    const matchedTokens = tokens.filter((token) => words.some((word) => word.startsWith(token)));
+    return {
+        matchedTokens,
+        lexical: keyword.has(entry.stableId) && tokens.length ? matchedTokens.length / tokens.length : 0,
+    };
+}
+function gatedHits(tokens, keyword, vector, byId, settings) {
     const hits = [];
-    for (const stableId of candidates) {
+    for (const stableId of new Set([...keyword.keys(), ...vector.keys()])) {
         const entry = byId.get(stableId);
         if (!entry)
             continue;
-        const words = queryTokens(entry.text);
-        const matchedTokens = tokens.filter((token) => words.some((word) => word.startsWith(token)));
-        const lexical = keyword.has(stableId) && tokens.length ? matchedTokens.length / tokens.length : 0;
+        const { matchedTokens, lexical } = lexicalCoverage(entry, tokens, keyword);
         const semantic = vector.get(stableId) ?? null;
         const lexicalAccepted = lexical >= settings.search.minScore && lexical >= 0.5;
         const semanticAccepted = semantic !== null &&
@@ -164,6 +146,7 @@ export async function search(store, query, settings, options = {}) {
         const score = relevance * recency * (0.85 + 0.15 * importance);
         if (score < settings.search.minScore)
             continue;
+        const words = queryTokens(entry.text);
         hits.push({
             entry,
             score,
@@ -180,10 +163,104 @@ export async function search(store, query, settings, options = {}) {
             },
         });
     }
-    hits.sort((left, right) => Number(right.explanation.exact) - Number(left.explanation.exact) ||
+    return hits.sort((left, right) => Number(right.explanation.exact) - Number(left.explanation.exact) ||
         right.score - left.score ||
         (right.explanation.semantic ?? 0) - (left.explanation.semantic ?? 0) ||
         left.entry.stableId.localeCompare(right.entry.stableId));
+}
+/**
+ * Reranker path: collect a bounded wide candidate set from lexical and vector
+ * search before the strict relevance gates, apply the project filter, and let
+ * the reranker decide final relevance and order without giving literal
+ * matches an unconditional priority.
+ */
+async function rerankedHits(query, tokens, keyword, vector, byId, settings) {
+    const reranker = settings.reranker;
+    // The project filter narrows the pool before top-N selection, so hidden
+    // entries cannot consume the candidate budget.
+    const visible = (scores) => {
+        const filtered = new Map();
+        for (const [stableId, score] of scores)
+            if (byId.has(stableId))
+                filtered.set(stableId, score);
+        return filtered;
+    };
+    const widest = (scores) => [...scores.entries()]
+        .sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))
+        .slice(0, reranker.candidates)
+        .flatMap(([stableId]) => {
+        const entry = byId.get(stableId);
+        return entry ? [entry] : [];
+    });
+    const candidates = [];
+    const seen = new Set();
+    for (const entry of [...widest(visible(keyword)), ...widest(visible(vector))]) {
+        if (seen.has(entry.stableId))
+            continue;
+        seen.add(entry.stableId);
+        candidates.push(entry);
+    }
+    if (!candidates.length)
+        return [];
+    // Reranker documents stay raw entry text without embedding prefixes.
+    const ranked = await rerankTexts(settings, query, candidates.map((entry) => entry.text));
+    const hits = [];
+    for (const { index, score } of ranked) {
+        if (score < reranker.minScore)
+            continue;
+        const entry = candidates[index];
+        const { matchedTokens, lexical } = lexicalCoverage(entry, tokens, keyword);
+        const semantic = vector.get(entry.stableId) ?? null;
+        const recency = recencyMultiplier(entry, settings.search.halfLifeDays);
+        const importance = Math.max(0, Math.min(1, (entry.importance - 1) / 9));
+        const words = queryTokens(entry.text);
+        hits.push({
+            entry,
+            score,
+            snippet: entry.text.slice(0, 240),
+            explanation: {
+                matchedTokens,
+                lexical,
+                semantic,
+                relevance: score,
+                recency,
+                importance,
+                exact: tokens.length > 0 && tokens.every((token) => words.includes(token)),
+                reason: "rerank",
+                rerank: { candidates: candidates.length, threshold: reranker.minScore },
+            },
+        });
+    }
+    return hits;
+}
+export async function search(store, query, settings, options = {}) {
+    if (!query.trim())
+        throw new Error("query is required");
+    const tokens = queryTokens(query);
+    const keyword = store.ftsSearch(query, settings.reranker ? Math.max(200, settings.reranker.candidates) : 200);
+    const vector = new Map();
+    const storedVectors = store.vectors();
+    if (settings.embedding &&
+        store.allEntries().length &&
+        store.getMeta("embeddingFingerprint") !== embeddingFingerprint(settings))
+        throw new Error("embedding index is missing or belongs to another model; run memomatic index");
+    const queryVector = (await embedTexts(settings, [query], "query"))?.[0];
+    if (queryVector?.length) {
+        for (const row of storedVectors) {
+            if (row.data.length !== queryVector.length)
+                throw new Error("embedding dimensions changed; run memomatic index");
+            const similarity = cosineSimilarity(queryVector, row.data);
+            if (similarity > 0.05)
+                vector.set(row.stableId, similarity);
+        }
+    }
+    const entries = store
+        .allEntries()
+        .filter((item) => options.project === undefined || item.project === options.project || item.kind === "user");
+    const byId = new Map(entries.map((entry) => [entry.stableId, entry]));
+    const hits = settings.reranker
+        ? await rerankedHits(query, tokens, keyword, vector, byId, settings)
+        : gatedHits(tokens, keyword, vector, byId, settings);
     const limited = hits.slice(0, settings.search.maxResults);
     if (options.markSurfaced !== false && limited.length)
         store.markSurfaced(limited.map((hit) => hit.entry.stableId), query);

package/dist/service.d.ts

@@ -45,7 +45,11 @@ export declare function searchMemoryResults(context: MemomaticContext, query: st
         exact: boolean;
         recency: number;
         importance: number;
-        reason: "lexical" | "semantic" | "both";
+        reason: "lexical" | "semantic" | "both" | "rerank";
+        rerank?: {
+            candidates: number;
+            threshold: number;
+        };
     } | undefined;
     source: string | null;
     visibility: import("./visibility.js").EntryVisibility;

package/dist/settings.d.ts

@@ -1,9 +1,28 @@
+export type EmbeddingSettings = {
+    url: string;
+    model: string;
+    /** Explicit query instruction; null keeps the built-in Qwen3 auto-detection. */
+    queryInstruction: string | null;
+    documentPrefix: string;
+    queryPrefix: string;
+    bearerEnv: string | null;
+    timeoutMs: number;
+    maxBatchTexts: number;
+    maxTextChars: number;
+};
+export type RerankerSettings = {
+    url: string;
+    model: string;
+    bearerEnv: string | null;
+    timeoutMs: number;
+    candidates: number;
+    minScore: number;
+    maxDocuments: number;
+    maxChars: number;
+};
 export type MemomaticSettings = {
-    embedding: {
-        url: string;
-        model: string;
-        queryInstruction?: string;
-    } | null;
+    embedding: EmbeddingSettings | null;
+    reranker: RerankerSettings | null;
     sessions: {
         model: string | null;
         variant: string | null;
@@ -39,10 +58,43 @@ export type MemomaticSettings = {
         days: number;
     };
 };
+export declare const DEFAULT_MAX_TEXT_CHARS = 8000;
+export declare const RERANKER_TEMPLATE_ALLOWANCE_CHARS = 1024;
+export declare const defaultEmbedding: (url: string, model: string) => EmbeddingSettings;
+export declare const defaultReranker: (url: string, model: string) => RerankerSettings;
 export declare const defaultSettings: () => MemomaticSettings;
 export declare function normalizeSettings(raw: unknown): MemomaticSettings;
 export declare function loadSettings(settingsFile: string): Promise<MemomaticSettings>;
-export declare function embedTexts(settings: MemomaticSettings, texts: string[], purpose?: "document" | "query", signal?: AbortSignal): Promise<Float32Array[] | null>;
+/** Splits long text at paragraph, line or space boundaries within the budget. */
+export declare function chunkText(text: string, maxChars: number): string[];
 export declare const MEMORY_QUERY_INSTRUCTION = "Given a search query, retrieve relevant personal memory entries that answer the query.";
-export declare function embeddingQuery(settings: MemomaticSettings, query: string): string;
+export declare function embeddingQueryText(embedding: EmbeddingSettings, query: string): string;
+/** Headroom a formatted query chunk needs beyond its raw text. */
+export declare function embeddingQueryOverhead(embedding: EmbeddingSettings): number;
+export declare function embeddingDocumentText(embedding: EmbeddingSettings, text: string): string;
+/** Headroom a formatted document chunk needs beyond its raw text. */
+export declare function embeddingDocumentOverhead(embedding: EmbeddingSettings): number;
+/**
+ * Embeds texts through an OpenAI-compatible `embeddings` endpoint. Raw inputs
+ * are split with chunkText first and every chunk is formatted afterwards, so
+ * each document chunk carries the document prefix and each query chunk the
+ * query prefix plus the Qwen instruction, all inside maxTextChars. Chunks are
+ * embedded chunk-wise and mean-pooled back onto one vector per input text.
+ * Request character budgets are a conservative proxy, not exact tokenization.
+ */
+export declare function embedTexts(settings: MemomaticSettings, texts: string[], purpose?: "document" | "query", signal?: AbortSignal): Promise<Float32Array[] | null>;
+export type RerankHit = {
+    index: number;
+    score: number;
+};
+/**
+ * Ranks documents against a query through a `/rerank`-style endpoint
+ * (`{model, query, documents, top_n}` request, `results[].index` and
+ * `results[].relevance_score` response). Documents are chunked so the query,
+ * one document chunk and the server template stay inside maxChars; a query
+ * too large to leave room for a chunk fails before the request. Responses
+ * must contain exactly top_n results, and chunk scores aggregate to one best
+ * score per input document.
+ */
+export declare function rerankTexts(settings: MemomaticSettings, query: string, documents: string[], signal?: AbortSignal): Promise<RerankHit[]>;
 export declare function embeddingFingerprint(settings: MemomaticSettings): string;

package/dist/settings.js

@@ -1,7 +1,36 @@
 import { readFile } from "node:fs/promises";
 import { createHash } from "node:crypto";
+// Conservative character proxy for an input budget of roughly 4096 tokens:
+// dense scripts need about two characters per token, so 8000 characters stay
+// inside the example budget without claiming exact tokenization. Long texts
+// are split at chunkText instead of measured with a tokenizer.
+export const DEFAULT_MAX_TEXT_CHARS = 8000;
+// Headroom for the server-side rerank template wrapped around query and document.
+export const RERANKER_TEMPLATE_ALLOWANCE_CHARS = 1024;
+export const defaultEmbedding = (url, model) => ({
+    url,
+    model,
+    queryInstruction: null,
+    documentPrefix: "",
+    queryPrefix: "",
+    bearerEnv: null,
+    timeoutMs: 30_000,
+    maxBatchTexts: 32,
+    maxTextChars: DEFAULT_MAX_TEXT_CHARS,
+});
+export const defaultReranker = (url, model) => ({
+    url,
+    model,
+    bearerEnv: null,
+    timeoutMs: 30_000,
+    candidates: 64,
+    minScore: 0,
+    maxDocuments: 64,
+    maxChars: DEFAULT_MAX_TEXT_CHARS,
+});
 export const defaultSettings = () => ({
     embedding: null,
+    reranker: null,
     sessions: {
         model: null,
         variant: null,
@@ -50,10 +79,56 @@ const SESSIONS_KEYS = [
     "idleMs",
     "opencodeUrl",
 ];
+function httpUrl(value, section) {
+    if (typeof value !== "string")
+        throw new Error(`${section}.url must be a string`);
+    let parsed;
+    try {
+        parsed = new URL(value);
+    }
+    catch {
+        throw new Error(`${section}.url must be an absolute http(s) URL`);
+    }
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
+        throw new Error(`${section}.url must use http or https`);
+    return value;
+}
+function bearerEnvName(value, section) {
+    if (value === null || value === undefined)
+        return null;
+    if (typeof value !== "string" || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(value))
+        throw new Error(`${section}.bearerEnv must name an environment variable`);
+    return value;
+}
+function boundedInt(value, section, key, min, max) {
+    if (!Number.isSafeInteger(value) || value < min || value > max)
+        throw new Error(`invalid ${section}.${key}; expected an integer between ${min} and ${max}`);
+    return value;
+}
+function optionalText(value, section, key) {
+    if (typeof value !== "string")
+        throw new Error(`${section}.${key} must be a string`);
+    return value;
+}
+function serviceEndpoint(raw, section, defaults, validate) {
+    if (raw === null || raw === undefined)
+        return null;
+    if (typeof raw !== "object")
+        throw new Error(`invalid ${section} settings`);
+    const url = httpUrl(raw.url, section);
+    if (typeof raw.model !== "string" || !raw.model.trim())
+        throw new Error(`${section}.model must be a non-empty string`);
+    const merged = defaults(url, raw.model);
+    for (const [key, value] of Object.entries(raw)) {
+        if (key in merged && value !== null && value !== undefined)
+            merged[key] = value;
+    }
+    validate(merged);
+    return merged;
+}
 export function normalizeSettings(raw) {
     const defaults = defaultSettings();
     const source = (raw ?? {});
-    const embedding = source.embedding;
     const search = mergeSection(defaults.search, source.search);
     const dream = mergeSection(defaults.dream, source.dream);
     // Legacy single-model configs put extraction knobs under `dream`; adopt them
@@ -106,16 +181,32 @@ export function normalizeSettings(raw) {
         search.halfLifeDays <= 0 ||
         ![search.minScore, search.minSemanticScore].every((value) => Number.isFinite(value) && value >= 0 && value <= 1))
         throw new Error("invalid search settings");
+    const embedding = serviceEndpoint((source.embedding ?? null), "embedding", (url, model) => defaultEmbedding(url, model), (merged) => {
+        if (merged.queryInstruction !== null && typeof merged.queryInstruction !== "string")
+            throw new Error("embedding.queryInstruction must be a string or null");
+        merged.queryInstruction =
+            typeof merged.queryInstruction === "string" ? merged.queryInstruction : null;
+        merged.documentPrefix = optionalText(merged.documentPrefix, "embedding", "documentPrefix");
+        merged.queryPrefix = optionalText(merged.queryPrefix, "embedding", "queryPrefix");
+        merged.bearerEnv = bearerEnvName(merged.bearerEnv, "embedding");
+        merged.timeoutMs = boundedInt(merged.timeoutMs, "embedding", "timeoutMs", 1, Number.MAX_SAFE_INTEGER);
+        merged.maxBatchTexts = boundedInt(merged.maxBatchTexts, "embedding", "maxBatchTexts", 1, 512);
+        merged.maxTextChars = boundedInt(merged.maxTextChars, "embedding", "maxTextChars", 256, 1_000_000);
+    });
+    const reranker = serviceEndpoint((source.reranker ?? null), "reranker", (url, model) => defaultReranker(url, model), (merged) => {
+        merged.bearerEnv = bearerEnvName(merged.bearerEnv, "reranker");
+        merged.timeoutMs = boundedInt(merged.timeoutMs, "reranker", "timeoutMs", 1, Number.MAX_SAFE_INTEGER);
+        merged.candidates = boundedInt(merged.candidates, "reranker", "candidates", 1, 1000);
+        // Scores are ranking signals whose scale depends on the model, so the
+        // threshold is any finite non-negative number, not a probability bound.
+        if (!Number.isFinite(merged.minScore) || merged.minScore < 0)
+            throw new Error("invalid reranker.minScore; expected a finite number of at least 0");
+        merged.maxDocuments = boundedInt(merged.maxDocuments, "reranker", "maxDocuments", 1, 512);
+        merged.maxChars = boundedInt(merged.maxChars, "reranker", "maxChars", 256, 1_000_000);
+    });
     return {
-        embedding: embedding && typeof embedding.url === "string" && typeof embedding.model === "string"
-            ? {
-                url: embedding.url,
-                model: embedding.model,
-                ...(typeof embedding.queryInstruction === "string"
-                    ? { queryInstruction: embedding.queryInstruction }
-                    : {}),
-            }
-            : null,
+        embedding,
+        reranker,
         sessions,
         dream,
         search,
@@ -132,25 +223,92 @@ export async function loadSettings(settingsFile) {
         throw error;
     }
 }
-export async function embedTexts(settings, texts, purpose = "document", signal) {
-    if (!settings.embedding || !texts.length)
-        return null;
-    const response = await fetch(settings.embedding.url, {
-        body: JSON.stringify({
-            input: texts.map((text) => (purpose === "query" ? embeddingQuery(settings, text) : text)),
-            model: settings.embedding.model,
-        }),
-        headers: { "content-type": "application/json" },
+/** Splits long text at paragraph, line or space boundaries within the budget. */
+export function chunkText(text, maxChars) {
+    if (text.length <= maxChars)
+        return [text];
+    const chunks = [];
+    let start = 0;
+    while (start < text.length) {
+        let end = Math.min(start + maxChars, text.length);
+        if (end < text.length) {
+            const window = text.slice(start, end);
+            const boundary = Math.max(window.lastIndexOf("\n\n"), window.lastIndexOf("\n"), window.lastIndexOf(" "));
+            if (boundary >= Math.floor(maxChars / 2))
+                end = start + boundary + 1;
+        }
+        const chunk = text.slice(start, end).trim();
+        if (chunk)
+            chunks.push(chunk);
+        start = end;
+    }
+    return chunks.length ? chunks : [text.slice(0, maxChars)];
+}
+function bearerToken(bearerEnv, service) {
+    if (!bearerEnv)
+        return undefined;
+    const token = process.env[bearerEnv];
+    if (!token)
+        throw new Error(`${service} bearerEnv "${bearerEnv}" is not set`);
+    // A control character makes the Authorization header value invalid; the
+    // fetch-level rejection echoes the token back, so reject it here where the
+    // diagnostic stays free of the secret.
+    if (/[\u0000-\u001f\u007f]/.test(token))
+        throw new Error(`${service} bearerEnv "${bearerEnv}" contains a control character; expected a single-line token`);
+    return token;
+}
+async function postJson(url, body, token, timeoutMs, signal, service) {
+    const response = await fetch(url, {
+        body: JSON.stringify(body),
+        headers: {
+            "content-type": "application/json",
+            ...(token ? { authorization: `Bearer ${token}` } : {}),
+        },
         method: "POST",
         signal: signal
-            ? AbortSignal.any([signal, AbortSignal.timeout(30_000)])
-            : AbortSignal.timeout(30_000),
+            ? AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)])
+            : AbortSignal.timeout(timeoutMs),
+    }).catch((error) => {
+        const failure = error;
+        if (failure.name === "TimeoutError")
+            throw new Error(`${service} request timed out after ${timeoutMs}ms`);
+        if (failure.name === "AbortError")
+            throw new Error(`${service} request was aborted`);
+        throw new Error(`${service} request failed: ${failure.message}`);
     });
     if (!response.ok)
-        throw new Error(`embedding request failed: ${response.status} ${await response.text()}`);
-    const payload = (await response.json());
-    const list = payload.data ?? [];
-    if (list.length !== texts.length)
+        throw new Error(`${service} request failed: ${response.status} ${(await response.text()).slice(0, 2000)}`);
+    try {
+        return await response.json();
+    }
+    catch {
+        throw new Error(`${service} response is not valid JSON`);
+    }
+}
+export const MEMORY_QUERY_INSTRUCTION = "Given a search query, retrieve relevant personal memory entries that answer the query.";
+function queryInstructionFor(embedding) {
+    return (embedding.queryInstruction ??
+        (/qwen3[-_]?embedding/i.test(embedding.model) ? MEMORY_QUERY_INSTRUCTION : ""));
+}
+export function embeddingQueryText(embedding, query) {
+    const instruction = queryInstructionFor(embedding);
+    return `${embedding.queryPrefix}${instruction ? `Instruct: ${instruction}\nQuery:${query}` : query}`;
+}
+/** Headroom a formatted query chunk needs beyond its raw text. */
+export function embeddingQueryOverhead(embedding) {
+    const instruction = queryInstructionFor(embedding);
+    return (embedding.queryPrefix.length + (instruction ? `Instruct: ${instruction}\nQuery:`.length : 0));
+}
+export function embeddingDocumentText(embedding, text) {
+    return `${embedding.documentPrefix}${text}`;
+}
+/** Headroom a formatted document chunk needs beyond its raw text. */
+export function embeddingDocumentOverhead(embedding) {
+    return embedding.documentPrefix.length;
+}
+function parseEmbeddingPayload(payload, count) {
+    const list = (payload.data ?? []);
+    if (!Array.isArray(list) || list.length !== count)
         throw new Error("embedding response is incomplete");
     const byIndex = new Map();
     for (const item of list) {
@@ -160,21 +318,145 @@ export async function embedTexts(settings, texts, purpose = "document", signal)
             !item.embedding.some((value) => value !== 0))
             throw new Error("embedding response is malformed");
         const index = item.index ?? byIndex.size;
-        if (!Number.isInteger(index) || index < 0 || index >= texts.length || byIndex.has(index))
+        if (!Number.isInteger(index) || index < 0 || index >= count || byIndex.has(index))
             throw new Error("embedding response has invalid indices");
         byIndex.set(index, item.embedding);
     }
-    const vectors = texts.map((_, index) => Float32Array.from(byIndex.get(index)));
+    const vectors = Array.from({ length: count }, (_, index) => Float32Array.from(byIndex.get(index)));
     if (!vectors.every((vector) => vector.length === vectors[0].length && vector.every(Number.isFinite)))
         throw new Error("embedding response has incompatible vectors");
     return vectors;
 }
-export const MEMORY_QUERY_INSTRUCTION = "Given a search query, retrieve relevant personal memory entries that answer the query.";
-export function embeddingQuery(settings, query) {
+function meanPool(vectors) {
+    if (vectors.length === 1)
+        return vectors[0];
+    const pooled = new Float32Array(vectors[0].length);
+    for (const vector of vectors) {
+        if (vector.length !== pooled.length)
+            throw new Error("embedding response has incompatible vectors");
+        for (let index = 0; index < vector.length; index += 1)
+            pooled[index] += vector[index];
+    }
+    for (let index = 0; index < pooled.length; index += 1)
+        pooled[index] /= vectors.length;
+    return pooled;
+}
+/**
+ * Embeds texts through an OpenAI-compatible `embeddings` endpoint. Raw inputs
+ * are split with chunkText first and every chunk is formatted afterwards, so
+ * each document chunk carries the document prefix and each query chunk the
+ * query prefix plus the Qwen instruction, all inside maxTextChars. Chunks are
+ * embedded chunk-wise and mean-pooled back onto one vector per input text.
+ * Request character budgets are a conservative proxy, not exact tokenization.
+ */
+export async function embedTexts(settings, texts, purpose = "document", signal) {
+    if (!settings.embedding || !texts.length)
+        return null;
     const embedding = settings.embedding;
-    const instruction = embedding?.queryInstruction ??
-        (/qwen3[-_]?embedding/i.test(embedding?.model ?? "") ? MEMORY_QUERY_INSTRUCTION : "");
-    return instruction ? `Instruct: ${instruction}\nQuery:${query}` : query;
+    const token = bearerToken(embedding.bearerEnv, "embedding");
+    const format = purpose === "query"
+        ? (text) => embeddingQueryText(embedding, text)
+        : (text) => embeddingDocumentText(embedding, text);
+    const overhead = purpose === "query" ? embeddingQueryOverhead(embedding) : embeddingDocumentOverhead(embedding);
+    const chunkBudget = embedding.maxTextChars - overhead;
+    if (chunkBudget < 1)
+        throw new Error(`embedding ${purpose} formatting needs ${overhead} characters of headroom and leaves no room inside maxTextChars=${embedding.maxTextChars}`);
+    // Split the raw text with formatting headroom reserved, then format each
+    // chunk so prefixes and instructions cover every piece, not just the first.
+    const chunkLists = texts.map((text) => chunkText(text, chunkBudget).map(format));
+    const chunks = chunkLists.flat();
+    const vectors = new Array(chunks.length);
+    for (let start = 0; start < chunks.length; start += embedding.maxBatchTexts) {
+        const batch = chunks.slice(start, start + embedding.maxBatchTexts);
+        const payload = await postJson(embedding.url, { input: batch, model: embedding.model, encoding_format: "float" }, token, embedding.timeoutMs, signal, "embedding");
+        parseEmbeddingPayload(payload, batch.length).forEach((vector, index) => (vectors[start + index] = vector));
+    }
+    const pooled = [];
+    let offset = 0;
+    for (const list of chunkLists) {
+        pooled.push(meanPool(vectors.slice(offset, offset + list.length)));
+        offset += list.length;
+    }
+    return pooled;
+}
+function parseRerankResults(payload, count, offset) {
+    const results = payload.results;
+    if (!Array.isArray(results))
+        throw new Error("reranker response is missing results");
+    // top_n equals the number of sent documents, so a complete answer ranks
+    // every candidate; an empty or shortened response is a failure, not an
+    // empty success.
+    if (results.length !== count)
+        throw new Error("reranker response is incomplete");
+    const seen = new Set();
+    const hits = [];
+    for (const item of results) {
+        const record = item;
+        const index = record?.index;
+        const score = record?.relevance_score;
+        if (!Number.isInteger(index) || index < 0 || index >= count)
+            throw new Error("reranker response has invalid indices");
+        if (typeof score !== "number" || !Number.isFinite(score))
+            throw new Error("reranker response is malformed");
+        if (seen.has(index))
+            throw new Error("reranker response has invalid indices");
+        seen.add(index);
+        hits.push({ index: offset + index, score });
+    }
+    return hits;
+}
+/**
+ * Ranks documents against a query through a `/rerank`-style endpoint
+ * (`{model, query, documents, top_n}` request, `results[].index` and
+ * `results[].relevance_score` response). Documents are chunked so the query,
+ * one document chunk and the server template stay inside maxChars; a query
+ * too large to leave room for a chunk fails before the request. Responses
+ * must contain exactly top_n results, and chunk scores aggregate to one best
+ * score per input document.
+ */
+export async function rerankTexts(settings, query, documents, signal) {
+    const reranker = settings.reranker;
+    if (!reranker || !documents.length)
+        return [];
+    const token = bearerToken(reranker.bearerEnv, "reranker");
+    // Query, one document chunk and the template allowance share maxChars. A
+    // small remainder is honored as-is; inflating it would push the request
+    // past the model input budget.
+    const remaining = reranker.maxChars - RERANKER_TEMPLATE_ALLOWANCE_CHARS - query.length;
+    if (remaining < 1)
+        throw new Error(`reranker query of ${query.length} characters leaves no document room inside maxChars=${reranker.maxChars}`);
+    const chunkLists = documents.map((document) => chunkText(document, remaining));
+    const offsets = [];
+    let total = 0;
+    for (const list of chunkLists) {
+        offsets.push(total);
+        total += list.length;
+    }
+    const scores = new Map();
+    const chunks = chunkLists.flat();
+    for (let start = 0; start < total; start += reranker.maxDocuments) {
+        const batch = chunks.slice(start, start + reranker.maxDocuments);
+        const payload = await postJson(reranker.url, {
+            model: reranker.model,
+            query,
+            documents: batch,
+            top_n: batch.length,
+        }, token, reranker.timeoutMs, signal, "reranker");
+        for (const { index, score } of parseRerankResults(payload, batch.length, start))
+            scores.set(index, Math.max(scores.get(index) ?? Number.NEGATIVE_INFINITY, score));
+    }
+    const hits = [];
+    chunkLists.forEach((list, index) => {
+        let best = null;
+        for (let offset = 0; offset < list.length; offset += 1) {
+            const score = scores.get(offsets[index] + offset);
+            if (score !== undefined && (best === null || score > best))
+                best = score;
+        }
+        if (best !== null)
+            hits.push({ index, score: best });
+    });
+    return hits.sort((left, right) => right.score - left.score || left.index - right.index);
 }
 export function embeddingFingerprint(settings) {
     return createHash("sha256")
@@ -182,7 +464,15 @@ export function embeddingFingerprint(settings) {
         ? {
             url: settings.embedding.url,
             model: settings.embedding.model,
-            documentFormat: "raw-v1",
+            document: {
+                prefix: settings.embedding.documentPrefix,
+                maxChars: settings.embedding.maxTextChars,
+                pooling: "mean-v1",
+                // Chunking now happens before formatting, so every chunk
+                // carries the prefix/instruction; bumping this version
+                // invalidates indexes built by the previous algorithm.
+                formatting: "per-chunk-v2",
+            },
         }
         : null))
         .digest("hex");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kisev/memomatic",
-  "version": "1.0.0-dev.99.g48fbb8555698",
+  "version": "12.0.1-dev.115.g8010bdfeea15",
   "description": "Personal learning memory for agents: tiered Markdown corpus, rebuildable index, dream sweep.",
   "homepage": "https://github.com/kisev/skills#readme",
   "bugs": {
@@ -34,7 +34,7 @@
     "pack:check": "node test/pack-allowlist.mjs"
   },
   "dependencies": {
-    "@kisev/safe-fs": "1.0.0-dev.99.g48fbb8555698",
+    "@kisev/safe-fs": "12.0.1-dev.115.g8010bdfeea15",
     "commander": "15.0.0",
     "jsonc-parser": "3.3.1"
   },