memwarden 0.0.1

package/dist/functions/search.js

@@ -0,0 +1,523 @@
+//
+// Search (mem::search): hybrid BM25 + vector (RRF) retrieval with a lazy index
+// rebuild, project/cwd over-fetch + canonical-path post-filter, a memory-scope
+// fallback, an optional Verified Recall firewall (safe_only), and three output
+// formats (full / compact / narrative) with token-budget packing. When an
+// embedding provider is active (the default: on-device MiniLM + TurboQuant)
+// the vector stream is fused in; with no provider it runs BM25-only.
+import { KV } from "../state/schema.js";
+import { SearchIndex } from "./search-index.js";
+import { VectorIndex } from "./vector-index.js";
+import { QuantizedVectorIndex } from "./quantized-vector-index.js";
+import { isQuantizedVectorEnabled, getQuantBits, getQuantRescoreDepth, getQuantSeed, } from "./config.js";
+import { memoryToObservation } from "./memory-utils.js";
+import { canonicalizePath } from "./paths.js";
+import { classifyProvenance } from "./verify.js";
+import { recordAccessBatch } from "./access-tracker.js";
+import { loadVectorIndex, persistVectorIndex } from "./vector-persistence.js";
+import { logger } from "./logger.js";
+import { metrics } from "../observability/metrics.js";
+let index = null;
+let vectorIndex = null;
+let currentEmbeddingProvider = null;
+export function getSearchIndex() {
+    if (!index)
+        index = new SearchIndex();
+    return index;
+}
+export function setVectorIndex(idx) {
+    vectorIndex = idx;
+}
+export function getVectorIndex() {
+    return vectorIndex;
+}
+/**
+ * Constructs the configured vector index: TurboQuant-backed when
+ * MEMWARDEN_QUANT_VECTOR=true, the full-precision VectorIndex otherwise.
+ * `dims` comes from the embedding provider that will feed the index.
+ */
+export function makeVectorIndex(dims) {
+    if (isQuantizedVectorEnabled()) {
+        return new QuantizedVectorIndex({
+            dims,
+            bits: getQuantBits(),
+            seed: getQuantSeed(),
+            rescoreDepth: getQuantRescoreDepth(),
+        });
+    }
+    return new VectorIndex();
+}
+export function setEmbeddingProvider(provider) {
+    currentEmbeddingProvider = provider;
+}
+export function getEmbeddingProvider() {
+    return currentEmbeddingProvider;
+}
+export function vectorIndexRemove(id) {
+    vectorIndex?.remove(id);
+}
+// Hard cap on embedding input length. Truncate defensively so a huge
+// content blob can't 400 the embed call or blow context budget on a single
+// doc. 16k chars ≈ 4k tokens, safely under every provider.
+const EMBED_MAX_CHARS = 16_000;
+export function clipEmbedInput(text) {
+    if (text.length <= EMBED_MAX_CHARS)
+        return text;
+    return text.slice(0, EMBED_MAX_CHARS);
+}
+// Single guarded vector-index write. Returns true on success. Soft-fails
+// (logs + no-op) on dimension mismatch or embed error so a downed embedder
+// never breaks the upstream save. With no provider configured this returns
+// false immediately; observe.ts treats false as "vector skipped", not an error.
+export async function vectorIndexAddGuarded(id, sessionId, text, context) {
+    const vi = vectorIndex;
+    const ep = currentEmbeddingProvider;
+    if (!vi || !ep)
+        return false;
+    try {
+        const embedding = await ep.embed(clipEmbedInput(text));
+        if (embedding.length !== ep.dimensions) {
+            logger.warn("vector-index add: dimension mismatch — skipping", {
+                kind: context.kind,
+                id: context.logId,
+                provider: ep.name,
+                expected: ep.dimensions,
+                received: embedding.length,
+            });
+            return false;
+        }
+        vi.add(id, sessionId, embedding);
+        return true;
+    }
+    catch (err) {
+        logger.warn("vector-index add: embed failed — skipping", {
+            kind: context.kind,
+            id: context.logId,
+            provider: ep.name,
+            error: err instanceof Error ? err.message : String(err),
+        });
+        return false;
+    }
+}
+// Rebuilds the BM25 index from KV. Walks the memories scope (so
+// mem::remember entries survive a restart) and every session's
+// observations. The vector index is cleared in lockstep so BM25 and vector
+// stay in sync; with no provider it stays empty. When a persisted
+// quantized index was just restored (vector-persistence.ts), pass
+// `preserveVectorIndex` to switch the vector side to INCREMENTAL SYNC:
+// restored codes are kept, only docs missing from the index are embedded,
+// and ghosts (ids in the blob that no longer exist in KV) are evicted at
+// the end of the walk.
+export async function rebuildIndex(kv, opts) {
+    const preserveVectors = opts?.preserveVectorIndex === true;
+    const idx = getSearchIndex();
+    idx.clear();
+    if (!preserveVectors)
+        vectorIndex?.clear();
+    // Ids seen in KV during this walk; used to evict ghosts from a restored
+    // vector index. Only tracked in preserve mode.
+    const liveIds = preserveVectors ? new Set() : null;
+    let count = 0;
+    // Memories live in their own KV scope outside per-session observation
+    // scopes, so they need a separate walk.
+    try {
+        const memories = await kv.list(KV.memories);
+        for (const memory of memories) {
+            if (memory.isLatest === false)
+                continue;
+            if (!memory.title || !memory.content)
+                continue;
+            idx.add(memoryToObservation(memory));
+            liveIds?.add(memory.id);
+            if (!preserveVectors || !vectorIndex?.has(memory.id)) {
+                await vectorIndexAddGuarded(memory.id, memory.sessionIds?.[0] ?? "memory", memory.title + " " + memory.content, { kind: "memory", logId: memory.id });
+            }
+            count++;
+        }
+    }
+    catch (err) {
+        logger.warn("rebuildIndex: failed to load memories", {
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+    const sessions = await kv.list(KV.sessions);
+    if (!sessions.length) {
+        evictGhostVectors(liveIds);
+        return count;
+    }
+    const obsPerSession = [];
+    const failedSessions = [];
+    for (let batch = 0; batch < sessions.length; batch += 10) {
+        const chunk = sessions.slice(batch, batch + 10);
+        const results = await Promise.all(chunk.map(async (s) => {
+            try {
+                return await kv.list(KV.observations(s.id));
+            }
+            catch {
+                failedSessions.push(s.id);
+                return [];
+            }
+        }));
+        obsPerSession.push(...results);
+    }
+    if (failedSessions.length > 0) {
+        logger.warn("rebuildIndex: failed to load observations for sessions", {
+            failedSessions,
+        });
+    }
+    for (const observations of obsPerSession) {
+        for (const obs of observations) {
+            if (obs.title && obs.narrative) {
+                idx.add(obs);
+                liveIds?.add(obs.id);
+                if (!preserveVectors || !vectorIndex?.has(obs.id)) {
+                    await vectorIndexAddGuarded(obs.id, obs.sessionId, obs.title + " " + obs.narrative, {
+                        kind: "observation",
+                        logId: obs.id,
+                    });
+                }
+                count++;
+            }
+        }
+    }
+    evictGhostVectors(liveIds);
+    return count;
+}
+// In preserve (incremental-sync) mode, removes vector entries whose ids no
+// longer exist in KV — docs deleted after the index blob was persisted.
+// No-op when liveIds is null (full-rebuild mode already cleared the index).
+function evictGhostVectors(liveIds) {
+    if (!liveIds || !vectorIndex)
+        return;
+    let evicted = 0;
+    for (const id of vectorIndex.ids()) {
+        if (!liveIds.has(id)) {
+            vectorIndex.remove(id);
+            evicted++;
+        }
+    }
+    if (evicted > 0) {
+        logger.info("vector index: evicted ghost entries after restore", {
+            evicted,
+        });
+    }
+}
+// Reciprocal Rank Fusion of two ranked lists that share the
+// {obsId, sessionId, score} shape (BM25 keyword + semantic vector). Score
+// becomes the summed RRF contribution; ties resolve by it. Same K as the
+// HybridSearch helper.
+const RRF_K = 60;
+function fuseRrf(a, b, limit) {
+    const acc = new Map();
+    const add = (list) => list.forEach((r, i) => {
+        const rrf = 1 / (RRF_K + i + 1);
+        const cur = acc.get(r.obsId);
+        if (cur) {
+            cur.score += rrf;
+            if (!cur.sessionId && r.sessionId)
+                cur.sessionId = r.sessionId;
+        }
+        else {
+            acc.set(r.obsId, { sessionId: r.sessionId, score: rrf });
+        }
+    });
+    add(a);
+    add(b);
+    return Array.from(acc.entries())
+        .map(([obsId, v]) => ({ obsId, sessionId: v.sessionId, score: v.score }))
+        .sort((x, y) => y.score - x.score)
+        .slice(0, limit);
+}
+export function registerSearchFunction(sdk, kv) {
+    sdk.registerFunction("mem::search", async (data) => {
+        const idx = getSearchIndex();
+        // Input validation / normalization.
+        if (typeof data?.query !== "string" || !data.query.trim()) {
+            throw new Error("mem::search: query must be a non-empty string");
+        }
+        const query = data.query.trim();
+        const MAX_LIMIT = 100;
+        let effectiveLimit = 20;
+        if (data.limit !== undefined) {
+            if (!Number.isInteger(data.limit) || data.limit < 1) {
+                throw new Error("mem::search: limit must be a positive integer");
+            }
+            effectiveLimit = Math.min(data.limit, MAX_LIMIT);
+        }
+        // Canonicalize the scope filters (resolve symlinks, trailing slashes,
+        // `..`) so /tmp and /private/tmp — or any two spellings of the same
+        // directory — match. Stored values are canonicalized the same way at
+        // comparison time below.
+        const projectFilter = typeof data.project === "string" && data.project.trim().length > 0
+            ? canonicalizePath(data.project)
+            : undefined;
+        const cwdFilter = typeof data.cwd === "string" && data.cwd.trim().length > 0
+            ? canonicalizePath(data.cwd)
+            : undefined;
+        // Verified Recall firewall: when on (recall surfaces default it on),
+        // drop results that reference files now deleted or content-changed, so
+        // stale memory is never injected. Needs a cwd to check against.
+        const safeOnly = data.safe_only === true && cwdFilter !== undefined;
+        const format = typeof data.format === "string" ? data.format : "full";
+        if (!["full", "compact", "narrative"].includes(format)) {
+            throw new Error("mem::search: format must be one of 'full', 'compact', or 'narrative'");
+        }
+        let tokenBudget;
+        if (data.token_budget !== undefined) {
+            if (!Number.isInteger(data.token_budget) || data.token_budget < 1) {
+                throw new Error("mem::search: token_budget must be a positive integer");
+            }
+            tokenBudget = data.token_budget;
+        }
+        if (idx.size === 0) {
+            // Restore persisted quantized codes first (no-op unless
+            // MEMWARDEN_QUANT_VECTOR is on and a valid blob exists), then
+            // rebuild BM25. With a successful restore the vector side runs in
+            // incremental-sync mode (embed only missing ids, evict ghosts);
+            // afterwards the reconciled index is persisted again so the blob
+            // converges with KV. One blob write per cold rebuild.
+            const restoredVectors = await loadVectorIndex(kv);
+            const count = await rebuildIndex(kv, {
+                preserveVectorIndex: restoredVectors,
+            });
+            const persisted = await persistVectorIndex(kv);
+            logger.info("Search index rebuilt", {
+                entries: count,
+                restoredVectors,
+                persisted,
+            });
+        }
+        // When filtering by project/cwd, over-fetch from the index so the
+        // post-filter still has a chance of returning `effectiveLimit` results.
+        // safe_only over-fetches much harder (and caps at SAFE_SCAN_CAP) so a run
+        // of stale high-ranking hits is unlikely to starve a verified result; if
+        // the scan window is exhausted we log it rather than hide it.
+        const SAFE_SCAN_CAP = 2000;
+        const filtering = !!(projectFilter || cwdFilter);
+        const fetchLimit = safeOnly
+            ? Math.min(SAFE_SCAN_CAP, Math.max(effectiveLimit * 50, 500))
+            : filtering
+                ? Math.max(effectiveLimit * 10, 100)
+                : effectiveLimit;
+        // Measure retrieval itself (not the one-time cold rebuild above) — the
+        // "is finding context fast?" number.
+        const searchStartedAt = performance.now();
+        const bm25Results = idx.search(query, fetchLimit);
+        // Fuse in the semantic stream when an embedding provider + vector index
+        // are present, so meaning-based queries (different words than the
+        // memory) resolve. Provider-less mode stays pure BM25. A failing
+        // embed falls back to BM25 rather than breaking search.
+        let results = bm25Results;
+        const vIdx = getVectorIndex();
+        const ep = currentEmbeddingProvider;
+        if (vIdx && ep && vIdx.size > 0) {
+            try {
+                const qVec = await ep.embed(clipEmbedInput(query));
+                if (qVec.length === ep.dimensions) {
+                    results = fuseRrf(bm25Results, vIdx.search(qVec, fetchLimit), fetchLimit);
+                }
+            }
+            catch (err) {
+                logger.warn("search: vector stream failed — BM25 only", {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+        metrics.recordSearch(performance.now() - searchStartedAt);
+        // Resolve session -> project/cwd once per sessionId we touch.
+        const sessionCache = new Map();
+        const loadSession = async (sessionId) => {
+            if (sessionCache.has(sessionId))
+                return sessionCache.get(sessionId);
+            const s = await kv.get(KV.sessions, sessionId);
+            sessionCache.set(sessionId, s ?? null);
+            return s ?? null;
+        };
+        // Cache for memory project lookups. Memories indexed via mem::remember
+        // use a synthetic sessionId that either has no KV.sessions entry or
+        // belongs to a different project. When loadSession returns null we
+        // fall through to a KV.memories probe so project-filtered search can
+        // include or exclude them correctly.
+        const memoryProjectCache = new Map();
+        const loadMemoryProject = async (obsId) => {
+            if (memoryProjectCache.has(obsId))
+                return memoryProjectCache.get(obsId);
+            const mem = await kv
+                .get(KV.memories, obsId)
+                .catch(() => null);
+            const proj = mem?.project ?? null;
+            memoryProjectCache.set(obsId, proj);
+            return proj;
+        };
+        // A candidate's observation (or a memory rendered as one). Cached so the
+        // Verified Recall firewall and the final assembly never load it twice.
+        const obsCache = new Map();
+        const loadObsOrMemory = async (r) => {
+            if (obsCache.has(r.obsId))
+                return obsCache.get(r.obsId);
+            let obs = await kv
+                .get(KV.observations(r.sessionId), r.obsId)
+                .catch(() => null);
+            if (!obs) {
+                const mem = await kv.get(KV.memories, r.obsId).catch(() => null);
+                obs = mem ? memoryToObservation(mem) : null;
+            }
+            obsCache.set(r.obsId, obs);
+            return obs;
+        };
+        // First pass: scope-filter, and — when safe_only is on — apply the
+        // Verified Recall firewall WHILE filling, so stale top hits don't starve
+        // out lower-ranked verified ones. We keep scanning the fetched results
+        // (fetchLimit, up to SAFE_SCAN_CAP) until we have effectiveLimit safe ones.
+        const candidateTarget = safeOnly
+            ? Math.min(fetchLimit, Math.max(effectiveLimit * 3, effectiveLimit + 20))
+            : effectiveLimit;
+        const candidates = [];
+        let staleDropped = 0;
+        for (const r of results) {
+            if (candidates.length >= candidateTarget)
+                break;
+            if (filtering) {
+                const s = await loadSession(r.sessionId);
+                if (s) {
+                    if (projectFilter && canonicalizePath(s.project) !== projectFilter)
+                        continue;
+                    if (cwdFilter && canonicalizePath(s.cwd) !== cwdFilter)
+                        continue;
+                }
+                else if (projectFilter) {
+                    // Synthetic/memory entry: a null memProject means "unknown" and is
+                    // let through for backward-compatibility; cwd filter doesn't apply.
+                    const memProject = await loadMemoryProject(r.obsId);
+                    if (memProject !== null &&
+                        canonicalizePath(memProject) !== projectFilter)
+                        continue;
+                }
+            }
+            if (safeOnly && cwdFilter) {
+                const obs = await loadObsOrMemory(r);
+                // Fail closed for stale/missing candidates. Sourced-unverified memory
+                // is allowed by design, but stale memory never gets injected.
+                if (!obs || classifyProvenance(obs.provenance, cwdFilter).status === "stale") {
+                    staleDropped++;
+                    continue;
+                }
+            }
+            candidates.push(r);
+        }
+        if (safeOnly && staleDropped > 0) {
+            logger.info("Verified Recall dropped stale results", { dropped: staleDropped });
+        }
+        // No silent cap: if we ran out of safe candidates AND exhausted the scan
+        // window, a verified result could exist beyond it — say so.
+        if (safeOnly &&
+            candidates.length < effectiveLimit &&
+            results.length >= fetchLimit) {
+            logger.warn("Verified Recall scan window exhausted; verified results may exist beyond it", {
+                scanned: results.length,
+                fetchLimit,
+            });
+        }
+        // Second pass: assemble results, reusing any observation loaded above.
+        const obsResults = await Promise.all(candidates.map((c) => loadObsOrMemory(c)));
+        const enriched = [];
+        for (let i = 0; i < candidates.length; i++) {
+            const obs = obsResults[i];
+            const cand = candidates[i];
+            if (obs) {
+                enriched.push({
+                    observation: obs,
+                    score: cand.score,
+                    sessionId: cand.sessionId,
+                });
+            }
+        }
+        // Safe recall NEVER silently drops a memory on a fuzzy contradiction
+        // heuristic — that would lose correct facts from a trust tool. The only
+        // thing safe_only firewalls is STALE memory (handled above, when the
+        // referenced files are deleted/changed). Conflict detection is advisory
+        // only and lives in mem::doctor, not in recall.
+        const recallResults = enriched.slice(0, effectiveLimit);
+        void recordAccessBatch(kv, recallResults.map((r) => r.observation.id));
+        const estimateTokens = (value) => Math.max(1, Math.ceil(JSON.stringify(value).length / 3));
+        const applyTokenBudget = (items) => {
+            if (!tokenBudget)
+                return {
+                    items,
+                    used: items.reduce((sum, item) => sum + estimateTokens(item), 0),
+                    truncated: false,
+                };
+            const selected = [];
+            let used = 0;
+            for (const item of items) {
+                const itemTokens = estimateTokens(item);
+                if (used + itemTokens > tokenBudget) {
+                    return {
+                        items: selected,
+                        used,
+                        truncated: selected.length < items.length,
+                    };
+                }
+                selected.push(item);
+                used += itemTokens;
+            }
+            return { items: selected, used, truncated: false };
+        };
+        if (format === "compact") {
+            const compactResults = recallResults.map((r) => ({
+                obsId: r.observation.id,
+                sessionId: r.sessionId,
+                title: r.observation.title,
+                type: r.observation.type,
+                score: r.score,
+                timestamp: r.observation.timestamp,
+            }));
+            const packed = applyTokenBudget(compactResults);
+            return {
+                format,
+                results: packed.items,
+                tokens_used: packed.used,
+                tokens_budget: tokenBudget,
+                truncated: packed.truncated,
+            };
+        }
+        if (format === "narrative") {
+            const narrativeResults = recallResults.map((r) => ({
+                obsId: r.observation.id,
+                sessionId: r.sessionId,
+                title: r.observation.title,
+                narrative: r.observation.narrative,
+                score: r.score,
+                timestamp: r.observation.timestamp,
+            }));
+            const packed = applyTokenBudget(narrativeResults);
+            const text = packed.items
+                .map((r, idxN) => `${idxN + 1}. ${r.title}\n${r.narrative}`)
+                .join("\n\n");
+            return {
+                format,
+                results: packed.items,
+                text,
+                tokens_used: packed.used,
+                tokens_budget: tokenBudget,
+                truncated: packed.truncated,
+            };
+        }
+        const packed = applyTokenBudget(recallResults);
+        // Avoid logging raw cwd/project (host paths). Log only that filters
+        // were active.
+        logger.info("Search completed", {
+            query,
+            results: packed.items.length,
+            hasProjectFilter: !!projectFilter,
+            hasCwdFilter: !!cwdFilter,
+        });
+        return {
+            format,
+            results: packed.items,
+            tokens_used: packed.used,
+            tokens_budget: tokenBudget,
+            truncated: packed.truncated,
+        };
+    });
+}

package/dist/functions/stemmer.d.ts

@@ -0,0 +1 @@
+export declare function stem(word: string): string;

package/dist/functions/stemmer.js

@@ -0,0 +1,110 @@
+//
+// A compact Porter stemmer. Porter's algorithm is public-domain; this code
+// implements its steps from scratch (plural/past-tense stripping, the step
+// 2-4 suffix maps, and final -e / double-l cleanup). The BM25 tokenizer
+// and the synonym map both run terms through it, so indexing and querying
+// reduce words the same way.
+// Porter's step-2 and step-3 suffix replacements (public algorithm tables).
+const LONG_SUFFIXES = [
+    ["ational", "ate"], ["tional", "tion"], ["enci", "ence"], ["anci", "ance"],
+    ["izer", "ize"], ["iser", "ise"], ["abli", "able"], ["alli", "al"],
+    ["entli", "ent"], ["eli", "e"], ["ousli", "ous"], ["ization", "ize"],
+    ["isation", "ise"], ["ation", "ate"], ["ator", "ate"], ["alism", "al"],
+    ["iveness", "ive"], ["fulness", "ful"], ["ousness", "ous"], ["aliti", "al"],
+    ["iviti", "ive"], ["biliti", "ble"],
+];
+const DERIVATIONAL = [
+    ["icate", "ic"], ["ative", ""], ["alize", "al"], ["alise", "al"],
+    ["iciti", "ic"], ["ical", "ic"], ["ful", ""], ["ness", ""],
+];
+const STEP4_SUFFIX = /(ement|ment|tion|sion|ance|ence|able|ible|ism|ate|iti|ous|ive|ize|ise|ant|ent|al|er|ic|ou)$/;
+const hasVowel = (s) => /[aeiou]/.test(s);
+// Porter's "measure": the count of vowel-then-consonant transitions.
+function measure(s) {
+    return (s.replace(/[^aeiouy]+/g, "C").replace(/[aeiouy]+/g, "V").match(/VC/g) ?? [])
+        .length;
+}
+function endsDoubledConsonant(s) {
+    const n = s.length;
+    return n >= 2 && s[n - 1] === s[n - 2] && !/[aeiou]/.test(s[n - 1] ?? "");
+}
+// consonant-vowel-consonant where the final consonant is not w, x, or y.
+function endsCvc(s) {
+    const n = s.length;
+    if (n < 3)
+        return false;
+    return (!/[aeiou]/.test(s[n - 3] ?? "") &&
+        /[aeiou]/.test(s[n - 2] ?? "") &&
+        !/[aeiouwxy]/.test(s[n - 1] ?? ""));
+}
+// Shared -ed / -ing tail cleanup (Porter step 1b second half).
+function restoreShortStem(s) {
+    if (s.endsWith("at") || s.endsWith("bl") || s.endsWith("iz"))
+        return s + "e";
+    if (endsDoubledConsonant(s) && !/[lsz]$/.test(s))
+        return s.slice(0, -1);
+    if (measure(s) === 1 && endsCvc(s))
+        return s + "e";
+    return s;
+}
+export function stem(word) {
+    if (word.length <= 2)
+        return word;
+    let w = word;
+    // 1a — plurals
+    if (w.endsWith("sses"))
+        w = w.slice(0, -2);
+    else if (w.endsWith("ies"))
+        w = w.slice(0, -2);
+    else if (w.endsWith("s") && !w.endsWith("ss"))
+        w = w.slice(0, -1);
+    // 1b — past tense / progressive
+    if (w.endsWith("eed")) {
+        if (measure(w.slice(0, -3)) > 0)
+            w = w.slice(0, -1);
+    }
+    else if (w.endsWith("ed") && hasVowel(w.slice(0, -2))) {
+        w = restoreShortStem(w.slice(0, -2));
+    }
+    else if (w.endsWith("ing") && hasVowel(w.slice(0, -3))) {
+        w = restoreShortStem(w.slice(0, -3));
+    }
+    // 1c — terminal y to i
+    if (w.endsWith("y") && hasVowel(w.slice(0, -1)))
+        w = w.slice(0, -1) + "i";
+    // 2 and 3 — suffix maps, applied only when a real stem remains
+    for (const [suffix, repl] of LONG_SUFFIXES) {
+        if (w.endsWith(suffix)) {
+            const base = w.slice(0, -suffix.length);
+            if (measure(base) > 0)
+                w = base + repl;
+            break;
+        }
+    }
+    for (const [suffix, repl] of DERIVATIONAL) {
+        if (w.endsWith(suffix)) {
+            const base = w.slice(0, -suffix.length);
+            if (measure(base) > 0)
+                w = base + repl;
+            break;
+        }
+    }
+    // 4 — strip a residual suffix from a long enough stem
+    const m4 = w.match(STEP4_SUFFIX);
+    if (m4) {
+        const base = w.slice(0, -m4[0].length);
+        if (measure(base) > 1)
+            w = base;
+    }
+    // 5a — drop a trailing e
+    if (w.endsWith("e")) {
+        const base = w.slice(0, -1);
+        if (measure(base) > 1 || (measure(base) === 1 && !endsCvc(base)))
+            w = base;
+    }
+    // 5b — collapse a doubled l
+    if (endsDoubledConsonant(w) && w.endsWith("l") && measure(w.slice(0, -1)) > 1) {
+        w = w.slice(0, -1);
+    }
+    return w;
+}

package/dist/functions/synonyms.d.ts

@@ -0,0 +1 @@
+export declare function getSynonyms(stemmedTerm: string): string[];