@aexol/spectral 0.9.199 → 0.9.201

package/dist/server/storage.js

@@ -28,6 +28,7 @@
  * later (e.g. if we ever need to colocate with a remote service).
  */
 import { createSqliteAdapter } from "./sqlite-adapter.js";
+import { MIN_TRIGRAM_TOKEN_LENGTH, bestTokenTrigramJaccard, buildFtsMatchQuery, trigramJaccard, } from "./text/trigrams.js";
 import { randomUUID } from "node:crypto";
 import { mkdirSync, readFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
@@ -149,6 +150,61 @@ CREATE TABLE IF NOT EXISTS dev_process_definitions (
   updated_at INTEGER NOT NULL
 );
 `;
+/**
+ * FTS5 trigram index over project-observation content, synced explicitly in
+ * code (same transaction as `insertProjectObservations`) rather than via
+ * triggers so the behavior is obvious and testable. `obs_id` stores the
+ * observation id so stale index rows can be deleted on INSERT OR REPLACE.
+ *
+ * Created outside SCHEMA_SQL on purpose: if the bundled SQLite build lacks
+ * FTS5, the store still opens and observation search degrades to LIKE
+ * filtering instead of failing at startup.
+ */
+const PROJECT_OBS_FTS_SQL = `
+CREATE VIRTUAL TABLE IF NOT EXISTS project_observations_fts USING fts5(
+  content,
+  project_id UNINDEXED,
+  obs_id UNINDEXED,
+  tokenize='trigram'
+);
+`;
+// ---- observational-memory ranking constants ------------------------------
+/** Weight of the text-match component (bm25-derived or trigram similarity). */
+const WEIGHT_TEXT_MATCH = 0.6;
+/** Weight of the relevance-tag component (critical=1.0 … low=0.25). */
+const WEIGHT_RELEVANCE = 0.25;
+/** Weight of the recency component. */
+const WEIGHT_RECENCY = 0.15;
+/** Recency decay constant: exp(-ageDays / RECENCY_DECAY_DAYS). */
+const RECENCY_DECAY_DAYS = 30;
+const MS_PER_DAY = 86_400_000;
+/** Relevance-tag weights used by the ranking formula. */
+const RELEVANCE_WEIGHTS = {
+    critical: 1.0,
+    high: 0.75,
+    medium: 0.5,
+    low: 0.25,
+};
+/** Minimum mean trigram Jaccard similarity for a typo-tolerant fallback match. */
+const TYPO_FALLBACK_MIN_MEAN_SIMILARITY = 0.3;
+/**
+ * Trigram Jaccard similarity at or above which a new observation is treated
+ * as a near-duplicate rewording of an existing row and replaces it.
+ */
+const NEAR_DUPLICATE_SIMILARITY = 0.9;
+/**
+ * Final ranking score for a candidate observation: a weighted blend of
+ * normalized text-match quality, relevance tag and recency decay. Every
+ * component is in [0, 1]; higher is better.
+ */
+function scoreObservation(textScore, relevance, createdAt, now) {
+    const ageDays = Math.max(0, (now - createdAt) / MS_PER_DAY);
+    const recency = Math.exp(-ageDays / RECENCY_DECAY_DAYS);
+    const relevanceWeight = RELEVANCE_WEIGHTS[relevance] ?? RELEVANCE_WEIGHTS.low;
+    return (textScore * WEIGHT_TEXT_MATCH +
+        relevanceWeight * WEIGHT_RELEVANCE +
+        recency * WEIGHT_RECENCY);
+}
 /**
  * Synchronous binding-file reader for a project at the given filesystem path.
  * Returns null when no `.aexol/aexol.jsonc` exists at that path.
@@ -269,9 +325,20 @@ export class SessionStore {
     stmtGetForkCompactSource;
     // Project observations: cross-session durable memory.
     stmtInsertProjectObs;
-    stmtSearchProjectObs;
     stmtGetProjectObsById;
     stmtGetProjectByCwd;
+    // Shared by the typo-tolerant search fallback and near-duplicate detection.
+    stmtListProjectObsByProject;
+    stmtDeleteProjectObs;
+    stmtCountProjectObs;
+    // FTS5 trigram index statements. Null when the bundled SQLite build has no
+    // FTS5 — the store then degrades to LIKE-only search (see `ftsEnabled`).
+    ftsEnabled = false;
+    stmtInsertProjectObsFts = null;
+    stmtDeleteProjectObsFts = null;
+    stmtDeleteProjectObsFtsByProject = null;
+    stmtSearchProjectObsFts = null;
+    stmtCountProjectObsFts = null;
     stmtUpsertProjectRecallSource;
     stmtGetProjectRecallSource;
     // Prompt queue statements
@@ -539,14 +606,15 @@ export class SessionStore {
         this.stmtGetForkCompactSource = this.db.prepare(`SELECT fork_compact_source_id FROM sessions WHERE id = ?`);
         this.stmtInsertProjectObs = this.db.prepare(`INSERT OR REPLACE INTO project_observations (id, project_id, session_id, content, relevance, created_at)
        VALUES (?, ?, ?, ?, ?, ?)`);
-        this.stmtSearchProjectObs = this.db.prepare(`SELECT id, project_id, session_id, content, relevance, created_at
-       FROM project_observations
-       WHERE project_id = ? AND content LIKE ?
-       ORDER BY created_at DESC
-       LIMIT 20`);
         this.stmtGetProjectObsById = this.db.prepare(`SELECT id, project_id, session_id, content, relevance, created_at
        FROM project_observations
        WHERE project_id = ? AND id = ?`);
+        this.stmtListProjectObsByProject = this.db.prepare(`SELECT id, project_id, session_id, content, relevance, created_at
+       FROM project_observations
+       WHERE project_id = ?
+       ORDER BY created_at DESC`);
+        this.stmtDeleteProjectObs = this.db.prepare(`DELETE FROM project_observations WHERE id = ? AND project_id = ?`);
+        this.stmtCountProjectObs = this.db.prepare(`SELECT COUNT(*) AS count FROM project_observations`);
         this.stmtGetProjectByCwd = this.db.prepare(`SELECT id FROM projects WHERE path = ? LIMIT 1`);
         this.stmtUpsertProjectRecallSource = this.db.prepare(`INSERT OR REPLACE INTO project_recall_sources
        (project_id, memory_id, entries_json, updated_at)
@@ -554,6 +622,35 @@ export class SessionStore {
         this.stmtGetProjectRecallSource = this.db.prepare(`SELECT project_id, memory_id, entries_json, updated_at
        FROM project_recall_sources
        WHERE project_id = ? AND memory_id = ?`);
+        // ---- project-observation FTS trigram index ---------------------------
+        // The index is a standalone FTS5 virtual table synced explicitly in code
+        // (same transaction as insertProjectObservations). If the SQLite build
+        // lacks FTS5 we log a warning and degrade to LIKE-only search instead of
+        // failing to open the store.
+        try {
+            this.db.exec(PROJECT_OBS_FTS_SQL);
+            this.stmtInsertProjectObsFts = this.db.prepare(`INSERT INTO project_observations_fts (content, project_id, obs_id) VALUES (?, ?, ?)`);
+            this.stmtDeleteProjectObsFts = this.db.prepare(`DELETE FROM project_observations_fts WHERE obs_id = ? AND project_id = ?`);
+            this.stmtDeleteProjectObsFtsByProject = this.db.prepare(`DELETE FROM project_observations_fts WHERE project_id = ?`);
+            this.stmtSearchProjectObsFts = this.db.prepare(`SELECT o.id, o.session_id, o.content, o.relevance, o.created_at,
+                bm25(project_observations_fts) AS bm25_score
+         FROM project_observations_fts
+         JOIN project_observations o
+           ON o.id = project_observations_fts.obs_id
+          AND o.project_id = project_observations_fts.project_id
+         WHERE project_observations_fts MATCH ?
+           AND project_observations_fts.project_id = ?`);
+            this.stmtCountProjectObsFts = this.db.prepare(`SELECT COUNT(*) AS count FROM project_observations_fts`);
+            this.backfillProjectObservationsFts();
+            this.ftsEnabled = true;
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            console.warn(`[storage] FTS5 trigram index unavailable (${msg}); ` +
+                `project-observation search degrades to a LIKE-conjunction ` +
+                `(exact token substring) scan without typo tolerance or bm25 ranking.`);
+            this.ftsEnabled = false;
+        }
         // ---- prompt queue statements ----------------------------------------
         this.stmtEnqueuePrompt = this.db.prepare(`INSERT INTO prompt_queue (id, session_id, content, images_json, position, created_at)
        VALUES (?, ?, ?, ?, ?, ?)`);
@@ -736,6 +833,11 @@ export class SessionStore {
     deleteProject(id) {
         const sessionIds = this.stmtListSessionIdsByProject.all(id).map((r) => r.id);
         const info = this.stmtDeleteProject.run(id);
+        if (info.changes > 0) {
+            // project_observations rows cascade via FK, but the standalone FTS
+            // index has no FK — clean it explicitly so it never accumulates orphans.
+            this.stmtDeleteProjectObsFtsByProject?.run(id);
+        }
         return { deleted: info.changes > 0, sessionIds };
     }
     /** Sessions belonging to a single project, newest-first. */
@@ -1358,29 +1460,202 @@ export class SessionStore {
     /**
      * Insert multiple project observations in a single transaction.
      * Uses INSERT OR REPLACE so re-running after the same compaction is idempotent.
+     *
+     * Write-time near-duplicate detection: an observation whose content is a
+     * near-identical rewording (trigram Jaccard >= NEAR_DUPLICATE_SIMILARITY)
+     * of an existing row replaces it — new wording wins — so refinements of
+     * the same fact don't accumulate as duplicates. The FTS5 trigram index is
+     * synced in the same transaction.
      */
     insertProjectObservations(projectId, sessionId, observations, createdAt) {
         if (observations.length === 0)
             return;
         const tx = this.db.transaction(() => {
             for (const obs of observations) {
+                // Near-duplicate detection: a reworded duplicate replaces the older
+                // row. Identical re-inserts also match (similarity 1.0), which keeps
+                // INSERT OR REPLACE idempotency intact.
+                const duplicate = this.findNearDuplicateObservation(projectId, obs.content);
+                if (duplicate) {
+                    this.stmtDeleteProjectObs.run(duplicate.id, projectId);
+                    this.stmtDeleteProjectObsFts?.run(duplicate.id, projectId);
+                }
                 this.stmtInsertProjectObs.run(obs.id, projectId, sessionId, obs.content, obs.relevance, createdAt);
+                if (this.ftsEnabled) {
+                    // INSERT OR REPLACE semantics: drop any stale index row for this id
+                    // before indexing the new content.
+                    this.stmtDeleteProjectObsFts?.run(obs.id, projectId);
+                    this.stmtInsertProjectObsFts?.run(obs.content, projectId, obs.id);
+                }
             }
         });
         tx();
     }
     /**
-     * Search project observations by substring match. Returns up to 20 most recent.
+     * Find an existing project observation whose content is a near-identical
+     * rewording of `content` (trigram Jaccard >= NEAR_DUPLICATE_SIMILARITY).
+     */
+    findNearDuplicateObservation(projectId, content) {
+        let best = null;
+        let bestSimilarity = 0;
+        for (const row of this.stmtListProjectObsByProject.all(projectId)) {
+            const similarity = trigramJaccard(content, row.content);
+            if (similarity > bestSimilarity) {
+                bestSimilarity = similarity;
+                best = row;
+            }
+        }
+        return bestSimilarity >= NEAR_DUPLICATE_SIMILARITY ? best : null;
+    }
+    /**
+     * Search project observations using n-gram matching.
+     *
+     * Query tokens are matched via the FTS5 trigram index: each token becomes
+     * a quoted phrase joined with AND, so a multi-word query matches
+     * observations containing ALL tokens anywhere (case-insensitive substring
+     * semantics). Tokens shorter than 3 characters cannot be matched by the
+     * trigram tokenizer; they are applied as a complementary LIKE filter over
+     * the FTS candidates (or as a plain LIKE conjunction when every token is
+     * short, or when the index is unavailable). When no candidate survives,
+     * a typo-tolerant fallback keeps observations whose token-level trigram
+     * similarity to the query is >= TYPO_FALLBACK_MIN_MEAN_SIMILARITY.
+     *
+     * Results are ranked by a blend of text-match quality, relevance tag and
+     * recency. The return shape is unchanged from the previous substring
+     * search: { content, relevance, createdAt, sessionId }.
      */
     searchProjectObservations(projectId, query, limit = 20) {
-        const pattern = `%${query}%`;
-        const rows = this.stmtSearchProjectObs.all(projectId, pattern);
-        return rows.slice(0, limit).map((r) => ({
+        const trimmed = query.trim();
+        if (!trimmed)
+            return [];
+        const tokens = trimmed.split(/\s+/).filter((t) => t.length > 0);
+        if (tokens.length === 0)
+            return [];
+        const now = Date.now();
+        return this.findObservationCandidates(projectId, tokens)
+            .map((candidate) => ({
+            candidate,
+            score: scoreObservation(candidate.textScore, candidate.relevance, candidate.created_at, now),
+        }))
+            .sort((a, b) => b.score - a.score || b.candidate.created_at - a.candidate.created_at)
+            .slice(0, limit)
+            .map(({ candidate }) => ({
+            content: candidate.content,
+            relevance: candidate.relevance,
+            createdAt: candidate.created_at,
+            sessionId: candidate.session_id,
+        }));
+    }
+    /**
+     * Collect candidate observations for a tokenized query, assigning each a
+     * normalized text-match score in (0, 1].
+     */
+    findObservationCandidates(projectId, tokens) {
+        const longTokens = tokens.filter((t) => t.length >= MIN_TRIGRAM_TOKEN_LENGTH);
+        const shortTokens = tokens.filter((t) => t.length < MIN_TRIGRAM_TOKEN_LENGTH);
+        // Primary path: FTS5 trigram MATCH over the trigram-matchable tokens.
+        if (longTokens.length > 0 && this.ftsEnabled) {
+            const candidates = this.searchObservationsFts(projectId, longTokens, shortTokens);
+            if (candidates.length > 0)
+                return candidates;
+        }
+        else if (!this.ftsEnabled && longTokens.length > 0) {
+            // FTS5 unavailable: LIKE-conjunction scan over the project's rows using
+            // ALL tokens as needles, preserving the pre-FTS substring semantics for
+            // non-token-aligned substrings (e.g. "igrate rollb").
+            const needles = tokens.map((t) => t.toLowerCase());
+            return this.stmtListProjectObsByProject
+                .all(projectId)
+                .filter((row) => {
+                const content = row.content.toLowerCase();
+                return needles.every((needle) => content.includes(needle));
+            })
+                .map((row) => ({ ...row, textScore: 1 }));
+        }
+        else if (longTokens.length === 0) {
+            // All tokens shorter than a trigram: plain LIKE conjunction.
+            const needles = shortTokens.map((t) => t.toLowerCase());
+            return this.stmtListProjectObsByProject
+                .all(projectId)
+                .filter((row) => {
+                const content = row.content.toLowerCase();
+                return needles.every((needle) => content.includes(needle));
+            })
+                .map((row) => ({ ...row, textScore: 1 }));
+        }
+        // Typo-tolerant fallback: nothing matched exactly. Scan the project's
+        // observations (small corpus) and keep near matches by trigram
+        // similarity, which substitutes for the bm25 component in the ranking.
+        return this.stmtListProjectObsByProject
+            .all(projectId)
+            .map((row) => ({ row, similarity: bestTokenTrigramJaccard(tokens, row.content) }))
+            .filter(({ similarity }) => similarity >= TYPO_FALLBACK_MIN_MEAN_SIMILARITY)
+            .map(({ row, similarity }) => ({ ...row, textScore: similarity }));
+    }
+    /**
+     * FTS5 trigram MATCH over `longTokens`, then a complementary LIKE filter
+     * for tokens too short for the trigram tokenizer. Returns an empty array
+     * when the index yields nothing usable (the caller then falls back).
+     */
+    searchObservationsFts(projectId, longTokens, shortTokens) {
+        const matchQuery = buildFtsMatchQuery(longTokens);
+        if (!matchQuery)
+            return [];
+        let rows;
+        try {
+            rows = this.stmtSearchProjectObsFts?.all(matchQuery, projectId) ?? [];
+        }
+        catch (err) {
+            // Defensive: a MATCH failure should degrade to the fallback path, not
+            // surface as a tool error.
+            const msg = err instanceof Error ? err.message : String(err);
+            console.warn(`[storage] project-observation FTS query failed: ${msg}`);
+            return [];
+        }
+        let candidates = rows.map((r) => ({
+            id: r.id,
+            project_id: projectId,
+            session_id: r.session_id,
             content: r.content,
             relevance: r.relevance,
-            createdAt: r.created_at,
-            sessionId: r.session_id,
+            created_at: r.created_at,
+            // bm25() is negative with lower = better in SQLite; 1 - exp(bm25)
+            // maps it to a bounded, higher-is-better text score in (0, 1].
+            textScore: 1 - Math.exp(r.bm25_score),
         }));
+        for (const token of shortTokens) {
+            const needle = token.toLowerCase();
+            candidates = candidates.filter((c) => c.content.toLowerCase().includes(needle));
+        }
+        return candidates;
+    }
+    /**
+     * Rebuild the FTS index from project_observations when it is out of sync
+     * (empty or partial relative to the base table). Handles databases created
+     * before the FTS index existed and any drift (e.g. projects deleted by an
+     * older build without FTS cleanup). No-op when the counts already match.
+     */
+    backfillProjectObservationsFts() {
+        const obsCount = this.stmtCountProjectObs.get()?.count ?? 0;
+        const ftsCount = this.stmtCountProjectObsFts?.get()?.count ?? 0;
+        if (obsCount === ftsCount)
+            return;
+        const tx = this.db.transaction(() => {
+            this.db.exec(`DELETE FROM project_observations_fts`);
+            if (obsCount > 0) {
+                this.db.exec(`INSERT INTO project_observations_fts (content, project_id, obs_id)
+           SELECT content, project_id, id FROM project_observations`);
+            }
+        });
+        tx();
+    }
+    /**
+     * Whether the FTS5 trigram index is active for project observations.
+     * Exposed for diagnostics and tests; false means search is running in
+     * LIKE-only degradation mode.
+     */
+    isProjectObsFtsEnabled() {
+        return this.ftsEnabled;
     }
     /**
      * Get a single project observation by id. Returns null if not found.

package/dist/server/text/trigrams.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Character-trigram text utilities for observational memory.
+ *
+ * Shared by the storage layer for:
+ *   - building FTS5 MATCH queries against the trigram-tokenized index
+ *     (`buildFtsMatchQuery`),
+ *   - typo-tolerant search fallback (`bestTokenTrigramJaccard`),
+ *   - write-time near-duplicate detection (`trigramJaccard`).
+ *
+ * Pure functions with no SQLite dependency so the logic is unit-testable
+ * in isolation.
+ */
+/**
+ * Minimum token length the SQLite trigram tokenizer can match. Tokens
+ * shorter than this cannot be satisfied by an FTS5 MATCH query and must be
+ * handled with complementary LIKE filtering (or the LIKE-only search path).
+ */
+export declare const MIN_TRIGRAM_TOKEN_LENGTH = 3;
+/**
+ * Per-token floor for {@link bestTokenTrigramJaccard}: every trigram-matchable
+ * query token must reach this Jaccard similarity against some content token,
+ * otherwise the content is considered not to match the query at all. This
+ * mirrors the AND semantics of the FTS MATCH path — every term must be
+ * (approximately) present — while still tolerating typos.
+ */
+export declare const TYPO_FALLBACK_MIN_TOKEN_SIMILARITY = 0.2;
+/** Normalize text for trigram comparison: lowercase and collapse whitespace. */
+export declare function normalizeForTrigrams(text: string): string;
+/**
+ * Build the set of character trigrams of `text`.
+ *
+ * Strings shorter than one trigram collapse to a single pseudo-token so
+ * identical short strings still compare equal (Jaccard 1.0). Empty input
+ * yields an empty set.
+ */
+export declare function trigramSet(text: string): Set<string>;
+/**
+ * Jaccard similarity between the character-trigram sets of two strings,
+ * in [0, 1]. Returns 0 when either side has no trigrams.
+ */
+export declare function trigramJaccard(a: string, b: string): number;
+/**
+ * Typo-tolerant similarity between a whole query and a piece of content:
+ * the mean, over query tokens, of the best Jaccard similarity between that
+ * token and any single content token.
+ *
+ * Comparing token-to-token (instead of query-to-full-content) keeps the
+ * score robust to sentence length — a one-word typo in a long observation
+ * would otherwise be diluted below any useful threshold.
+ *
+ * Only `queryTokens` of trigram-matchable length are scored; shorter tokens
+ * are ignored so they do not drag the mean down for matches they can never
+ * express. Returns 0 — no match — when any such token fails to reach
+ * TYPO_FALLBACK_MIN_TOKEN_SIMILARITY against every content token (the token
+ * is absent, mirroring the AND semantics of the FTS path), or when no query
+ * token qualifies.
+ */
+export declare function bestTokenTrigramJaccard(queryTokens: string[], content: string): number;
+/**
+ * Build an FTS5 MATCH query from search tokens.
+ *
+ * Each token becomes a quoted phrase (`"token"`); phrases are joined with
+ * AND. With the trigram tokenizer each quoted phrase acts as a
+ * case-insensitive substring match, so a multi-token query matches content
+ * containing all tokens anywhere. Embedded double quotes are escaped by
+ * doubling, which is how FTS5 string literals represent a literal quote.
+ *
+ * Returns an empty string when no non-empty tokens are supplied — callers
+ * must guard against that before running a MATCH query.
+ */
+export declare function buildFtsMatchQuery(tokens: string[]): string;
+//# sourceMappingURL=trigrams.d.ts.map

package/dist/server/text/trigrams.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trigrams.d.ts","sourceRoot":"","sources":["../../../src/server/text/trigrams.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH;;;;GAIG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAE1C;;;;;;GAMG;AACH,eAAO,MAAM,kCAAkC,MAAM,CAAC;AAEtD,gFAAgF;AAChF,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAWpD;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAI3D;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uBAAuB,CACrC,WAAW,EAAE,MAAM,EAAE,EACrB,OAAO,EAAE,MAAM,GACd,MAAM,CAoBR;AAaD;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAK3D"}

package/dist/server/text/trigrams.js

@@ -0,0 +1,128 @@
+/**
+ * Character-trigram text utilities for observational memory.
+ *
+ * Shared by the storage layer for:
+ *   - building FTS5 MATCH queries against the trigram-tokenized index
+ *     (`buildFtsMatchQuery`),
+ *   - typo-tolerant search fallback (`bestTokenTrigramJaccard`),
+ *   - write-time near-duplicate detection (`trigramJaccard`).
+ *
+ * Pure functions with no SQLite dependency so the logic is unit-testable
+ * in isolation.
+ */
+/**
+ * Minimum token length the SQLite trigram tokenizer can match. Tokens
+ * shorter than this cannot be satisfied by an FTS5 MATCH query and must be
+ * handled with complementary LIKE filtering (or the LIKE-only search path).
+ */
+export const MIN_TRIGRAM_TOKEN_LENGTH = 3;
+/**
+ * Per-token floor for {@link bestTokenTrigramJaccard}: every trigram-matchable
+ * query token must reach this Jaccard similarity against some content token,
+ * otherwise the content is considered not to match the query at all. This
+ * mirrors the AND semantics of the FTS MATCH path — every term must be
+ * (approximately) present — while still tolerating typos.
+ */
+export const TYPO_FALLBACK_MIN_TOKEN_SIMILARITY = 0.2;
+/** Normalize text for trigram comparison: lowercase and collapse whitespace. */
+export function normalizeForTrigrams(text) {
+    return text.toLowerCase().replace(/\s+/g, " ").trim();
+}
+/**
+ * Build the set of character trigrams of `text`.
+ *
+ * Strings shorter than one trigram collapse to a single pseudo-token so
+ * identical short strings still compare equal (Jaccard 1.0). Empty input
+ * yields an empty set.
+ */
+export function trigramSet(text) {
+    const normalized = normalizeForTrigrams(text);
+    if (normalized.length === 0)
+        return new Set();
+    if (normalized.length < MIN_TRIGRAM_TOKEN_LENGTH) {
+        return new Set([normalized]);
+    }
+    const grams = new Set();
+    for (let i = 0; i + MIN_TRIGRAM_TOKEN_LENGTH <= normalized.length; i++) {
+        grams.add(normalized.slice(i, i + MIN_TRIGRAM_TOKEN_LENGTH));
+    }
+    return grams;
+}
+/**
+ * Jaccard similarity between the character-trigram sets of two strings,
+ * in [0, 1]. Returns 0 when either side has no trigrams.
+ */
+export function trigramJaccard(a, b) {
+    const setA = trigramSet(a);
+    const setB = trigramSet(b);
+    return jaccard(setA, setB);
+}
+/**
+ * Typo-tolerant similarity between a whole query and a piece of content:
+ * the mean, over query tokens, of the best Jaccard similarity between that
+ * token and any single content token.
+ *
+ * Comparing token-to-token (instead of query-to-full-content) keeps the
+ * score robust to sentence length — a one-word typo in a long observation
+ * would otherwise be diluted below any useful threshold.
+ *
+ * Only `queryTokens` of trigram-matchable length are scored; shorter tokens
+ * are ignored so they do not drag the mean down for matches they can never
+ * express. Returns 0 — no match — when any such token fails to reach
+ * TYPO_FALLBACK_MIN_TOKEN_SIMILARITY against every content token (the token
+ * is absent, mirroring the AND semantics of the FTS path), or when no query
+ * token qualifies.
+ */
+export function bestTokenTrigramJaccard(queryTokens, content) {
+    const scorable = queryTokens.filter((t) => t.length >= MIN_TRIGRAM_TOKEN_LENGTH);
+    if (scorable.length === 0)
+        return 0;
+    const contentTokens = content.split(/\s+/).filter((t) => t.length > 0);
+    const contentTokenGrams = contentTokens.map((token) => trigramSet(token));
+    let total = 0;
+    for (const queryToken of scorable) {
+        let best = 0;
+        const queryGrams = trigramSet(queryToken);
+        for (const grams of contentTokenGrams) {
+            const similarity = jaccard(queryGrams, grams);
+            if (similarity > best)
+                best = similarity;
+            if (best === 1)
+                break;
+        }
+        if (best < TYPO_FALLBACK_MIN_TOKEN_SIMILARITY)
+            return 0;
+        total += best;
+    }
+    return total / scorable.length;
+}
+/** Jaccard similarity between two pre-built sets, in [0, 1]. */
+function jaccard(a, b) {
+    if (a.size === 0 || b.size === 0)
+        return 0;
+    let intersection = 0;
+    for (const gram of a) {
+        if (b.has(gram))
+            intersection++;
+    }
+    const union = a.size + b.size - intersection;
+    return union === 0 ? 0 : intersection / union;
+}
+/**
+ * Build an FTS5 MATCH query from search tokens.
+ *
+ * Each token becomes a quoted phrase (`"token"`); phrases are joined with
+ * AND. With the trigram tokenizer each quoted phrase acts as a
+ * case-insensitive substring match, so a multi-token query matches content
+ * containing all tokens anywhere. Embedded double quotes are escaped by
+ * doubling, which is how FTS5 string literals represent a literal quote.
+ *
+ * Returns an empty string when no non-empty tokens are supplied — callers
+ * must guard against that before running a MATCH query.
+ */
+export function buildFtsMatchQuery(tokens) {
+    return tokens
+        .filter((token) => token.length > 0)
+        .map((token) => `"${token.replace(/"/g, '""')}"`)
+        .join(" AND ");
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aexol/spectral",
-  "version": "0.9.199",
+  "version": "0.9.201",
   "description": "AI coding agent for Aexol with relay-based browser access.",
   "type": "module",
   "private": false,