@jestek-dev/scripture-engine 0.7.0 → 0.14.0

package/dist/corpus/repository.js

@@ -48,6 +48,15 @@ export class CorpusRepository {
     constructor(database) {
         this.database = database;
     }
+    /**
+     * The alias vocabulary for the reference did-you-mean (0.11.0/QR-4),
+     * fetched through the port once per repository instance and cached: ~270
+     * rows that cannot change under a running engine (the artifact is
+     * immutable), so re-reading them per query would be waste, and caching
+     * keeps the engine's no-I/O covenant intact — the ONE read still goes
+     * through ContentQueryPort.
+     */
+    bookAliasCache = null;
     async close() {
         await this.database.close();
     }
@@ -92,6 +101,21 @@ export class CorpusRepository {
         const result = await this.database.execute('SELECT 1 AS present FROM verses WHERE book_id = ? AND chapter = ? AND verse = ? LIMIT 1', [bookId, chapter, verse]);
         return result.rows.length > 0;
     }
+    async listBookAliases() {
+        if (this.bookAliasCache)
+            return this.bookAliasCache;
+        const result = await this.database.execute(`SELECT a.alias_key AS aliasKey, b.id AS bookId, b.name AS bookName,
+              b.chapter_count AS chapterCount
+       FROM book_aliases a JOIN books b ON b.id = a.book_id
+       ORDER BY a.alias_key`);
+        this.bookAliasCache = result.rows.map((row) => ({
+            aliasKey: str(row, 'aliasKey'),
+            bookId: num(row, 'bookId'),
+            bookName: str(row, 'bookName'),
+            chapterCount: num(row, 'chapterCount'),
+        }));
+        return this.bookAliasCache;
+    }
     async resolveReference(input) {
         return await resolveReferenceAttempt(input, this);
     }
@@ -222,6 +246,176 @@ export class CorpusRepository {
        FROM token_stats WHERE token IN (${placeholders}) GROUP BY token`, unique);
         return new Map(result.rows.map((row) => [str(row, 'token'), num(row, 'df')]));
     }
+    /**
+     * Whether this artifact carries the precomputed spelling index
+     * (schema v7, 0.12.0/QR-5). Presence-probed like the other optional
+     * layers: a v6 artifact simply has no tables, the probe returns false, and
+     * the engine gracefully does not correct — behaving exactly as the
+     * pre-spelling engine did. That probe IS the rollback story: rebuild the
+     * artifact without the tables and behavior reverts with no engine change.
+     */
+    async hasSpellingIndex() {
+        try {
+            const result = await this.database.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'spelling_terms'");
+            return result.rows.length > 0;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Whether this artifact carries the derived pericope tiling (schema v8,
+     * CO-3 PR 1). Presence-and-rows probed like the other optional layers: a
+     * v7 artifact has no table, an emptied table disables the (future)
+     * grouping step silently, and behavior reverts to pre-pericope output
+     * with no engine change — the probe IS the rollback story.
+     */
+    async hasPericopes() {
+        try {
+            const table = await this.database.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'pericopes'");
+            if (table.rows.length === 0)
+                return false;
+            const rows = await this.database.execute('SELECT 1 AS present FROM pericopes LIMIT 1');
+            return rows.rows.length > 0;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * The pericopes containing any of the given verse ids, batched as ONE
+     * bounded query over the ranked window (G11): the window's min..max verse
+     * span overlaps few pericopes, and the caller maps verses to rows. Rows
+     * come back ordered by start verse for platform-stable iteration.
+     *
+     * NO CALL SITES in discover() yet (CO-3 PR 1 capability): the grouping
+     * behavior that consumes this lands with the PR 2 ENGINE_VERSION bump.
+     * boundaryVotes is the summed boundary vote at the pericope's start verse
+     * — the countable fact the artifact stores, so a future explanation and
+     * the shipped data cannot disagree.
+     */
+    async pericopesContaining(verseIds) {
+        const unique = [...new Set(verseIds)];
+        if (unique.length === 0)
+            return [];
+        const result = await this.database.execute(`SELECT start_verse_id AS startVerseId, end_verse_id AS endVerseId,
+              boundary_votes AS boundaryVotes, source_id AS sourceId
+       FROM pericopes
+       WHERE end_verse_id >= ? AND start_verse_id <= ?
+       ORDER BY start_verse_id`, [Math.min(...unique), Math.max(...unique)]);
+        const spans = result.rows.map((row) => ({
+            startVerseId: num(row, 'startVerseId'),
+            endVerseId: num(row, 'endVerseId'),
+            boundaryVotes: num(row, 'boundaryVotes'),
+            sourceId: str(row, 'sourceId'),
+        }));
+        // The min..max window can overlap pericopes containing none of the
+        // asked-for verses; keep only real containers so the caller's mapping
+        // stays honest.
+        return spans.filter((span) => unique.some((verseId) => verseId >= span.startVerseId && verseId <= span.endVerseId));
+    }
+    /**
+     * Whether this artifact carries the mined TSK cross-reference phrase keys
+     * (schema v9, B3 Phase A). Presence-and-rows probed like pericopes: a v8
+     * artifact has no table, an emptied table reads false, and (future)
+     * phrase-labeled behavior reverts to plain cross_references output with no
+     * engine change — the probe IS the rollback story.
+     */
+    async hasCrossReferencePhrases() {
+        try {
+            const table = await this.database.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'cross_reference_phrases'");
+            if (table.rows.length === 0)
+                return false;
+            const rows = await this.database.execute('SELECT 1 AS present FROM cross_reference_phrases LIMIT 1');
+            return rows.rows.length > 0;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * The cross-reference phrase triples whose FROM verse is one of the given
+     * verse ids, batched as ONE bounded query over the window's min..max span
+     * (G11) and filtered back to the asked-for verses. Rows come back ordered
+     * by (from, phrase, start, end, source) for platform-stable iteration —
+     * sorted HERE, in engine code, by UTF-16 code units (the same comparison
+     * buildConceptLayer's fingerprint feed uses), never by the port's SQL
+     * collation: SQLite's BINARY collation compares UTF-8 bytes, which
+     * disagrees with JS on some non-ASCII strings, and the ordering contract
+     * must not depend on which side compares.
+     *
+     * NO CALL SITES in discover() yet (B3 Phase A capability): the labeling
+     * and off-phrase-discount behavior that consumes this lands with the
+     * Phase B ENGINE_VERSION bump behind J26/J55.
+     */
+    async crossReferencePhrasesFor(verseIds) {
+        const unique = [...new Set(verseIds)];
+        if (unique.length === 0)
+            return [];
+        const asked = new Set(unique);
+        const result = await this.database.execute(`SELECT from_verse_id AS fromVerseId, normalized_phrase AS normalizedPhrase,
+              to_start_verse_id AS toStartVerseId, to_end_verse_id AS toEndVerseId,
+              source_id AS sourceId
+       FROM cross_reference_phrases
+       WHERE from_verse_id >= ? AND from_verse_id <= ?
+       ORDER BY from_verse_id`, [Math.min(...unique), Math.max(...unique)]);
+        // The min..max window can include from-verses nobody asked about; keep
+        // only the asked-for ones so the caller's mapping stays honest.
+        return result.rows
+            .map((row) => ({
+            fromVerseId: num(row, 'fromVerseId'),
+            normalizedPhrase: str(row, 'normalizedPhrase'),
+            toStartVerseId: num(row, 'toStartVerseId'),
+            toEndVerseId: num(row, 'toEndVerseId'),
+            sourceId: str(row, 'sourceId'),
+        }))
+            .filter((row) => asked.has(row.fromVerseId))
+            // source_id joins the tie-break so two sources naming the same
+            // (from, phrase, target) triple can never come back in
+            // platform-unspecified order once a second phrase source exists.
+            .sort((a, b) => a.fromVerseId - b.fromVerseId ||
+            (a.normalizedPhrase < b.normalizedPhrase ? -1 : a.normalizedPhrase > b.normalizedPhrase ? 1 : 0) ||
+            a.toStartVerseId - b.toStartVerseId ||
+            a.toEndVerseId - b.toEndVerseId ||
+            (a.sourceId < b.sourceId ? -1 : a.sourceId > b.sourceId ? 1 : 0));
+    }
+    /**
+     * Which of the given tokens exist in the artifact's spelling vocabulary
+     * (corpus tokens ∪ book aliases ∪ lexicon tokens ∪ translation tokens ∪
+     * Layer B verse terms).
+     * This is the OOV gate's second half: a token with corpus df 0 that is
+     * still a known name or curated word is IN vocabulary and never corrected.
+     */
+    async spellingTermsPresent(tokens) {
+        const unique = [...new Set(tokens)];
+        if (unique.length === 0)
+            return new Set();
+        const placeholders = unique.map(() => '?').join(', ');
+        const result = await this.database.execute(`SELECT term FROM spelling_terms WHERE term IN (${placeholders})`, unique);
+        return new Set(result.rows.map((row) => str(row, 'term')));
+    }
+    /**
+     * Dictionary terms whose precomputed delete variants intersect the given
+     * keys — the SymSpell candidate lookup (0.12.0/QR-5). Proposes only: every
+     * candidate is re-verified with the bounded Damerau DP before it may win
+     * (see intents/spelling.ts). ORDER BY term for a platform-stable row order,
+     * though the picker is proven row-order independent anyway.
+     */
+    async spellingCandidates(deleteKeys) {
+        const unique = [...new Set(deleteKeys)];
+        if (unique.length === 0)
+            return [];
+        const placeholders = unique.map(() => '?').join(', ');
+        const result = await this.database.execute(`SELECT DISTINCT d.term AS term, t.document_count AS documentCount
+       FROM spelling_deletes d
+       JOIN spelling_terms t ON t.term = d.term
+       WHERE d.delete_key IN (${placeholders})
+       ORDER BY d.term`, unique);
+        return result.rows.map((row) => ({
+            term: str(row, 'term'),
+            documentCount: num(row, 'documentCount'),
+        }));
+    }
 }
 /** Longest fragment length worth searching; below this, phrases are noise. */
 const MIN_FRAGMENT_WORDS = 3;
@@ -318,7 +512,8 @@ export class ConceptRepository {
               v.book_id AS bookId, b.name AS bookName,
               v.chapter AS chapter, v.verse AS verse, v.text AS text,
               a.concept_id AS conceptId, c.label AS conceptLabel,
-              a.source_id AS sourceId, a.weight AS weight, a.locator AS locator
+              a.source_id AS sourceId, a.weight AS weight, a.locator AS locator,
+              a.start_verse_id AS anchorStartVerseId, a.end_verse_id AS anchorEndVerseId
        FROM concept_anchors a
        JOIN concepts c ON c.id = a.concept_id
        JOIN verses v ON v.verse_id BETWEEN a.start_verse_id AND a.end_verse_id
@@ -333,6 +528,8 @@ export class ConceptRepository {
             sourceId: str(row, 'sourceId'),
             weight: num(row, 'weight'),
             locator: typeof row['locator'] === 'string' ? row['locator'] : null,
+            anchorStartVerseId: num(row, 'anchorStartVerseId'),
+            anchorEndVerseId: num(row, 'anchorEndVerseId'),
         }));
     }
     /**
@@ -418,6 +615,70 @@ export class ConceptRepository {
      * it is a long way from the passage saying it. G6 caps it so no volume of
      * homiletical vocabulary can outrank a curated anchor or a verbatim quote.
      */
+    /**
+     * Verses whose CROSS-TRANSLATION vocabulary matches the query.
+     *
+     * This is what lets someone search in the translation they learned a verse
+     * in. The stems here appear in some English translation of the verse but not
+     * in the one shipped, so a query using that wording reaches the verse
+     * anyway. See pipeline/src/schema.ts for what is and is not stored.
+     *
+     * Ranked by how MANY query stems a verse accounts for, then by verse id. No
+     * IDF weighting: these stems are already the residue after the shipped
+     * wording is subtracted, so a stem appearing here is by construction
+     * something the shipped text does not say.
+     */
+    async searchTranslationTokens(tokens, limit = 60) {
+        const unique = [...new Set(tokens)];
+        if (unique.length === 0)
+            return [];
+        const placeholders = unique.map(() => '?').join(', ');
+        const result = await this.database.execute(`WITH hits AS (
+         SELECT vtt.verse_id AS vid,
+                group_concat(vtt.token, ' ') AS tokens,
+                COUNT(*) AS matched
+         FROM verse_translation_tokens vtt
+         WHERE vtt.token IN (${placeholders})
+         GROUP BY vtt.verse_id
+       )
+       SELECT v.id AS id, v.verse_id AS verseId,
+              v.translation_id AS translationId, t.code AS translationCode,
+              v.book_id AS bookId, b.name AS bookName,
+              v.chapter AS chapter, v.verse AS verse, v.text AS text,
+              h.tokens AS tokens, h.matched AS matched
+       FROM hits h
+       JOIN verses v ON v.verse_id = h.vid
+       JOIN translations t ON t.id = v.translation_id
+       JOIN books b ON b.id = v.book_id
+       ORDER BY h.matched DESC, v.verse_id, t.code
+       LIMIT ?`, [...unique, limit]);
+        return result.rows.map((row) => ({
+            ...mapVerse(row),
+            matchedTokens: [...new Set(str(row, 'tokens').split(' ').filter(Boolean))].sort(),
+        }));
+    }
+    /** How many verses carry each stem — the df for weighting alternate wording. */
+    async translationTokenDocumentCounts(tokens) {
+        const unique = [...new Set(tokens)];
+        if (unique.length === 0)
+            return new Map();
+        const placeholders = unique.map(() => '?').join(', ');
+        const result = await this.database.execute(`SELECT token, COUNT(DISTINCT verse_id) AS n
+       FROM verse_translation_tokens
+       WHERE token IN (${placeholders})
+       GROUP BY token`, unique);
+        return new Map(result.rows.map((row) => [str(row, 'token'), num(row, 'n')]));
+    }
+    /** Whether the artifact carries cross-translation vocabulary at all. */
+    async hasTranslationTokens() {
+        try {
+            const result = await this.database.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'verse_translation_tokens'");
+            return result.rows.length > 0;
+        }
+        catch {
+            return false;
+        }
+    }
     async searchPassageTerms(terms, limit = 60) {
         const unique = [...new Set(terms)];
         if (unique.length === 0)
@@ -459,6 +720,76 @@ export class ConceptRepository {
         const result = await this.database.execute("SELECT COUNT(*) AS n FROM sqlite_master WHERE type='table' AND name='verse_terms'");
         return num(result.rows[0] ?? { n: 0 }, 'n') > 0;
     }
+    /**
+     * Whether this artifact carries any curated phrase/hymn aliases
+     * (0.13.0/QR-6). Presence-AND-ROWS probed, deliberately stricter than the
+     * other layer probes: schema v7 ships the table EMPTY (QR-5), and an
+     * engine that ran the alias step against an empty table would pay a query
+     * per research() call for nothing — and, more importantly, the rollback
+     * story is "rebuild without alias rows", which must restore pre-QR-6
+     * behavior exactly. No table, or an empty one, and 0.13.0 behaves as
+     * 0.12.0 did.
+     */
+    async hasCuratedAliases() {
+        try {
+            const table = await this.database.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'curated_aliases'");
+            if (table.rows.length === 0)
+                return false;
+            const rows = await this.database.execute('SELECT 1 AS present FROM curated_aliases LIMIT 1');
+            return rows.rows.length > 0;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * The curated aliases whose whole-query key equals the given normalized
+     * phrase. EQUALITY, never containment — the line that keeps a curated
+     * phrase table from becoming a hidden second ranking system; brittleness
+     * to extra words is accepted BY DESIGN. `normalized_raw` is UNIQUE, so
+     * this returns at most one row; it is typed as a list so the caller does
+     * not encode that schema fact.
+     */
+    async matchAliases(normalizedQuery) {
+        if (!normalizedQuery)
+            return [];
+        const result = await this.database.execute(`SELECT a.id AS id, a.title AS title, a.concept_id AS conceptId,
+              c.label AS conceptLabel,
+              a.start_verse_id AS startVerseId, a.end_verse_id AS endVerseId,
+              a.source_id AS sourceId, a.weight AS weight, a.locator AS locator
+       FROM curated_aliases a
+       LEFT JOIN concepts c ON c.id = a.concept_id
+       WHERE a.normalized_raw = ?
+       ORDER BY a.id`, [normalizedQuery]);
+        return result.rows.map((row) => ({
+            id: num(row, 'id'),
+            title: str(row, 'title'),
+            conceptId: typeof row['conceptId'] === 'string' ? row['conceptId'] : null,
+            conceptLabel: typeof row['conceptLabel'] === 'string' ? row['conceptLabel'] : null,
+            startVerseId: typeof row['startVerseId'] === 'number' ? row['startVerseId'] : null,
+            endVerseId: typeof row['endVerseId'] === 'number' ? row['endVerseId'] : null,
+            sourceId: str(row, 'sourceId'),
+            weight: num(row, 'weight'),
+            locator: typeof row['locator'] === 'string' ? row['locator'] : null,
+        }));
+    }
+    /**
+     * Verses of an explicit alias verse range (the XOR's other arm). A range
+     * absent from this corpus returns no rows — the alias then contributes
+     * nothing, honestly, rather than being guessed at.
+     */
+    async aliasRangeVerses(startVerseId, endVerseId) {
+        const result = await this.database.execute(`SELECT v.id AS id, v.verse_id AS verseId,
+              v.translation_id AS translationId, t.code AS translationCode,
+              v.book_id AS bookId, b.name AS bookName,
+              v.chapter AS chapter, v.verse AS verse, v.text AS text
+       FROM verses v
+       JOIN translations t ON t.id = v.translation_id
+       JOIN books b ON b.id = v.book_id
+       WHERE v.verse_id BETWEEN ? AND ?
+       ORDER BY v.verse_id, t.code`, [startVerseId, endVerseId]);
+        return result.rows.map(mapVerse);
+    }
     async hasConceptLayer() {
         const result = await this.database.execute("SELECT COUNT(*) AS n FROM sqlite_master WHERE type='table' AND name='concepts'");
         return num(result.rows[0] ?? { n: 0 }, 'n') > 0;

package/dist/createEngine.d.ts

@@ -10,8 +10,9 @@
  * Curated expansion (concepts, cross-references) attaches at step 5 in Phase
  * 2 without changing anything above it.
  */
+import { type PericopeRow } from './corpus/repository.js';
 import { type RankOptions } from './ranking/rank.js';
-import type { ConceptMatch, ContentQueryPort, PassageResult, RelatedResult, ResearchResult, SongInput } from './types.js';
+import type { ConceptMatch, ContentQueryPort, DiscoveryResult, PassageResult, RelatedResult, ResearchResult, ScriptureVerse, SongInput } from './types.js';
 export interface EngineOptions {
     /**
      * Throw if the artifact was tokenized by a different tokenizer version.
@@ -56,3 +57,77 @@ export interface ScriptureEngine {
     readonly engineVersion: string;
 }
 export declare function createEngine(database: ContentQueryPort, options?: EngineOptions): Promise<ScriptureEngine>;
+/** A grouping span's own extent and the source(s) that named it. */
+export interface GroupingSpanInfo {
+    readonly startVerseId: number;
+    readonly endVerseId: number;
+    readonly sourceIds: ReadonlySet<string>;
+}
+/**
+ * Collapse the surfaced verses of ONE grouping unit into a single
+ * passage-level row. Two mechanisms feed it, in FIXED authority order:
+ *
+ * 1. Curated anchor spans (0.10.0 stage 7 semantics, unchanged: span
+ *    MEMBERSHIP, not rank adjacency) — a passage a human named for a theme.
+ *    Checked first: a verse any anchor span claims belongs to the anchor
+ *    path and is never considered for pericope runs, so pericope provenance
+ *    can never usurp anchor provenance (the anchor-grouping-explained
+ *    fixture pins this).
+ * 2. Derived pericopes (0.14.0/CO-3 PR 2) — a structural sectioning fact
+ *    (OpenBible section counts). Deliberately MORE conservative than the
+ *    anchor path, because nobody named THIS passage for THIS query: members
+ *    must be consecutive in rank AND verseId-consecutive AND share one
+ *    pericope. A boundary is never crossed (the Terah fixture pins this),
+ *    and ±1 verse-id adjacency structurally cannot cross a chapter — the
+ *    documented v1 limitation that grouping never crosses a chapter even
+ *    when a pericope does.
+ *
+ * Since 0.14.0 a merged row also SAYS why its verses travel together: it
+ * carries `verses[]` (each member's own evidence, uncollapsed) and a typed
+ * `grouping` naming the section span and the source that drew it — for
+ * anchor runs the anchor's own source(s), for pericope runs
+ * 'openbible-sections' plus the summed boundary vote at the section's start
+ * verse, read from the same artifact row the derivation stored. Grouping
+ * contributes ZERO points: the merged score is the max of the members,
+ * never a sum — a passage must not outrank by having more mediocre verses —
+ * and the row's `reference` spans the HITS, never the whole section.
+ *
+ * Why this exists: a ranged anchor emits one candidate per verse, and results
+ * carrying authoritative evidence are deliberately exempt from group
+ * diversification — a genuine multi-verse hit must never be thinned for the
+ * sake of variety. Correct for exact-phrase matches; wrong for a curated span,
+ * where the results ARE one passage. `communion` returned 1 Corinthians
+ * 11:23, :24, :25 and :26 at identical scores, spending the whole top of the
+ * list on a passage a human had already grouped.
+ *
+ * Until 0.10.0 this required RANK adjacency, which made the collapse depend
+ * on what happened to rank in between: `praise` filled five slots with
+ * individual verses of Psalm 150 because they ranked non-adjacently, and one
+ * differently-scored verse of a span broke the whole merge. The span is the
+ * unit because a person chose it — whether its verses rank consecutively is
+ * an accident of the other evidence. Now every surfaced member of a span
+ * merges into one row at the position of its best-ranked member; the members
+ * below drop and the results shift up. That is the point: the passage
+ * occupies one slot, not N.
+ *
+ * Determinism: pass 1 assigns each result a governing span — the span, among
+ * the spans it belongs to, covering the most surfaced results (ties broken by
+ * span key ascending) — and pass 2 emits in rank order, so the output is a
+ * pure function of the ranker's total order and data already in the artifact.
+ * Nothing is inferred, and equal inputs collapse identically on every
+ * platform.
+ *
+ * The merged row is honest about what surfaced: its reference spans the
+ * surfaced members (canonical min..max), the excerpt is their texts in
+ * canonical verse order, the score is the best member's (existing policy),
+ * and reasons merge strongest-per-label so the chips explain the passage
+ * rather than an arbitrary one of its verses.
+ */
+export declare function collapseRuns(results: readonly DiscoveryResult[], verses: ReadonlyMap<string, ScriptureVerse>, anchorSpans: ReadonlyMap<string, ReadonlySet<string>>, spanInfo: ReadonlyMap<string, GroupingSpanInfo>, pericopeOf: ReadonlyMap<number, PericopeRow>): readonly DiscoveryResult[];
+/**
+ * The 0.13.0 entry point, kept for compatibility: anchor-span collapse only,
+ * no pericope runs, merged rows in their exact pre-0.14.0 shape (no
+ * `verses`/`grouping` — those need the span provenance the 5-argument
+ * `collapseRuns` receives). `discover()` no longer calls this.
+ */
+export declare function collapseAnchorRuns(results: readonly DiscoveryResult[], verses: ReadonlyMap<string, ScriptureVerse>, anchorSpans: ReadonlyMap<string, ReadonlySet<string>>): readonly DiscoveryResult[];