@jestek-dev/scripture-engine 0.7.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +35 -0
- package/dist/config/engineVersion.d.ts +1 -1
- package/dist/config/engineVersion.js +47 -1
- package/dist/corpus/repository.d.ts +189 -2
- package/dist/corpus/repository.js +332 -1
- package/dist/createEngine.d.ts +76 -1
- package/dist/createEngine.js +716 -52
- package/dist/index.d.ts +20 -14
- package/dist/index.js +15 -13
- package/dist/intents/concept.d.ts +115 -2
- package/dist/intents/concept.js +323 -6
- package/dist/intents/lexical.d.ts +42 -3
- package/dist/intents/lexical.js +111 -11
- package/dist/intents/spelling.d.ts +93 -0
- package/dist/intents/spelling.js +124 -0
- package/dist/internal.d.ts +31 -0
- package/dist/internal.js +31 -0
- package/dist/ranking/budgets.d.ts +12 -0
- package/dist/ranking/budgets.js +39 -0
- package/dist/ranking/rank.js +20 -3
- package/dist/reasons/display.d.ts +86 -0
- package/dist/reasons/display.js +105 -0
- package/dist/reasons/types.d.ts +1 -1
- package/dist/reference/reference.d.ts +57 -0
- package/dist/reference/reference.js +210 -48
- package/dist/tokenizer/index.d.ts +35 -0
- package/dist/tokenizer/index.js +37 -1
- package/dist/types.d.ts +104 -0
- package/package.json +16 -1
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chip display polish (0.10.0, CO-2/F22).
|
|
3
|
+
*
|
|
4
|
+
* A result's ordering is decided by applyBudgets and rank(); its CHIPS are
|
|
5
|
+
* what a person reads. These two concerns diverge at the margin: a chip whose
|
|
6
|
+
* points are real but tiny still counts toward the score, yet displaying it
|
|
7
|
+
* claims an explanatory weight the number itself denies. This module is the
|
|
8
|
+
* display seam — it never changes points, scores, or order; it only decides
|
|
9
|
+
* which chips are worth a reader's attention. Both constants are reviewed
|
|
10
|
+
* data, mirrored into `eval/budgets.json` signalBudgets for the G6
|
|
11
|
+
* reviewed-constants check.
|
|
12
|
+
*/
|
|
13
|
+
import type { Reason } from './types.js';
|
|
14
|
+
import type { SpellingCorrection } from '../types.js';
|
|
15
|
+
/**
|
|
16
|
+
* Chips display their points at one decimal. Below this value a chip prints
|
|
17
|
+
* "0.0" — asserting a contribution the display itself denies. Fixed by that
|
|
18
|
+
* display precision (0.05 is the smallest value that rounds to 0.1), not a
|
|
19
|
+
* tuning knob. Measured at introduction (84-query battery, full 25-deep
|
|
20
|
+
* windows, full reconstructed corpus): no current chip is below it — the
|
|
21
|
+
* rule is a structural guard on the display contract, not a tuned
|
|
22
|
+
* suppressor.
|
|
23
|
+
*/
|
|
24
|
+
export declare const CHIP_DISPLAY_MIN_POINTS = 0.05;
|
|
25
|
+
/**
|
|
26
|
+
* Display floor for passage_terms chips (the homiletical-vocabulary hint).
|
|
27
|
+
*
|
|
28
|
+
* Derived from reviewed data, not tuned: the weakest evidence G5 can admit —
|
|
29
|
+
* a single term at the admission floor (`eval/budgets.json`
|
|
30
|
+
* distinctiveness.minPmi = 2.0) distilled from a one-verse note — earns
|
|
31
|
+
* 8 × log1p(1)/log1p(6) × 2/(2+6) ≈ 0.712 points, and the floor sits just
|
|
32
|
+
* beneath it. Everything that clears admission undiluted still displays;
|
|
33
|
+
* the same evidence diluted below that line (a floor-grade term inherited
|
|
34
|
+
* from a whole-chapter essay, or scaled down by the aggregate caps) is
|
|
35
|
+
* withheld as a chip while its points still count. Measured at introduction:
|
|
36
|
+
* the weakest passage_terms chip anywhere in the battery's full windows is
|
|
37
|
+
* 0.896 — nothing currently withheld; the floor exists so future admitted
|
|
38
|
+
* data cannot decorate results with sub-admission-grade chips.
|
|
39
|
+
*/
|
|
40
|
+
export declare const PASSAGE_TERM_CHIP_DISPLAY_FLOOR = 0.7;
|
|
41
|
+
/**
|
|
42
|
+
* Withhold chips that fail the display rules. Pure and order-preserving;
|
|
43
|
+
* points, scores and result order are untouched by construction — callers
|
|
44
|
+
* apply this AFTER ranking and collapsing, to the reasons of final results.
|
|
45
|
+
*
|
|
46
|
+
* Covenant guard: explanations are part of the contract, so a result is
|
|
47
|
+
* never stripped of its last chip — when every chip fails the rules, the
|
|
48
|
+
* strongest one stays, honestly showing how little the result rests on.
|
|
49
|
+
*
|
|
50
|
+
* Returns the input array unchanged (same reference) when nothing is
|
|
51
|
+
* withheld, so untouched results stay byte-identical.
|
|
52
|
+
*/
|
|
53
|
+
/**
|
|
54
|
+
* The correction citation a chip label carries for one corrected token —
|
|
55
|
+
* shared between tokenEvidence's decoration and the display-level pin below,
|
|
56
|
+
* so "is this correction visibly cited?" is checked against the exact string
|
|
57
|
+
* that renders it.
|
|
58
|
+
*/
|
|
59
|
+
export declare function correctionCitation(typed: string): string;
|
|
60
|
+
/**
|
|
61
|
+
* Guarantee every correction is VISIBLY cited on a result of a corrected
|
|
62
|
+
* query (0.12.0/QR-5 round-2, J31: "every correction shown"; covenant 5:
|
|
63
|
+
* explanations are the contract).
|
|
64
|
+
*
|
|
65
|
+
* The token-chip decoration (`Shared word: hell (corrected from "hello")`)
|
|
66
|
+
* only exists on results whose evidence includes the corrected token's
|
|
67
|
+
* token_overlap chip. Exactly the harm-class corrections tend to surface
|
|
68
|
+
* results through concept/passage evidence instead — `hello` → "hell" ranks
|
|
69
|
+
* pages of `Theme: Hell` rows with no visible trace that the query was
|
|
70
|
+
* rewritten. A citation the user cannot see is not a citation, so this pin
|
|
71
|
+
* runs LAST (after polish, like the last-chip rule) and decorates the
|
|
72
|
+
* strongest chip of any result whose displayed chips do not already carry
|
|
73
|
+
* every correction: `Theme: Hell (query corrected from "hello")`.
|
|
74
|
+
*
|
|
75
|
+
* The wording is query-level on purpose: on a mixed query ("gods forgivness")
|
|
76
|
+
* a result may rank on the UNcorrected tokens alone, so claiming the result
|
|
77
|
+
* matched via the correction would be false — what is always true, for every
|
|
78
|
+
* result of the response, is that the QUERY was corrected. Display-only by
|
|
79
|
+
* construction: points, scores, order and the page are already decided;
|
|
80
|
+
* labels change, families and points never do.
|
|
81
|
+
*
|
|
82
|
+
* Returns the input array unchanged (same reference) when every correction
|
|
83
|
+
* is already visible, so untouched results stay byte-identical.
|
|
84
|
+
*/
|
|
85
|
+
export declare function pinCorrectionCitations(reasons: readonly Reason[], corrections: readonly SpellingCorrection[]): readonly Reason[];
|
|
86
|
+
export declare function polishChipsForDisplay(reasons: readonly Reason[]): readonly Reason[];
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chip display polish (0.10.0, CO-2/F22).
|
|
3
|
+
*
|
|
4
|
+
* A result's ordering is decided by applyBudgets and rank(); its CHIPS are
|
|
5
|
+
* what a person reads. These two concerns diverge at the margin: a chip whose
|
|
6
|
+
* points are real but tiny still counts toward the score, yet displaying it
|
|
7
|
+
* claims an explanatory weight the number itself denies. This module is the
|
|
8
|
+
* display seam — it never changes points, scores, or order; it only decides
|
|
9
|
+
* which chips are worth a reader's attention. Both constants are reviewed
|
|
10
|
+
* data, mirrored into `eval/budgets.json` signalBudgets for the G6
|
|
11
|
+
* reviewed-constants check.
|
|
12
|
+
*/
|
|
13
|
+
/**
|
|
14
|
+
* Chips display their points at one decimal. Below this value a chip prints
|
|
15
|
+
* "0.0" — asserting a contribution the display itself denies. Fixed by that
|
|
16
|
+
* display precision (0.05 is the smallest value that rounds to 0.1), not a
|
|
17
|
+
* tuning knob. Measured at introduction (84-query battery, full 25-deep
|
|
18
|
+
* windows, full reconstructed corpus): no current chip is below it — the
|
|
19
|
+
* rule is a structural guard on the display contract, not a tuned
|
|
20
|
+
* suppressor.
|
|
21
|
+
*/
|
|
22
|
+
export const CHIP_DISPLAY_MIN_POINTS = 0.05;
|
|
23
|
+
/**
|
|
24
|
+
* Display floor for passage_terms chips (the homiletical-vocabulary hint).
|
|
25
|
+
*
|
|
26
|
+
* Derived from reviewed data, not tuned: the weakest evidence G5 can admit —
|
|
27
|
+
* a single term at the admission floor (`eval/budgets.json`
|
|
28
|
+
* distinctiveness.minPmi = 2.0) distilled from a one-verse note — earns
|
|
29
|
+
* 8 × log1p(1)/log1p(6) × 2/(2+6) ≈ 0.712 points, and the floor sits just
|
|
30
|
+
* beneath it. Everything that clears admission undiluted still displays;
|
|
31
|
+
* the same evidence diluted below that line (a floor-grade term inherited
|
|
32
|
+
* from a whole-chapter essay, or scaled down by the aggregate caps) is
|
|
33
|
+
* withheld as a chip while its points still count. Measured at introduction:
|
|
34
|
+
* the weakest passage_terms chip anywhere in the battery's full windows is
|
|
35
|
+
* 0.896 — nothing currently withheld; the floor exists so future admitted
|
|
36
|
+
* data cannot decorate results with sub-admission-grade chips.
|
|
37
|
+
*/
|
|
38
|
+
export const PASSAGE_TERM_CHIP_DISPLAY_FLOOR = 0.7;
|
|
39
|
+
/**
|
|
40
|
+
* Withhold chips that fail the display rules. Pure and order-preserving;
|
|
41
|
+
* points, scores and result order are untouched by construction — callers
|
|
42
|
+
* apply this AFTER ranking and collapsing, to the reasons of final results.
|
|
43
|
+
*
|
|
44
|
+
* Covenant guard: explanations are part of the contract, so a result is
|
|
45
|
+
* never stripped of its last chip — when every chip fails the rules, the
|
|
46
|
+
* strongest one stays, honestly showing how little the result rests on.
|
|
47
|
+
*
|
|
48
|
+
* Returns the input array unchanged (same reference) when nothing is
|
|
49
|
+
* withheld, so untouched results stay byte-identical.
|
|
50
|
+
*/
|
|
51
|
+
/**
|
|
52
|
+
* The correction citation a chip label carries for one corrected token —
|
|
53
|
+
* shared between tokenEvidence's decoration and the display-level pin below,
|
|
54
|
+
* so "is this correction visibly cited?" is checked against the exact string
|
|
55
|
+
* that renders it.
|
|
56
|
+
*/
|
|
57
|
+
export function correctionCitation(typed) {
|
|
58
|
+
return `corrected from "${typed}"`;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Guarantee every correction is VISIBLY cited on a result of a corrected
|
|
62
|
+
* query (0.12.0/QR-5 round-2, J31: "every correction shown"; covenant 5:
|
|
63
|
+
* explanations are the contract).
|
|
64
|
+
*
|
|
65
|
+
* The token-chip decoration (`Shared word: hell (corrected from "hello")`)
|
|
66
|
+
* only exists on results whose evidence includes the corrected token's
|
|
67
|
+
* token_overlap chip. Exactly the harm-class corrections tend to surface
|
|
68
|
+
* results through concept/passage evidence instead — `hello` → "hell" ranks
|
|
69
|
+
* pages of `Theme: Hell` rows with no visible trace that the query was
|
|
70
|
+
* rewritten. A citation the user cannot see is not a citation, so this pin
|
|
71
|
+
* runs LAST (after polish, like the last-chip rule) and decorates the
|
|
72
|
+
* strongest chip of any result whose displayed chips do not already carry
|
|
73
|
+
* every correction: `Theme: Hell (query corrected from "hello")`.
|
|
74
|
+
*
|
|
75
|
+
* The wording is query-level on purpose: on a mixed query ("gods forgivness")
|
|
76
|
+
* a result may rank on the UNcorrected tokens alone, so claiming the result
|
|
77
|
+
* matched via the correction would be false — what is always true, for every
|
|
78
|
+
* result of the response, is that the QUERY was corrected. Display-only by
|
|
79
|
+
* construction: points, scores, order and the page are already decided;
|
|
80
|
+
* labels change, families and points never do.
|
|
81
|
+
*
|
|
82
|
+
* Returns the input array unchanged (same reference) when every correction
|
|
83
|
+
* is already visible, so untouched results stay byte-identical.
|
|
84
|
+
*/
|
|
85
|
+
export function pinCorrectionCitations(reasons, corrections) {
|
|
86
|
+
if (reasons.length === 0 || corrections.length === 0)
|
|
87
|
+
return reasons;
|
|
88
|
+
const missing = corrections.filter((correction) => !reasons.some((reason) => reason.label.includes(correctionCitation(correction.typed))));
|
|
89
|
+
if (missing.length === 0)
|
|
90
|
+
return reasons;
|
|
91
|
+
const cited = missing.map((correction) => `"${correction.typed}"`).join(', ');
|
|
92
|
+
const [strongest, ...rest] = reasons;
|
|
93
|
+
return [{ ...strongest, label: `${strongest.label} (query corrected from ${cited})` }, ...rest];
|
|
94
|
+
}
|
|
95
|
+
export function polishChipsForDisplay(reasons) {
|
|
96
|
+
if (reasons.length === 0)
|
|
97
|
+
return reasons;
|
|
98
|
+
const kept = reasons.filter((reason) => reason.points >= CHIP_DISPLAY_MIN_POINTS &&
|
|
99
|
+
(reason.family !== 'passage_terms' || reason.points >= PASSAGE_TERM_CHIP_DISPLAY_FLOOR));
|
|
100
|
+
if (kept.length === reasons.length)
|
|
101
|
+
return reasons;
|
|
102
|
+
// reasons arrive strongest-first from applyBudgets / the collapse merge,
|
|
103
|
+
// so [0] is the strongest chip.
|
|
104
|
+
return kept.length > 0 ? kept : [reasons[0]];
|
|
105
|
+
}
|
package/dist/reasons/types.d.ts
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* individually and in aggregate, which is what structurally prevents a large
|
|
17
17
|
* new dataset from outranking an exact match (guardrail G6).
|
|
18
18
|
*/
|
|
19
|
-
export type SignalFamily = 'reference' | 'exact_phrase' | 'concept_anchor' | 'concept_lexicon' | 'token_overlap' | 'proximity' | 'passage_terms' | 'cross_reference' | 'co_citation';
|
|
19
|
+
export type SignalFamily = 'reference' | 'exact_phrase' | 'concept_anchor' | 'concept_lexicon' | 'token_overlap' | 'translation_variant' | 'proximity' | 'passage_terms' | 'cross_reference' | 'co_citation';
|
|
20
20
|
export declare const AUTHORITATIVE_FAMILIES: readonly SignalFamily[];
|
|
21
21
|
export declare function isAuthoritative(family: SignalFamily): boolean;
|
|
22
22
|
/**
|
|
@@ -13,19 +13,76 @@ export interface ResolvedReference {
|
|
|
13
13
|
readonly endId: number;
|
|
14
14
|
readonly label: string;
|
|
15
15
|
}
|
|
16
|
+
/**
|
|
17
|
+
* One shipped book-alias row with its canonical book, for the did-you-mean
|
|
18
|
+
* (0.11.0/QR-4). Read through the port exactly once per engine instance and
|
|
19
|
+
* cached by the implementation — the engine does no I/O of its own.
|
|
20
|
+
*/
|
|
21
|
+
export interface BookAliasEntry {
|
|
22
|
+
readonly aliasKey: string;
|
|
23
|
+
readonly bookId: number;
|
|
24
|
+
readonly bookName: string;
|
|
25
|
+
readonly chapterCount: number;
|
|
26
|
+
}
|
|
16
27
|
export interface ReferenceResolver {
|
|
17
28
|
resolveBookAlias(aliasKey: string): Promise<ResolvedBook | null>;
|
|
18
29
|
getChapterVerseCount(bookId: number, chapter: number): Promise<number | null>;
|
|
19
30
|
verseExists(bookId: number, chapter: number, verse: number): Promise<boolean>;
|
|
31
|
+
/** Every shipped alias key with its canonical book — the did-you-mean vocabulary. */
|
|
32
|
+
listBookAliases(): Promise<readonly BookAliasEntry[]>;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* A cited did-you-mean on an invalid-reference dead end (0.11.0/QR-4).
|
|
36
|
+
*
|
|
37
|
+
* Suggestion only, NEVER an auto-resolution: curated aliases (human-reviewed)
|
|
38
|
+
* auto-resolve, edit-distance matches only suggest. A machine-guessed book
|
|
39
|
+
* handing reference-level authority to the wrong passage is the one place a
|
|
40
|
+
* spelling error could do maximal theological harm, so the guess is surfaced
|
|
41
|
+
* as a question, with its edit distance as the citation. (J34/J35.)
|
|
42
|
+
*/
|
|
43
|
+
export interface ReferenceSuggestion {
|
|
44
|
+
/** Canonical name of the unique in-policy book match. */
|
|
45
|
+
readonly book: string;
|
|
46
|
+
/** The full suggested reference label, validated to exist in the corpus. */
|
|
47
|
+
readonly reference: string;
|
|
48
|
+
/** Integer Damerau–Levenshtein distance from the typed book text — the citation. */
|
|
49
|
+
readonly distance: number;
|
|
20
50
|
}
|
|
21
51
|
export type ReferenceResolutionAttempt = {
|
|
22
52
|
readonly kind: 'not-reference';
|
|
23
53
|
} | {
|
|
24
54
|
readonly kind: 'invalid-reference';
|
|
55
|
+
/** Present when a unique in-policy near-miss book validated. See ReferenceSuggestion. */
|
|
56
|
+
readonly suggestion?: ReferenceSuggestion;
|
|
57
|
+
/**
|
|
58
|
+
* True for bare-number shapes whose book never resolved and earned no
|
|
59
|
+
* suggestion (J36): "plans 29 11" is a memory query, not a committed
|
|
60
|
+
* reference, and dead-ending it serves nobody. Explicit-separator
|
|
61
|
+
* locators (a colon or dot) state reference intent and stay
|
|
62
|
+
* invalid-reference. Callers with a discovery path honor this flag;
|
|
63
|
+
* lookups (passage(), related()) have nothing to fall through to.
|
|
64
|
+
*/
|
|
65
|
+
readonly fallthroughToDiscovery: boolean;
|
|
25
66
|
} | {
|
|
26
67
|
readonly kind: 'resolved';
|
|
27
68
|
readonly reference: ResolvedReference;
|
|
28
69
|
};
|
|
70
|
+
/**
|
|
71
|
+
* The ONE edit-policy table (Phase 5 design invariant), keyed by normalized
|
|
72
|
+
* key length: <5 → never suggest; 5–8 → edit distance 1; ≥9 → edit distance 2.
|
|
73
|
+
* A transposition counts as one edit (Damerau). QR-5's token correction
|
|
74
|
+
* mirrors these numbers; eval cross-checks the two stay equal. (J31/J35.)
|
|
75
|
+
*/
|
|
76
|
+
export declare const SUGGESTION_MIN_KEY_LENGTH = 5;
|
|
77
|
+
export declare const SUGGESTION_EDIT1_MAX_KEY_LENGTH = 8;
|
|
78
|
+
export declare function editDistanceBudget(keyLength: number): number;
|
|
79
|
+
/**
|
|
80
|
+
* Bounded integer Damerau–Levenshtein (optimal string alignment): unit-cost
|
|
81
|
+
* insert/delete/substitute plus adjacent transposition at cost 1. Pure
|
|
82
|
+
* integer DP — no floats in decisions — and bounded: returns null when the
|
|
83
|
+
* distance exceeds `bound`, so callers never rank on an out-of-policy guess.
|
|
84
|
+
*/
|
|
85
|
+
export declare function damerauLevenshtein(a: string, b: string, bound: number): number | null;
|
|
29
86
|
export declare function normalizeBookAlias(input: string): string;
|
|
30
87
|
export declare function resolveReferenceAttempt(input: string, resolver: ReferenceResolver): Promise<ReferenceResolutionAttempt>;
|
|
31
88
|
export declare function resolveReference(input: string, resolver: ReferenceResolver): Promise<ResolvedReference | null>;
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
const COMPACT_DOT_RE = /^([1-3]?\s?[A-Za-z]+)\.(\d{1,3})\.(\d{1,3})$/;
|
|
2
2
|
const BOOK_LOCATOR_RE = /^(.*?)\s*\.?\s*(\d{1,3}(?:\s*[:.]\s*\d{1,3})?(?:\s*-\s*(?:\d{1,3}\s*[:.]\s*)?\d{1,3})?)$/;
|
|
3
3
|
const LOCATOR_RE = /^(\d{1,3})(?:\s*[:.]\s*(\d{1,3}))?(?:\s*-\s*(?:(\d{1,3})\s*[:.]\s*)?(\d{1,3}))?$/;
|
|
4
|
+
/**
|
|
5
|
+
* The space-separated chapter/verse form (0.11.0/QR-4): "John 3 16",
|
|
6
|
+
* "1 corinthians 13 4", "John 3 1-5" — the most common phone-typed shape.
|
|
7
|
+
* The book text must end in a non-digit so an all-numeric query ("3 16")
|
|
8
|
+
* never manufactures a book candidate out of its own leading number.
|
|
9
|
+
*/
|
|
10
|
+
const SPACE_LOCATOR_RE = /^(.*?[^\s\d])\s+(\d{1,3})\s+(\d{1,3})(?:\s*-\s*(\d{1,3}))?$/;
|
|
4
11
|
const DASH_CHARS_RE = /[‒–—―−]/g;
|
|
5
12
|
const ROMAN_PREFIX_RE = /^(iii|ii|i)(?=\s|$)/;
|
|
6
13
|
const ROMAN_TO_ARABIC = {
|
|
@@ -8,6 +15,52 @@ const ROMAN_TO_ARABIC = {
|
|
|
8
15
|
ii: '2',
|
|
9
16
|
iii: '3',
|
|
10
17
|
};
|
|
18
|
+
/**
|
|
19
|
+
* The ONE edit-policy table (Phase 5 design invariant), keyed by normalized
|
|
20
|
+
* key length: <5 → never suggest; 5–8 → edit distance 1; ≥9 → edit distance 2.
|
|
21
|
+
* A transposition counts as one edit (Damerau). QR-5's token correction
|
|
22
|
+
* mirrors these numbers; eval cross-checks the two stay equal. (J31/J35.)
|
|
23
|
+
*/
|
|
24
|
+
export const SUGGESTION_MIN_KEY_LENGTH = 5;
|
|
25
|
+
export const SUGGESTION_EDIT1_MAX_KEY_LENGTH = 8;
|
|
26
|
+
export function editDistanceBudget(keyLength) {
|
|
27
|
+
if (keyLength < SUGGESTION_MIN_KEY_LENGTH)
|
|
28
|
+
return 0;
|
|
29
|
+
return keyLength <= SUGGESTION_EDIT1_MAX_KEY_LENGTH ? 1 : 2;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Bounded integer Damerau–Levenshtein (optimal string alignment): unit-cost
|
|
33
|
+
* insert/delete/substitute plus adjacent transposition at cost 1. Pure
|
|
34
|
+
* integer DP — no floats in decisions — and bounded: returns null when the
|
|
35
|
+
* distance exceeds `bound`, so callers never rank on an out-of-policy guess.
|
|
36
|
+
*/
|
|
37
|
+
export function damerauLevenshtein(a, b, bound) {
|
|
38
|
+
if (a === b)
|
|
39
|
+
return 0;
|
|
40
|
+
if (Math.abs(a.length - b.length) > bound)
|
|
41
|
+
return null;
|
|
42
|
+
const rows = a.length + 1;
|
|
43
|
+
const cols = b.length + 1;
|
|
44
|
+
const d = [];
|
|
45
|
+
for (let i = 0; i < rows; i += 1) {
|
|
46
|
+
d.push(new Array(cols).fill(0));
|
|
47
|
+
d[i][0] = i;
|
|
48
|
+
}
|
|
49
|
+
for (let j = 0; j < cols; j += 1)
|
|
50
|
+
d[0][j] = j;
|
|
51
|
+
for (let i = 1; i < rows; i += 1) {
|
|
52
|
+
for (let j = 1; j < cols; j += 1) {
|
|
53
|
+
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
54
|
+
let value = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + cost);
|
|
55
|
+
if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) {
|
|
56
|
+
value = Math.min(value, d[i - 2][j - 2] + 1);
|
|
57
|
+
}
|
|
58
|
+
d[i][j] = value;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
const distance = d[a.length][b.length];
|
|
62
|
+
return distance <= bound ? distance : null;
|
|
63
|
+
}
|
|
11
64
|
export function normalizeBookAlias(input) {
|
|
12
65
|
return input
|
|
13
66
|
.trim()
|
|
@@ -15,43 +68,71 @@ export function normalizeBookAlias(input) {
|
|
|
15
68
|
.replace(ROMAN_PREFIX_RE, (value) => ROMAN_TO_ARABIC[value])
|
|
16
69
|
.replace(/[^a-z0-9]/g, '');
|
|
17
70
|
}
|
|
18
|
-
function
|
|
71
|
+
function syntaxFromLocator(bookText, locator) {
|
|
72
|
+
return {
|
|
73
|
+
bookText,
|
|
74
|
+
startNumber: Number(locator[1]),
|
|
75
|
+
startVerse: locator[2] === undefined ? null : Number(locator[2]),
|
|
76
|
+
endChapter: locator[3] === undefined ? null : Number(locator[3]),
|
|
77
|
+
endNumber: locator[4] === undefined ? null : Number(locator[4]),
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
function parseCandidates(input) {
|
|
19
81
|
const trimmed = input.trim().replace(DASH_CHARS_RE, '-');
|
|
20
82
|
if (!trimmed)
|
|
21
83
|
return null;
|
|
22
84
|
const compact = COMPACT_DOT_RE.exec(trimmed);
|
|
23
85
|
if (compact) {
|
|
24
86
|
return {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
87
|
+
candidates: [
|
|
88
|
+
{
|
|
89
|
+
bookText: compact[1],
|
|
90
|
+
startNumber: Number(compact[2]),
|
|
91
|
+
startVerse: Number(compact[3]),
|
|
92
|
+
endChapter: null,
|
|
93
|
+
endNumber: null,
|
|
94
|
+
},
|
|
95
|
+
],
|
|
96
|
+
explicitSeparator: true,
|
|
30
97
|
};
|
|
31
98
|
}
|
|
99
|
+
const candidates = [];
|
|
100
|
+
let explicitSeparator = false;
|
|
32
101
|
const split = BOOK_LOCATOR_RE.exec(trimmed);
|
|
33
|
-
if (
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
102
|
+
if (split && split[1]?.trim()) {
|
|
103
|
+
const locator = LOCATOR_RE.exec(split[2]);
|
|
104
|
+
if (locator) {
|
|
105
|
+
candidates.push(syntaxFromLocator(split[1], locator));
|
|
106
|
+
explicitSeparator = /[:.]/.test(split[2]);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
const spaced = SPACE_LOCATOR_RE.exec(trimmed);
|
|
110
|
+
if (spaced) {
|
|
111
|
+
candidates.push({
|
|
112
|
+
bookText: spaced[1],
|
|
113
|
+
startNumber: Number(spaced[2]),
|
|
114
|
+
startVerse: Number(spaced[3]),
|
|
115
|
+
endChapter: null,
|
|
116
|
+
endNumber: spaced[4] === undefined ? null : Number(spaced[4]),
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
if (candidates.length === 0)
|
|
37
120
|
return null;
|
|
38
|
-
return {
|
|
39
|
-
bookText: split[1],
|
|
40
|
-
startNumber: Number(locator[1]),
|
|
41
|
-
startVerse: locator[2] === undefined ? null : Number(locator[2]),
|
|
42
|
-
endChapter: locator[3] === undefined ? null : Number(locator[3]),
|
|
43
|
-
endNumber: locator[4] === undefined ? null : Number(locator[4]),
|
|
44
|
-
};
|
|
121
|
+
return { candidates, explicitSeparator };
|
|
45
122
|
}
|
|
46
123
|
function verseId(bookId, chapter, verse) {
|
|
47
124
|
return bookId * 1_000_000 + chapter * 1_000 + verse;
|
|
48
125
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
126
|
+
/**
|
|
127
|
+
* Renders the label — the explanation surface of a resolved reference
|
|
128
|
+
* (working agreement 5). `wholeChapter` is threaded explicitly from the parse
|
|
129
|
+
* branch that knows it (0.11.0/QR-4): until then the book-and-chapter form
|
|
130
|
+
* was keyed on "starts at verse 1", which mislabeled "John 3:1-5" as
|
|
131
|
+
* "John 3" — the right passage under the wrong name, a contract failure.
|
|
132
|
+
*/
|
|
133
|
+
function labelFor(book, startChapter, startVerse, endChapter, endVerse, wholeChapter) {
|
|
134
|
+
if (wholeChapter)
|
|
135
|
+
return `${book.name} ${startChapter}`;
|
|
55
136
|
if (startChapter === endChapter) {
|
|
56
137
|
return startVerse === endVerse
|
|
57
138
|
? `${book.name} ${startChapter}:${startVerse}`
|
|
@@ -59,20 +140,16 @@ function labelFor(book, startChapter, startVerse, endChapter, endVerse) {
|
|
|
59
140
|
}
|
|
60
141
|
return `${book.name} ${startChapter}:${startVerse}-${endChapter}:${endVerse}`;
|
|
61
142
|
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
if (!aliasKey)
|
|
68
|
-
return { kind: 'not-reference' };
|
|
69
|
-
const book = await resolver.resolveBookAlias(aliasKey);
|
|
70
|
-
if (!book)
|
|
71
|
-
return { kind: 'invalid-reference' };
|
|
143
|
+
/**
|
|
144
|
+
* Resolve a parsed locator against a committed book. Null means the locator
|
|
145
|
+
* is invalid for this book (bad chapter, missing verse, inverted range).
|
|
146
|
+
*/
|
|
147
|
+
async function resolveWithBook(book, syntax, resolver) {
|
|
72
148
|
let startChapter;
|
|
73
149
|
let startVerse;
|
|
74
150
|
let endChapter;
|
|
75
151
|
let endVerse;
|
|
152
|
+
let wholeChapter = false;
|
|
76
153
|
if (book.chapterCount === 1 && syntax.startVerse === null) {
|
|
77
154
|
startChapter = 1;
|
|
78
155
|
startVerse = syntax.startNumber;
|
|
@@ -81,14 +158,15 @@ export async function resolveReferenceAttempt(input, resolver) {
|
|
|
81
158
|
}
|
|
82
159
|
else if (syntax.startVerse === null) {
|
|
83
160
|
if (syntax.endNumber !== null)
|
|
84
|
-
return
|
|
161
|
+
return null;
|
|
85
162
|
startChapter = syntax.startNumber;
|
|
86
163
|
startVerse = 1;
|
|
87
164
|
endChapter = startChapter;
|
|
88
165
|
const count = await resolver.getChapterVerseCount(book.id, startChapter);
|
|
89
166
|
if (count === null)
|
|
90
|
-
return
|
|
167
|
+
return null;
|
|
91
168
|
endVerse = count;
|
|
169
|
+
wholeChapter = true;
|
|
92
170
|
}
|
|
93
171
|
else {
|
|
94
172
|
startChapter = syntax.startNumber;
|
|
@@ -104,26 +182,110 @@ export async function resolveReferenceAttempt(input, resolver) {
|
|
|
104
182
|
endVerse < 1 ||
|
|
105
183
|
!(await resolver.verseExists(book.id, startChapter, startVerse)) ||
|
|
106
184
|
!(await resolver.verseExists(book.id, endChapter, endVerse))) {
|
|
107
|
-
return
|
|
185
|
+
return null;
|
|
108
186
|
}
|
|
109
187
|
const startId = verseId(book.id, startChapter, startVerse);
|
|
110
188
|
const endId = verseId(book.id, endChapter, endVerse);
|
|
111
189
|
if (endId < startId)
|
|
112
|
-
return
|
|
190
|
+
return null;
|
|
113
191
|
return {
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
endId,
|
|
123
|
-
label: labelFor(book, startChapter, startVerse, endChapter, endVerse),
|
|
124
|
-
},
|
|
192
|
+
book,
|
|
193
|
+
startChapter,
|
|
194
|
+
startVerse,
|
|
195
|
+
endChapter,
|
|
196
|
+
endVerse,
|
|
197
|
+
startId,
|
|
198
|
+
endId,
|
|
199
|
+
label: labelFor(book, startChapter, startVerse, endChapter, endVerse, wholeChapter),
|
|
125
200
|
};
|
|
126
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* The cited did-you-mean (0.11.0/QR-4), run only after every candidate's book
|
|
204
|
+
* failed to resolve. Walks the same candidates in the same order and, for the
|
|
205
|
+
* first one whose typed book text has a UNIQUE in-policy alias match whose
|
|
206
|
+
* locator validates, returns the suggestion.
|
|
207
|
+
*
|
|
208
|
+
* Determinism: the winner is the book at minimum distance; the outcome is a
|
|
209
|
+
* pure function of the typed key and the alias SET (per-book minima and a
|
|
210
|
+
* uniqueness test are row-order independent by construction — pinned by the
|
|
211
|
+
* shuffle test). A tie across books suggests nothing for that candidate: a
|
|
212
|
+
* guess between two books is exactly what this feature refuses to make.
|
|
213
|
+
*/
|
|
214
|
+
async function suggestForCandidates(candidates, resolver) {
|
|
215
|
+
let aliases = null;
|
|
216
|
+
for (const syntax of candidates) {
|
|
217
|
+
const typedKey = normalizeBookAlias(syntax.bookText);
|
|
218
|
+
const bound = editDistanceBudget(typedKey.length);
|
|
219
|
+
if (bound === 0)
|
|
220
|
+
continue;
|
|
221
|
+
aliases ??= await resolver.listBookAliases();
|
|
222
|
+
let min = bound + 1;
|
|
223
|
+
const bestPerBook = new Map();
|
|
224
|
+
for (const row of aliases) {
|
|
225
|
+
const distance = damerauLevenshtein(typedKey, row.aliasKey, bound);
|
|
226
|
+
if (distance === null)
|
|
227
|
+
continue;
|
|
228
|
+
const existing = bestPerBook.get(row.bookId);
|
|
229
|
+
if (!existing || distance < existing.distance) {
|
|
230
|
+
bestPerBook.set(row.bookId, {
|
|
231
|
+
distance,
|
|
232
|
+
name: row.bookName,
|
|
233
|
+
chapterCount: row.chapterCount,
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
if (distance < min)
|
|
237
|
+
min = distance;
|
|
238
|
+
}
|
|
239
|
+
if (min > bound)
|
|
240
|
+
continue;
|
|
241
|
+
const winners = [...bestPerBook.entries()].filter(([, entry]) => entry.distance === min);
|
|
242
|
+
if (winners.length !== 1)
|
|
243
|
+
continue;
|
|
244
|
+
const [bookId, winner] = winners[0];
|
|
245
|
+
const book = { id: bookId, name: winner.name, chapterCount: winner.chapterCount };
|
|
246
|
+
// Validate through the exact resolution path the suggestion invites the
|
|
247
|
+
// user to take, so a suggestion never names a passage that cannot exist.
|
|
248
|
+
const reference = await resolveWithBook(book, syntax, resolver);
|
|
249
|
+
if (!reference)
|
|
250
|
+
continue;
|
|
251
|
+
return { book: book.name, reference: reference.label, distance: min };
|
|
252
|
+
}
|
|
253
|
+
return null;
|
|
254
|
+
}
|
|
255
|
+
export async function resolveReferenceAttempt(input, resolver) {
|
|
256
|
+
const parsed = parseCandidates(input);
|
|
257
|
+
if (!parsed)
|
|
258
|
+
return { kind: 'not-reference' };
|
|
259
|
+
// Commit rule: the FIRST candidate whose book resolves wins outright. The
|
|
260
|
+
// first candidate is the pre-0.11.0 parse, so every previously-resolving
|
|
261
|
+
// input is preserved bit-for-bit; a committed book with a bad locator is
|
|
262
|
+
// invalid-reference exactly as before, never re-parsed.
|
|
263
|
+
let sawBookText = false;
|
|
264
|
+
for (const syntax of parsed.candidates) {
|
|
265
|
+
const aliasKey = normalizeBookAlias(syntax.bookText);
|
|
266
|
+
if (!aliasKey)
|
|
267
|
+
continue;
|
|
268
|
+
sawBookText = true;
|
|
269
|
+
const book = await resolver.resolveBookAlias(aliasKey);
|
|
270
|
+
if (!book)
|
|
271
|
+
continue;
|
|
272
|
+
const reference = await resolveWithBook(book, syntax, resolver);
|
|
273
|
+
return reference
|
|
274
|
+
? { kind: 'resolved', reference }
|
|
275
|
+
: { kind: 'invalid-reference', fallthroughToDiscovery: false };
|
|
276
|
+
}
|
|
277
|
+
if (!sawBookText)
|
|
278
|
+
return { kind: 'not-reference' };
|
|
279
|
+
// No candidate's book resolves: cite a did-you-mean if a unique in-policy
|
|
280
|
+
// match validates (J35) …
|
|
281
|
+
const suggestion = await suggestForCandidates(parsed.candidates, resolver);
|
|
282
|
+
if (suggestion) {
|
|
283
|
+
return { kind: 'invalid-reference', suggestion, fallthroughToDiscovery: false };
|
|
284
|
+
}
|
|
285
|
+
// … otherwise explicit-separator queries stay invalid-reference (committed
|
|
286
|
+
// reference intent) and bare-number shapes fall through to discovery (J36).
|
|
287
|
+
return { kind: 'invalid-reference', fallthroughToDiscovery: !parsed.explicitSeparator };
|
|
288
|
+
}
|
|
127
289
|
export async function resolveReference(input, resolver) {
|
|
128
290
|
const attempt = await resolveReferenceAttempt(input, resolver);
|
|
129
291
|
return attempt.kind === 'resolved' ? attempt.reference : null;
|
|
@@ -22,6 +22,41 @@ export declare function normalizeToken(raw: string): string | null;
|
|
|
22
22
|
* set form used for overlap scoring and concept-lexicon matching.
|
|
23
23
|
*/
|
|
24
24
|
export declare function significantWords(text: string): string[];
|
|
25
|
+
/**
|
|
26
|
+
* The same significant-token set, each token paired with the SURFACE form it
|
|
27
|
+
* was normalized from — the first raw word (lowercased, punctuation-stripped)
|
|
28
|
+
* that produced it. Added for the spelling-correction citation (0.12.0/QR-5):
|
|
29
|
+
* a correction chip must cite what the user actually typed ("beleived"),
|
|
30
|
+
* never the stem the tokenizer made of it ("beleiv").
|
|
31
|
+
*
|
|
32
|
+
* This is a PAIRING, not a second tokenizer: `significantWords` delegates
|
|
33
|
+
* here, the token stream is byte-identical to what it always was (invariance-
|
|
34
|
+
* tested), and there is still no options parameter. TOKENIZER_VERSION stays
|
|
35
|
+
* 1.0.0.
|
|
36
|
+
*/
|
|
37
|
+
export declare function significantWordsWithSurface(text: string): {
|
|
38
|
+
token: string;
|
|
39
|
+
surface: string;
|
|
40
|
+
}[];
|
|
41
|
+
/**
|
|
42
|
+
* Whole-query phrase normalization for the curated alias table
|
|
43
|
+
* (0.13.0/QR-6): lowercase, apostrophes removed, punctuation folded to
|
|
44
|
+
* spaces, whitespace collapsed — and NOTHING else. Stopwords are KEPT and no
|
|
45
|
+
* stemming, archaic folding, or lemma lookup applies, because the phrases
|
|
46
|
+
* this key serves are exactly the stopword-heavy lines the token pipeline
|
|
47
|
+
* cannot represent ("it is well with my soul" -> `well soul`). Matching is
|
|
48
|
+
* whole-string EQUALITY against curated_aliases.normalized_raw, never
|
|
49
|
+
* containment, so the minimal normalization is the safety property: the less
|
|
50
|
+
* this folds, the less an alias can accidentally swallow.
|
|
51
|
+
*
|
|
52
|
+
* This is an ADDITIVE surface over the same `rawWords` core every other
|
|
53
|
+
* tokenizer output uses — not a second tokenizer, and there is still no
|
|
54
|
+
* options parameter. The token stream is untouched (invariance-tested) and
|
|
55
|
+
* TOKENIZER_VERSION stays 1.0.0. The pipeline's alias importer imports THIS
|
|
56
|
+
* function (never a mirror), so build-side keys and query-side keys cannot
|
|
57
|
+
* drift.
|
|
58
|
+
*/
|
|
59
|
+
export declare function normalizedPhrase(text: string): string;
|
|
25
60
|
/**
|
|
26
61
|
* Positional token stream: every significant occurrence, with the word index
|
|
27
62
|
* it came from. Proximity scoring (intent 3) needs positions, which the
|