@jestek-dev/scripture-engine 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/engineVersion.d.ts +19 -0
- package/dist/config/engineVersion.js +19 -0
- package/dist/corpus/repository.d.ts +193 -0
- package/dist/corpus/repository.js +466 -0
- package/dist/createEngine.d.ts +58 -0
- package/dist/createEngine.js +338 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.js +18 -0
- package/dist/intents/concept.d.ts +69 -0
- package/dist/intents/concept.js +143 -0
- package/dist/intents/lexical.d.ts +51 -0
- package/dist/intents/lexical.js +136 -0
- package/dist/ranking/budgets.d.ts +65 -0
- package/dist/ranking/budgets.js +149 -0
- package/dist/ranking/rank.d.ts +42 -0
- package/dist/ranking/rank.js +71 -0
- package/dist/reasons/types.d.ts +55 -0
- package/dist/reasons/types.js +20 -0
- package/dist/reference/reference.d.ts +31 -0
- package/dist/reference/reference.js +130 -0
- package/dist/reference/verseId.d.ts +12 -0
- package/dist/reference/verseId.js +36 -0
- package/dist/tokenizer/index.d.ts +39 -0
- package/dist/tokenizer/index.js +207 -0
- package/dist/types.d.ts +125 -0
- package/dist/types.js +11 -0
- package/package.json +27 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
const COMPACT_DOT_RE = /^([1-3]?\s?[A-Za-z]+)\.(\d{1,3})\.(\d{1,3})$/;
|
|
2
|
+
const BOOK_LOCATOR_RE = /^(.*?)\s*\.?\s*(\d{1,3}(?:\s*[:.]\s*\d{1,3})?(?:\s*-\s*(?:\d{1,3}\s*[:.]\s*)?\d{1,3})?)$/;
|
|
3
|
+
const LOCATOR_RE = /^(\d{1,3})(?:\s*[:.]\s*(\d{1,3}))?(?:\s*-\s*(?:(\d{1,3})\s*[:.]\s*)?(\d{1,3}))?$/;
|
|
4
|
+
const DASH_CHARS_RE = /[‒–—―−]/g;
|
|
5
|
+
const ROMAN_PREFIX_RE = /^(iii|ii|i)(?=\s|$)/;
|
|
6
|
+
const ROMAN_TO_ARABIC = {
|
|
7
|
+
i: '1',
|
|
8
|
+
ii: '2',
|
|
9
|
+
iii: '3',
|
|
10
|
+
};
|
|
11
|
+
export function normalizeBookAlias(input) {
|
|
12
|
+
return input
|
|
13
|
+
.trim()
|
|
14
|
+
.toLowerCase()
|
|
15
|
+
.replace(ROMAN_PREFIX_RE, (value) => ROMAN_TO_ARABIC[value])
|
|
16
|
+
.replace(/[^a-z0-9]/g, '');
|
|
17
|
+
}
|
|
18
|
+
function parseSyntax(input) {
|
|
19
|
+
const trimmed = input.trim().replace(DASH_CHARS_RE, '-');
|
|
20
|
+
if (!trimmed)
|
|
21
|
+
return null;
|
|
22
|
+
const compact = COMPACT_DOT_RE.exec(trimmed);
|
|
23
|
+
if (compact) {
|
|
24
|
+
return {
|
|
25
|
+
bookText: compact[1],
|
|
26
|
+
startNumber: Number(compact[2]),
|
|
27
|
+
startVerse: Number(compact[3]),
|
|
28
|
+
endChapter: null,
|
|
29
|
+
endNumber: null,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
const split = BOOK_LOCATOR_RE.exec(trimmed);
|
|
33
|
+
if (!split || !split[1]?.trim())
|
|
34
|
+
return null;
|
|
35
|
+
const locator = LOCATOR_RE.exec(split[2]);
|
|
36
|
+
if (!locator)
|
|
37
|
+
return null;
|
|
38
|
+
return {
|
|
39
|
+
bookText: split[1],
|
|
40
|
+
startNumber: Number(locator[1]),
|
|
41
|
+
startVerse: locator[2] === undefined ? null : Number(locator[2]),
|
|
42
|
+
endChapter: locator[3] === undefined ? null : Number(locator[3]),
|
|
43
|
+
endNumber: locator[4] === undefined ? null : Number(locator[4]),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function verseId(bookId, chapter, verse) {
|
|
47
|
+
return bookId * 1_000_000 + chapter * 1_000 + verse;
|
|
48
|
+
}
|
|
49
|
+
function labelFor(book, startChapter, startVerse, endChapter, endVerse) {
|
|
50
|
+
if (startChapter === endChapter && startVerse === 1) {
|
|
51
|
+
return startVerse === endVerse
|
|
52
|
+
? `${book.name} ${startChapter}:${startVerse}`
|
|
53
|
+
: `${book.name} ${startChapter}`;
|
|
54
|
+
}
|
|
55
|
+
if (startChapter === endChapter) {
|
|
56
|
+
return startVerse === endVerse
|
|
57
|
+
? `${book.name} ${startChapter}:${startVerse}`
|
|
58
|
+
: `${book.name} ${startChapter}:${startVerse}-${endVerse}`;
|
|
59
|
+
}
|
|
60
|
+
return `${book.name} ${startChapter}:${startVerse}-${endChapter}:${endVerse}`;
|
|
61
|
+
}
|
|
62
|
+
export async function resolveReferenceAttempt(input, resolver) {
|
|
63
|
+
const syntax = parseSyntax(input);
|
|
64
|
+
if (!syntax)
|
|
65
|
+
return { kind: 'not-reference' };
|
|
66
|
+
const aliasKey = normalizeBookAlias(syntax.bookText);
|
|
67
|
+
if (!aliasKey)
|
|
68
|
+
return { kind: 'not-reference' };
|
|
69
|
+
const book = await resolver.resolveBookAlias(aliasKey);
|
|
70
|
+
if (!book)
|
|
71
|
+
return { kind: 'invalid-reference' };
|
|
72
|
+
let startChapter;
|
|
73
|
+
let startVerse;
|
|
74
|
+
let endChapter;
|
|
75
|
+
let endVerse;
|
|
76
|
+
if (book.chapterCount === 1 && syntax.startVerse === null) {
|
|
77
|
+
startChapter = 1;
|
|
78
|
+
startVerse = syntax.startNumber;
|
|
79
|
+
endChapter = 1;
|
|
80
|
+
endVerse = syntax.endNumber ?? syntax.startNumber;
|
|
81
|
+
}
|
|
82
|
+
else if (syntax.startVerse === null) {
|
|
83
|
+
if (syntax.endNumber !== null)
|
|
84
|
+
return { kind: 'invalid-reference' };
|
|
85
|
+
startChapter = syntax.startNumber;
|
|
86
|
+
startVerse = 1;
|
|
87
|
+
endChapter = startChapter;
|
|
88
|
+
const count = await resolver.getChapterVerseCount(book.id, startChapter);
|
|
89
|
+
if (count === null)
|
|
90
|
+
return { kind: 'invalid-reference' };
|
|
91
|
+
endVerse = count;
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
startChapter = syntax.startNumber;
|
|
95
|
+
startVerse = syntax.startVerse;
|
|
96
|
+
endChapter = syntax.endChapter ?? startChapter;
|
|
97
|
+
endVerse = syntax.endNumber ?? startVerse;
|
|
98
|
+
}
|
|
99
|
+
if (startChapter < 1 ||
|
|
100
|
+
endChapter < 1 ||
|
|
101
|
+
startChapter > book.chapterCount ||
|
|
102
|
+
endChapter > book.chapterCount ||
|
|
103
|
+
startVerse < 1 ||
|
|
104
|
+
endVerse < 1 ||
|
|
105
|
+
!(await resolver.verseExists(book.id, startChapter, startVerse)) ||
|
|
106
|
+
!(await resolver.verseExists(book.id, endChapter, endVerse))) {
|
|
107
|
+
return { kind: 'invalid-reference' };
|
|
108
|
+
}
|
|
109
|
+
const startId = verseId(book.id, startChapter, startVerse);
|
|
110
|
+
const endId = verseId(book.id, endChapter, endVerse);
|
|
111
|
+
if (endId < startId)
|
|
112
|
+
return { kind: 'invalid-reference' };
|
|
113
|
+
return {
|
|
114
|
+
kind: 'resolved',
|
|
115
|
+
reference: {
|
|
116
|
+
book,
|
|
117
|
+
startChapter,
|
|
118
|
+
startVerse,
|
|
119
|
+
endChapter,
|
|
120
|
+
endVerse,
|
|
121
|
+
startId,
|
|
122
|
+
endId,
|
|
123
|
+
label: labelFor(book, startChapter, startVerse, endChapter, endVerse),
|
|
124
|
+
},
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
export async function resolveReference(input, resolver) {
|
|
128
|
+
const attempt = await resolveReferenceAttempt(input, resolver);
|
|
129
|
+
return attempt.kind === 'resolved' ? attempt.reference : null;
|
|
130
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Verse ids encode book/chapter/verse as a single integer: BBCCCVVV
|
|
3
|
+
* (bookId * 1_000_000 + chapter * 1_000 + verse). This keeps verse ids
|
|
4
|
+
* sortable, stable across imports, and cheap to index in SQLite.
|
|
5
|
+
*/
|
|
6
|
+
export interface VerseLocation {
|
|
7
|
+
bookId: number;
|
|
8
|
+
chapter: number;
|
|
9
|
+
verse: number;
|
|
10
|
+
}
|
|
11
|
+
export declare function makeVerseId(bookId: number, chapter: number, verse: number): number;
|
|
12
|
+
export declare function parseVerseId(verseId: number): VerseLocation;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Verse ids encode book/chapter/verse as a single integer: BBCCCVVV
|
|
3
|
+
* (bookId * 1_000_000 + chapter * 1_000 + verse). This keeps verse ids
|
|
4
|
+
* sortable, stable across imports, and cheap to index in SQLite.
|
|
5
|
+
*/
|
|
6
|
+
const MIN_BOOK_ID = 1;
|
|
7
|
+
const MAX_BOOK_ID = 66;
|
|
8
|
+
const MIN_CHAPTER = 1;
|
|
9
|
+
const MAX_CHAPTER = 999;
|
|
10
|
+
const MIN_VERSE = 1;
|
|
11
|
+
const MAX_VERSE = 999;
|
|
12
|
+
export function makeVerseId(bookId, chapter, verse) {
|
|
13
|
+
if (!Number.isInteger(bookId) || bookId < MIN_BOOK_ID || bookId > MAX_BOOK_ID) {
|
|
14
|
+
throw new Error(`makeVerseId: bookId ${bookId} is out of range 1-66`);
|
|
15
|
+
}
|
|
16
|
+
if (!Number.isInteger(chapter) || chapter < MIN_CHAPTER || chapter > MAX_CHAPTER) {
|
|
17
|
+
throw new Error(`makeVerseId: chapter ${chapter} is out of range 1-999`);
|
|
18
|
+
}
|
|
19
|
+
if (!Number.isInteger(verse) || verse < MIN_VERSE || verse > MAX_VERSE) {
|
|
20
|
+
throw new Error(`makeVerseId: verse ${verse} is out of range 1-999`);
|
|
21
|
+
}
|
|
22
|
+
return bookId * 1_000_000 + chapter * 1_000 + verse;
|
|
23
|
+
}
|
|
24
|
+
export function parseVerseId(verseId) {
|
|
25
|
+
if (!Number.isInteger(verseId) || verseId < 1_001_001 || verseId > 66_999_999) {
|
|
26
|
+
throw new Error(`parseVerseId: ${verseId} is not a valid verse id`);
|
|
27
|
+
}
|
|
28
|
+
const bookId = Math.floor(verseId / 1_000_000);
|
|
29
|
+
const chapter = Math.floor((verseId % 1_000_000) / 1_000);
|
|
30
|
+
const verse = verseId % 1_000;
|
|
31
|
+
// Re-validate via makeVerseId so range-checking logic isn't duplicated,
|
|
32
|
+
// and so a value like 1000000 (chapter 0) is rejected rather than
|
|
33
|
+
// silently accepted.
|
|
34
|
+
makeVerseId(bookId, chapter, verse);
|
|
35
|
+
return { bookId, chapter, verse };
|
|
36
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE shared tokenizer. Ported from LH Worship Setlist's
|
|
3
|
+
* `src/lib/bible/keywords.ts`, which already solved the hard part: a stopword
|
|
4
|
+
* list that covers KJV-era pronouns and verb forms, so modern lyric/query
|
|
5
|
+
* vocabulary matches archaic scripture text without every `thou`/`hath`
|
|
6
|
+
* polluting the result.
|
|
7
|
+
*
|
|
8
|
+
* "THE" is load-bearing. The build pipeline and the runtime MUST tokenize
|
|
9
|
+
* identically — corpus term profiles are precomputed at build time, and a
|
|
10
|
+
* runtime that stems differently would compare mismatched vocabularies and
|
|
11
|
+
* silently return wrong rankings. That is why TOKENIZER_VERSION is stamped
|
|
12
|
+
* into the artifact and verified on open, and why this module has no options
|
|
13
|
+
* parameter: a per-caller tokenizer setting is exactly the bug it prevents.
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Normalize one surface form to its index token: archaic fold, then stem.
|
|
17
|
+
* Words shorter than 3 characters and stopwords return null.
|
|
18
|
+
*/
|
|
19
|
+
export declare function normalizeToken(raw: string): string | null;
|
|
20
|
+
/**
|
|
21
|
+
* Significant tokens in order of first occurrence, deduplicated. This is the
|
|
22
|
+
* set form used for overlap scoring and concept-lexicon matching.
|
|
23
|
+
*/
|
|
24
|
+
export declare function significantWords(text: string): string[];
|
|
25
|
+
/**
|
|
26
|
+
* Positional token stream: every significant occurrence, with the word index
|
|
27
|
+
* it came from. Proximity scoring (intent 3) needs positions, which the
|
|
28
|
+
* deduplicated form above deliberately discards. Positions are indices into
|
|
29
|
+
* the raw word sequence, so distance survives dropped stopwords rather than
|
|
30
|
+
* being compressed by them.
|
|
31
|
+
*/
|
|
32
|
+
export declare function tokenStream(text: string): readonly {
|
|
33
|
+
token: string;
|
|
34
|
+
position: number;
|
|
35
|
+
}[];
|
|
36
|
+
/** Exposed for gate tooling and tests; never mutate. */
|
|
37
|
+
export declare const TOKENIZER_STOPWORD_COUNT: number;
|
|
38
|
+
export declare const TOKENIZER_ARCHAIC_FORM_COUNT: number;
|
|
39
|
+
export declare const TOKENIZER_LEMMA_COUNT: number;
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE shared tokenizer. Ported from LH Worship Setlist's
|
|
3
|
+
* `src/lib/bible/keywords.ts`, which already solved the hard part: a stopword
|
|
4
|
+
* list that covers KJV-era pronouns and verb forms, so modern lyric/query
|
|
5
|
+
* vocabulary matches archaic scripture text without every `thou`/`hath`
|
|
6
|
+
* polluting the result.
|
|
7
|
+
*
|
|
8
|
+
* "THE" is load-bearing. The build pipeline and the runtime MUST tokenize
|
|
9
|
+
* identically — corpus term profiles are precomputed at build time, and a
|
|
10
|
+
* runtime that stems differently would compare mismatched vocabularies and
|
|
11
|
+
* silently return wrong rankings. That is why TOKENIZER_VERSION is stamped
|
|
12
|
+
* into the artifact and verified on open, and why this module has no options
|
|
13
|
+
* parameter: a per-caller tokenizer setting is exactly the bug it prevents.
|
|
14
|
+
*/
|
|
15
|
+
// Common English function words plus KJV-era pronouns/verb forms.
|
|
16
|
+
const STOPWORDS = new Set([
|
|
17
|
+
// articles / conjunctions / prepositions
|
|
18
|
+
'the', 'a', 'an', 'and', 'or', 'but', 'nor', 'for', 'so', 'yet',
|
|
19
|
+
'of', 'in', 'on', 'at', 'to', 'by', 'with', 'from', 'up', 'down',
|
|
20
|
+
'into', 'onto', 'over', 'under', 'after', 'before', 'between', 'through',
|
|
21
|
+
'about', 'above', 'below', 'again', 'further', 'then', 'once', 'than',
|
|
22
|
+
'as', 'if', 'because', 'while', 'until', 'unless', 'though', 'although',
|
|
23
|
+
'out', 'off', 'not', 'no', 'too', 'very', 'just', 'also', 'only',
|
|
24
|
+
'own', 'same', 'such',
|
|
25
|
+
// pronouns
|
|
26
|
+
'i', 'me', 'my', 'mine', 'myself',
|
|
27
|
+
'we', 'us', 'our', 'ours', 'ourselves',
|
|
28
|
+
'you', 'your', 'yours', 'yourself', 'yourselves',
|
|
29
|
+
'he', 'him', 'his', 'himself',
|
|
30
|
+
'she', 'her', 'hers', 'herself',
|
|
31
|
+
'it', 'its', 'itself',
|
|
32
|
+
'they', 'them', 'their', 'theirs', 'themselves',
|
|
33
|
+
'who', 'whom', 'whose', 'which', 'what', 'this', 'that', 'these', 'those',
|
|
34
|
+
// verbs (be/have + modals)
|
|
35
|
+
'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
|
|
36
|
+
'have', 'has', 'had', 'having',
|
|
37
|
+
'will', 'would', 'can', 'could', 'may', 'might', 'must', 'shall', 'should',
|
|
38
|
+
// NOTE: the do-family (do/does/did/doing/done) is deliberately ABSENT.
|
|
39
|
+
// Setlist's list dropped it as auxiliary noise, which is correct for bare
|
|
40
|
+
// overlap counting but wrong here: "doers of the word", "hearing and
|
|
41
|
+
// doing", and "he who hears and does" are the exact theological vocabulary
|
|
42
|
+
// this engine exists to match. Keeping it costs little, because unlike
|
|
43
|
+
// Setlist we weight tokens statistically — a token this common earns a
|
|
44
|
+
// near-zero IDF contribution and can never clear the G5 distinctiveness
|
|
45
|
+
// floor to enter a passage term profile. Frequency is handled by
|
|
46
|
+
// statistics; the stopword list is too blunt an instrument to decide which
|
|
47
|
+
// verbs carry doctrine.
|
|
48
|
+
// misc high-frequency function words
|
|
49
|
+
'there', 'here', 'when', 'where', 'why', 'how', 'all', 'each', 'every',
|
|
50
|
+
'both', 'few', 'more', 'most', 'other', 'some', 'any', 'one', 'upon',
|
|
51
|
+
'let', 'lest', 'yes',
|
|
52
|
+
// KJV-isms
|
|
53
|
+
'thee', 'thou', 'thy', 'thine', 'ye',
|
|
54
|
+
'hath', 'hast', 'doth', 'dost', 'art', 'wilt', 'wouldst', 'shalt', 'shouldst',
|
|
55
|
+
'unto', 'verily', 'behold', 'saith', 'whence', 'whither', 'hither', 'thither',
|
|
56
|
+
'wherefore', 'forasmuch', 'peradventure', 'howbeit', 'thereof', 'thereto',
|
|
57
|
+
'herein', 'wherein', 'whereof', 'thereby', 'hereby', 'hereof', 'whosoever',
|
|
58
|
+
]);
|
|
59
|
+
/**
|
|
60
|
+
* Archaic verb forms folded to their modern stem BEFORE stemming, so a query
|
|
61
|
+
* typed as "hearing and doing" can reach text that reads "he that heareth
|
|
62
|
+
* these sayings of mine, and doeth them". Suffix stripping alone cannot do
|
|
63
|
+
* this: `doeth` -> `doeth` under any -s/-ed/-ing rule.
|
|
64
|
+
*
|
|
65
|
+
* Deliberately a short, reviewed table of high-frequency forms rather than a
|
|
66
|
+
* generative rule — `-eth`/`-est` stripping produces false merges (`death`,
|
|
67
|
+
* `breath`, `harvest`) that would quietly corrupt term profiles. Growth of
|
|
68
|
+
* this table is a reviewed change like any other data addition.
|
|
69
|
+
*/
|
|
70
|
+
const ARCHAIC_FORMS = new Map([
|
|
71
|
+
['doeth', 'do'], ['doest', 'do'],
|
|
72
|
+
['heareth', 'hear'], ['hearest', 'hear'],
|
|
73
|
+
['keepeth', 'keep'], ['keepest', 'keep'],
|
|
74
|
+
['believeth', 'believe'], ['believest', 'believe'],
|
|
75
|
+
['walketh', 'walk'], ['walkest', 'walk'],
|
|
76
|
+
['speaketh', 'speak'], ['speakest', 'speak'],
|
|
77
|
+
['loveth', 'love'], ['lovest', 'love'],
|
|
78
|
+
['giveth', 'give'], ['givest', 'give'],
|
|
79
|
+
['knoweth', 'know'], ['knowest', 'know'],
|
|
80
|
+
['maketh', 'make'], ['makest', 'make'],
|
|
81
|
+
['cometh', 'come'], ['comest', 'come'],
|
|
82
|
+
['seeketh', 'seek'], ['seekest', 'seek'],
|
|
83
|
+
['calleth', 'call'], ['callest', 'call'],
|
|
84
|
+
['dwelleth', 'dwell'], ['dwellest', 'dwell'],
|
|
85
|
+
['abideth', 'abide'], ['abidest', 'abide'],
|
|
86
|
+
['trusteth', 'trust'], ['trustest', 'trust'],
|
|
87
|
+
['worketh', 'work'], ['workest', 'work'],
|
|
88
|
+
['receiveth', 'receive'], ['sendeth', 'send'],
|
|
89
|
+
['buildeth', 'build'], ['heareth', 'hear'],
|
|
90
|
+
['obeyeth', 'obey'], ['forgiveth', 'forgive'],
|
|
91
|
+
]);
|
|
92
|
+
/**
|
|
93
|
+
* Irregular and short-word lemmas that suffix stemming provably cannot reach.
|
|
94
|
+
*
|
|
95
|
+
* The stemmer refuses to cut below a 4-character stem — a necessary rule that
|
|
96
|
+
* prevents `ties`->`t`, but one that silently strands the shortest and most
|
|
97
|
+
* common verbs: `doing` (5-3 < 4), `does`, `done`, `heard`. Those are exactly
|
|
98
|
+
* the words in "hearing and doing", so without this table the engine's
|
|
99
|
+
* motivating query cannot match the text it is meant to find.
|
|
100
|
+
*
|
|
101
|
+
* Agent nouns fold to their verb (`doer` -> `do`) because "doers of the word"
|
|
102
|
+
* and "he who does the word" are the same claim, and a search for one must
|
|
103
|
+
* find the other.
|
|
104
|
+
*
|
|
105
|
+
* Reviewed table, not a generative rule: every entry is a deliberate merge,
|
|
106
|
+
* because a wrong merge is invisible at query time and corrupts every term
|
|
107
|
+
* profile built from it.
|
|
108
|
+
*/
|
|
109
|
+
const IRREGULAR_LEMMAS = new Map([
|
|
110
|
+
// do-family: the concept "hearing and doing" lives or dies here
|
|
111
|
+
['does', 'do'], ['doing', 'do'], ['done', 'do'], ['did', 'do'],
|
|
112
|
+
['doer', 'do'], ['doers', 'do'],
|
|
113
|
+
// hear-family: short forms and agent nouns
|
|
114
|
+
['heard', 'hear'], ['hearer', 'hear'], ['hearers', 'hear'],
|
|
115
|
+
['hears', 'hear'], ['hearken', 'hear'], ['hearkened', 'hear'],
|
|
116
|
+
// other high-frequency irregulars whose stems fall below the 4-char floor
|
|
117
|
+
['said', 'say'], ['says', 'say'], ['saith', 'say'], ['sayings', 'say'],
|
|
118
|
+
['saying', 'say'], ['spoken', 'speak'], ['spoke', 'speak'],
|
|
119
|
+
['kept', 'keep'], ['keeps', 'keep'],
|
|
120
|
+
['gave', 'give'], ['given', 'give'], ['gives', 'give'],
|
|
121
|
+
['knew', 'know'], ['known', 'know'], ['knows', 'know'],
|
|
122
|
+
['built', 'build'], ['builds', 'build'],
|
|
123
|
+
['sought', 'seek'], ['seeks', 'seek'],
|
|
124
|
+
['stood', 'stand'], ['stands', 'stand'],
|
|
125
|
+
['fell', 'fall'], ['fallen', 'fall'], ['falls', 'fall'],
|
|
126
|
+
['obeys', 'obey'], ['obeyed', 'obey'],
|
|
127
|
+
]);
|
|
128
|
+
/** Light suffix stemming: -ing, -ies, -ed, -es, -s. Stem must stay >= 4 chars. */
|
|
129
|
+
function stem(word) {
|
|
130
|
+
if (word.endsWith('ing') && word.length - 3 >= 4)
|
|
131
|
+
return word.slice(0, -3);
|
|
132
|
+
if (word.endsWith('ies') && word.length - 3 >= 4)
|
|
133
|
+
return `${word.slice(0, -3)}y`;
|
|
134
|
+
if (word.endsWith('ed') && word.length - 2 >= 4)
|
|
135
|
+
return word.slice(0, -2);
|
|
136
|
+
if (word.endsWith('es') && word.length - 2 >= 4)
|
|
137
|
+
return word.slice(0, -2);
|
|
138
|
+
if (word.endsWith('s') && !word.endsWith('ss') && word.length - 1 >= 4) {
|
|
139
|
+
return word.slice(0, -1);
|
|
140
|
+
}
|
|
141
|
+
return word;
|
|
142
|
+
}
|
|
143
|
+
/** Split into lowercase word forms with punctuation and apostrophes removed. */
|
|
144
|
+
function rawWords(text) {
|
|
145
|
+
return text
|
|
146
|
+
.toLowerCase()
|
|
147
|
+
.replace(/['‘’‛ʼ]/g, '')
|
|
148
|
+
.replace(/[^\p{L}\p{N}\s]/gu, ' ')
|
|
149
|
+
.split(/\s+/)
|
|
150
|
+
.filter(Boolean);
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Normalize one surface form to its index token: archaic fold, then stem.
|
|
154
|
+
* Words shorter than 3 characters and stopwords return null.
|
|
155
|
+
*/
|
|
156
|
+
export function normalizeToken(raw) {
|
|
157
|
+
// Length floor applies to the surface form, but AFTER lemma lookup, so
|
|
158
|
+
// two-letter results of a deliberate merge (none today) stay reachable.
|
|
159
|
+
if (STOPWORDS.has(raw))
|
|
160
|
+
return null;
|
|
161
|
+
const modern = ARCHAIC_FORMS.get(raw) ?? raw;
|
|
162
|
+
const lemma = IRREGULAR_LEMMAS.get(modern);
|
|
163
|
+
if (lemma !== undefined)
|
|
164
|
+
return STOPWORDS.has(lemma) ? null : lemma;
|
|
165
|
+
if (raw.length < 3)
|
|
166
|
+
return null;
|
|
167
|
+
const stemmed = stem(modern);
|
|
168
|
+
if (STOPWORDS.has(stemmed))
|
|
169
|
+
return null;
|
|
170
|
+
return stemmed;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Significant tokens in order of first occurrence, deduplicated. This is the
|
|
174
|
+
* set form used for overlap scoring and concept-lexicon matching.
|
|
175
|
+
*/
|
|
176
|
+
export function significantWords(text) {
|
|
177
|
+
const seen = new Set();
|
|
178
|
+
const result = [];
|
|
179
|
+
for (const raw of rawWords(text)) {
|
|
180
|
+
const token = normalizeToken(raw);
|
|
181
|
+
if (token === null || seen.has(token))
|
|
182
|
+
continue;
|
|
183
|
+
seen.add(token);
|
|
184
|
+
result.push(token);
|
|
185
|
+
}
|
|
186
|
+
return result;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Positional token stream: every significant occurrence, with the word index
|
|
190
|
+
* it came from. Proximity scoring (intent 3) needs positions, which the
|
|
191
|
+
* deduplicated form above deliberately discards. Positions are indices into
|
|
192
|
+
* the raw word sequence, so distance survives dropped stopwords rather than
|
|
193
|
+
* being compressed by them.
|
|
194
|
+
*/
|
|
195
|
+
export function tokenStream(text) {
|
|
196
|
+
const stream = [];
|
|
197
|
+
rawWords(text).forEach((raw, position) => {
|
|
198
|
+
const token = normalizeToken(raw);
|
|
199
|
+
if (token !== null)
|
|
200
|
+
stream.push({ token, position });
|
|
201
|
+
});
|
|
202
|
+
return stream;
|
|
203
|
+
}
|
|
204
|
+
/** Exposed for gate tooling and tests; never mutate. */
|
|
205
|
+
export const TOKENIZER_STOPWORD_COUNT = STOPWORDS.size;
|
|
206
|
+
export const TOKENIZER_ARCHAIC_FORM_COUNT = ARCHAIC_FORMS.size;
|
|
207
|
+
export const TOKENIZER_LEMMA_COUNT = IRREGULAR_LEMMAS.size;
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The consumer-facing contract. Ported from Maskil's
|
|
3
|
+
* `app/src/scripture/types.ts` and generalized so Setlist and Versed can use
|
|
4
|
+
* the same engine without inheriting Maskil-shaped assumptions.
|
|
5
|
+
*
|
|
6
|
+
* `ContentQueryPort` is the ONLY seam to the outside world. Maskil supplies
|
|
7
|
+
* OP-SQLite on device, the pipeline and eval harness supply better-sqlite3 in
|
|
8
|
+
* Node, and a future hosted consumer could supply an HTTP-backed adapter —
|
|
9
|
+
* none of which the engine knows or cares about.
|
|
10
|
+
*/
|
|
11
|
+
import type { Reason } from './reasons/types.js';
|
|
12
|
+
export type ContentScalar = string | number | boolean | null | ArrayBuffer | ArrayBufferView;
|
|
13
|
+
export interface ContentQueryResult {
|
|
14
|
+
readonly rows: readonly Readonly<Record<string, ContentScalar>>[];
|
|
15
|
+
}
|
|
16
|
+
export interface ContentQueryPort {
|
|
17
|
+
execute(query: string, params?: readonly ContentScalar[]): Promise<ContentQueryResult>;
|
|
18
|
+
close(): Promise<void>;
|
|
19
|
+
}
|
|
20
|
+
export interface ScriptureVerse {
|
|
21
|
+
readonly id: number;
|
|
22
|
+
readonly verseId: number;
|
|
23
|
+
readonly translationId: number;
|
|
24
|
+
readonly translationCode: string;
|
|
25
|
+
readonly bookId: number;
|
|
26
|
+
readonly bookName: string;
|
|
27
|
+
readonly chapter: number;
|
|
28
|
+
readonly verse: number;
|
|
29
|
+
readonly text: string;
|
|
30
|
+
}
|
|
31
|
+
export interface ScripturePassage {
|
|
32
|
+
readonly reference: string;
|
|
33
|
+
readonly bookId: number;
|
|
34
|
+
readonly chapterCount: number;
|
|
35
|
+
readonly startChapter: number;
|
|
36
|
+
readonly endChapter: number;
|
|
37
|
+
readonly verses: readonly ScriptureVerse[];
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Every result carries the identities that make it reproducible. A consumer
|
|
41
|
+
* can record `(engineVersion, corpusFingerprint, layerFingerprint, query)`
|
|
42
|
+
* and any other consumer, on any platform, can regenerate the identical
|
|
43
|
+
* ordering. This is the machine-checkable form of the "deterministic" claim.
|
|
44
|
+
*/
|
|
45
|
+
export interface ResultIdentity {
|
|
46
|
+
readonly engineVersion: string;
|
|
47
|
+
/** Identifies the scripture text. */
|
|
48
|
+
readonly corpusFingerprint: string;
|
|
49
|
+
/**
|
|
50
|
+
* Identifies the curated concept and homiletical layers.
|
|
51
|
+
*
|
|
52
|
+
* Separate from the corpus fingerprint because they change for different
|
|
53
|
+
* reasons and at different rates — and because without it, editing a
|
|
54
|
+
* concept would silently alter rankings while every published identity
|
|
55
|
+
* stayed the same. Reproducing a result requires BOTH.
|
|
56
|
+
*/
|
|
57
|
+
readonly layerFingerprint: string;
|
|
58
|
+
}
|
|
59
|
+
export interface DiscoveryResult {
|
|
60
|
+
readonly targetId: string;
|
|
61
|
+
readonly reference: string;
|
|
62
|
+
readonly excerpt: string;
|
|
63
|
+
readonly score: number;
|
|
64
|
+
readonly reasons: readonly Reason[];
|
|
65
|
+
}
|
|
66
|
+
export type ResearchOutcome = {
|
|
67
|
+
readonly kind: 'reference';
|
|
68
|
+
readonly passage: ScripturePassage;
|
|
69
|
+
} | {
|
|
70
|
+
readonly kind: 'invalid-reference';
|
|
71
|
+
readonly query: string;
|
|
72
|
+
} | {
|
|
73
|
+
readonly kind: 'discovery';
|
|
74
|
+
readonly query: string;
|
|
75
|
+
readonly results: readonly DiscoveryResult[];
|
|
76
|
+
};
|
|
77
|
+
export type ResearchResult = ResearchOutcome & ResultIdentity;
|
|
78
|
+
/** Concept-resolution output for `engine.themes()`. */
|
|
79
|
+
export interface ConceptMatch {
|
|
80
|
+
readonly conceptId: string;
|
|
81
|
+
readonly label: string;
|
|
82
|
+
/** Which lexicon entry matched, so the UI can show why this concept fired. */
|
|
83
|
+
readonly matchedOn: string;
|
|
84
|
+
readonly anchors: readonly string[];
|
|
85
|
+
}
|
|
86
|
+
/** `engine.passage()` — a lookup, with invalid references typed rather than thrown. */
|
|
87
|
+
export type PassageResult = ({
|
|
88
|
+
readonly kind: 'passage';
|
|
89
|
+
readonly passage: ScripturePassage;
|
|
90
|
+
} & ResultIdentity) | ({
|
|
91
|
+
readonly kind: 'invalid-reference';
|
|
92
|
+
readonly query: string;
|
|
93
|
+
} & ResultIdentity);
|
|
94
|
+
/**
|
|
95
|
+
* `engine.related()` — what a curated source connects to a passage.
|
|
96
|
+
*
|
|
97
|
+
* Deliberately NOT "verses that resemble this one". Every entry here exists
|
|
98
|
+
* because a human recorded a link: a cross-reference edge, or a concept whose
|
|
99
|
+
* anchors include this passage. Similarity is what `research()` does.
|
|
100
|
+
*/
|
|
101
|
+
export type RelatedResult = ({
|
|
102
|
+
readonly kind: 'related';
|
|
103
|
+
readonly reference: string;
|
|
104
|
+
/** Concepts whose curated anchors include this passage. */
|
|
105
|
+
readonly concepts: readonly ConceptMatch[];
|
|
106
|
+
readonly results: readonly DiscoveryResult[];
|
|
107
|
+
} & ResultIdentity) | ({
|
|
108
|
+
readonly kind: 'invalid-reference';
|
|
109
|
+
readonly query: string;
|
|
110
|
+
} & ResultIdentity);
|
|
111
|
+
/**
|
|
112
|
+
* Multi-field input for `engine.forSong()`.
|
|
113
|
+
*
|
|
114
|
+
* Setlist matches a sermon theme to songs; Maskil starts from a song being
|
|
115
|
+
* written. Both need more than one field, and neither wants to pre-concatenate
|
|
116
|
+
* them, because the fields differ in evidential weight — a stated theme is a
|
|
117
|
+
* claim about meaning, a lyric line is raw text that happens to be present.
|
|
118
|
+
*/
|
|
119
|
+
export interface SongInput {
|
|
120
|
+
readonly title?: string;
|
|
121
|
+
readonly themes?: readonly string[];
|
|
122
|
+
readonly lyrics?: string;
|
|
123
|
+
/** A passage the song is built on. Seeds curated expansion, not text search. */
|
|
124
|
+
readonly foundationalRef?: string;
|
|
125
|
+
}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The consumer-facing contract. Ported from Maskil's
|
|
3
|
+
* `app/src/scripture/types.ts` and generalized so Setlist and Versed can use
|
|
4
|
+
* the same engine without inheriting Maskil-shaped assumptions.
|
|
5
|
+
*
|
|
6
|
+
* `ContentQueryPort` is the ONLY seam to the outside world. Maskil supplies
|
|
7
|
+
* OP-SQLite on device, the pipeline and eval harness supply better-sqlite3 in
|
|
8
|
+
* Node, and a future hosted consumer could supply an HTTP-backed adapter —
|
|
9
|
+
* none of which the engine knows or cares about.
|
|
10
|
+
*/
|
|
11
|
+
export {};
|
package/package.json
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@jestek-dev/scripture-engine",
|
|
3
|
+
"version": "0.7.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Pure deterministic Scripture retrieval and ranking core. Zero I/O, zero runtime AI.",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"default": "./dist/index.js"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"files": [
|
|
15
|
+
"dist"
|
|
16
|
+
],
|
|
17
|
+
"scripts": {
|
|
18
|
+
"prepack": "npm run build",
|
|
19
|
+
"build": "tsc",
|
|
20
|
+
"typecheck": "tsc --noEmit",
|
|
21
|
+
"test": "vitest run"
|
|
22
|
+
},
|
|
23
|
+
"devDependencies": {
|
|
24
|
+
"typescript": "5.9.3",
|
|
25
|
+
"vitest": "4.1.10"
|
|
26
|
+
}
|
|
27
|
+
}
|