@jestek-dev/scripture-engine 0.7.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +35 -0
- package/dist/config/engineVersion.d.ts +1 -1
- package/dist/config/engineVersion.js +47 -1
- package/dist/corpus/repository.d.ts +189 -2
- package/dist/corpus/repository.js +332 -1
- package/dist/createEngine.d.ts +76 -1
- package/dist/createEngine.js +716 -52
- package/dist/index.d.ts +20 -14
- package/dist/index.js +15 -13
- package/dist/intents/concept.d.ts +115 -2
- package/dist/intents/concept.js +323 -6
- package/dist/intents/lexical.d.ts +42 -3
- package/dist/intents/lexical.js +111 -11
- package/dist/intents/spelling.d.ts +93 -0
- package/dist/intents/spelling.js +124 -0
- package/dist/internal.d.ts +31 -0
- package/dist/internal.js +31 -0
- package/dist/ranking/budgets.d.ts +12 -0
- package/dist/ranking/budgets.js +39 -0
- package/dist/ranking/rank.js +20 -3
- package/dist/reasons/display.d.ts +86 -0
- package/dist/reasons/display.js +105 -0
- package/dist/reasons/types.d.ts +1 -1
- package/dist/reference/reference.d.ts +57 -0
- package/dist/reference/reference.js +210 -48
- package/dist/tokenizer/index.d.ts +35 -0
- package/dist/tokenizer/index.js +37 -1
- package/dist/types.d.ts +104 -0
- package/package.json +16 -1
package/dist/index.d.ts
CHANGED
|
@@ -1,19 +1,25 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* PUBLIC API for @jestek-dev/scripture-engine — the stable tier consumers
|
|
3
|
+
* pin (plan P7.2 / CO-5; docs/COMPATIBILITY.md is the compatibility matrix).
|
|
3
4
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* This entry is deliberately small: `createEngine` over a `ContentQueryPort`,
|
|
6
|
+
* the five methods on `ScriptureEngine` (`research`, `themes`, `passage`,
|
|
7
|
+
* `related`, `forSong`), the result types they return — including the
|
|
8
|
+
* additive `suggestion` (0.11.0) and `corrections` (0.12.0) citation fields
|
|
9
|
+
* and the `verses`/`grouping` fields (0.14.0) — and the version constants.
|
|
10
|
+
* Invalid input is a typed kind, never an exception (§5 of
|
|
11
|
+
* docs/implementation-plan.md, restated in docs/COMPATIBILITY.md).
|
|
12
|
+
*
|
|
13
|
+
* Everything else — the tokenizer, the ranker, repository internals, the
|
|
14
|
+
* reviewed constants eval mirrors — lives behind
|
|
15
|
+
* `@jestek-dev/scripture-engine/internal` and carries no stability promise.
|
|
16
|
+
* The exact public surface is pinned by eval/test/public-surface.test.ts;
|
|
17
|
+
* widening it is a §5 consumer-contract change, not a convenience edit.
|
|
8
18
|
*/
|
|
9
19
|
export { ENGINE_VERSION, TOKENIZER_VERSION } from './config/engineVersion.js';
|
|
10
20
|
export { createEngine, type EngineOptions, type ScriptureEngine } from './createEngine.js';
|
|
11
|
-
export {
|
|
12
|
-
export {
|
|
13
|
-
export {
|
|
14
|
-
export
|
|
15
|
-
export {
|
|
16
|
-
export { AUTHORITATIVE_FAMILIES, isAuthoritative, type Evidence, type Provenance, type Reason, type SignalFamily, } from './reasons/types.js';
|
|
17
|
-
export { applyBudgets, DEFAULT_BUDGETS, type BudgetedScore, type FamilyBudget, type SignalBudgets, } from './ranking/budgets.js';
|
|
18
|
-
export { DEFAULT_LIMIT, DEFAULT_MAX_PER_GROUP, rank, type Candidate, type RankedResult, type RankOptions, } from './ranking/rank.js';
|
|
19
|
-
export type { ConceptMatch, PassageResult, RelatedResult, SongInput, ContentQueryPort, ContentQueryResult, ContentScalar, DiscoveryResult, ResearchOutcome, ResearchResult, ResultIdentity, ScripturePassage, ScriptureVerse, } from './types.js';
|
|
21
|
+
export type { Evidence, Provenance, Reason, SignalFamily } from './reasons/types.js';
|
|
22
|
+
export type { ReferenceSuggestion } from './reference/reference.js';
|
|
23
|
+
export type { FamilyBudget, SignalBudgets } from './ranking/budgets.js';
|
|
24
|
+
export type { RankOptions } from './ranking/rank.js';
|
|
25
|
+
export type { ConceptMatch, ContentQueryPort, ContentQueryResult, ContentScalar, DiscoveryResult, GroupedVerse, PassageResult, RelatedResult, ResearchOutcome, ResearchResult, ResultGrouping, ResultIdentity, ScripturePassage, ScriptureVerse, SongInput, SpellingCorrection, } from './types.js';
|
package/dist/index.js
CHANGED
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* PUBLIC API for @jestek-dev/scripture-engine — the stable tier consumers
|
|
3
|
+
* pin (plan P7.2 / CO-5; docs/COMPATIBILITY.md is the compatibility matrix).
|
|
3
4
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* This entry is deliberately small: `createEngine` over a `ContentQueryPort`,
|
|
6
|
+
* the five methods on `ScriptureEngine` (`research`, `themes`, `passage`,
|
|
7
|
+
* `related`, `forSong`), the result types they return — including the
|
|
8
|
+
* additive `suggestion` (0.11.0) and `corrections` (0.12.0) citation fields
|
|
9
|
+
* and the `verses`/`grouping` fields (0.14.0) — and the version constants.
|
|
10
|
+
* Invalid input is a typed kind, never an exception (§5 of
|
|
11
|
+
* docs/implementation-plan.md, restated in docs/COMPATIBILITY.md).
|
|
12
|
+
*
|
|
13
|
+
* Everything else — the tokenizer, the ranker, repository internals, the
|
|
14
|
+
* reviewed constants eval mirrors — lives behind
|
|
15
|
+
* `@jestek-dev/scripture-engine/internal` and carries no stability promise.
|
|
16
|
+
* The exact public surface is pinned by eval/test/public-surface.test.ts;
|
|
17
|
+
* widening it is a §5 consumer-contract change, not a convenience edit.
|
|
8
18
|
*/
|
|
9
19
|
export { ENGINE_VERSION, TOKENIZER_VERSION } from './config/engineVersion.js';
|
|
10
20
|
export { createEngine } from './createEngine.js';
|
|
11
|
-
export { CorpusRepository, MAX_CANDIDATES, MAX_PHRASE_LENGTH, } from './corpus/repository.js';
|
|
12
|
-
export { groupIdFor, mergeCandidates, phraseEvidence, queryIdfTotal, referenceLabel, targetIdFor, tokenEvidence, } from './intents/lexical.js';
|
|
13
|
-
export { normalizeToken, significantWords, tokenStream, TOKENIZER_ARCHAIC_FORM_COUNT, TOKENIZER_STOPWORD_COUNT, } from './tokenizer/index.js';
|
|
14
|
-
export { normalizeBookAlias, resolveReference, resolveReferenceAttempt, } from './reference/reference.js';
|
|
15
|
-
export { makeVerseId, parseVerseId } from './reference/verseId.js';
|
|
16
|
-
export { AUTHORITATIVE_FAMILIES, isAuthoritative, } from './reasons/types.js';
|
|
17
|
-
export { applyBudgets, DEFAULT_BUDGETS, } from './ranking/budgets.js';
|
|
18
|
-
export { DEFAULT_LIMIT, DEFAULT_MAX_PER_GROUP, rank, } from './ranking/rank.js';
|
|
@@ -10,8 +10,34 @@
|
|
|
10
10
|
* the source. Where that source is us, the chip says "LH editorial" and the
|
|
11
11
|
* reader can weigh it accordingly.
|
|
12
12
|
*/
|
|
13
|
-
import type { ConceptAnchorRow, CrossReferenceRow } from '../corpus/repository.js';
|
|
13
|
+
import type { ConceptAnchorRow, CrossReferenceRow, CuratedAliasRow } from '../corpus/repository.js';
|
|
14
14
|
import type { Evidence } from '../reasons/types.js';
|
|
15
|
+
import type { ScriptureVerse } from '../types.js';
|
|
16
|
+
export declare const MIN_AUTHORITATIVE_BARE_CUE_IDF_SHARE = 0.2;
|
|
17
|
+
/**
|
|
18
|
+
* One verse, one concept, ONE scored contribution (0.10.0 stage 6).
|
|
19
|
+
*
|
|
20
|
+
* The importer emits one anchor row per (entry × source), and overlapping
|
|
21
|
+
* ranges within a concept emit one row per range — so the same verse could
|
|
22
|
+
* reach the ranker several times for the SAME concept and be summed as if
|
|
23
|
+
* the entries were independent evidence. They are not: two sources naming
|
|
24
|
+
* one verse for one theme is agreement about a single fact, and G7 files
|
|
25
|
+
* agreement as one budget, not two. This is how a duplicated 1 Peter 5:7
|
|
26
|
+
* outscored peace-of-god's own weight-1.0 anchor.
|
|
27
|
+
*
|
|
28
|
+
* Groups by (conceptId, translationCode, verseId); the carrier is chosen
|
|
29
|
+
* deterministically (weight desc → sourceId asc → locator asc → anchor start
|
|
30
|
+
* asc) and provenance is NOT dropped: the surviving row's sourceId becomes
|
|
31
|
+
* the '+'-joined ascending union of the group's sources — the same
|
|
32
|
+
* convention passage_terms already uses — so one chip honestly names every
|
|
33
|
+
* agreeing source (covenant: explanations are contract; sources are named,
|
|
34
|
+
* never adjudicated). Cross-CONCEPT stacking is deliberately untouched: two
|
|
35
|
+
* different matched concepts naming one verse are two different claims.
|
|
36
|
+
* Groups keep first-occurrence order, so unmerged inputs pass through
|
|
37
|
+
* byte-identical and the output is a pure, deterministic function of the
|
|
38
|
+
* input order (the repository's ORDER BY makes that order stable).
|
|
39
|
+
*/
|
|
40
|
+
export declare function dedupeConceptAnchors(anchors: readonly ConceptAnchorRow[]): readonly ConceptAnchorRow[];
|
|
15
41
|
/**
|
|
16
42
|
* Anchor evidence.
|
|
17
43
|
*
|
|
@@ -25,7 +51,61 @@ import type { Evidence } from '../reasons/types.js';
|
|
|
25
51
|
* evidence that the user meant the curated concept; a four-token match
|
|
26
52
|
* ("be doers of the word") is close to unambiguous.
|
|
27
53
|
*/
|
|
28
|
-
export declare function conceptAnchorEvidence(anchor: ConceptAnchorRow, matchedTokenCount: number): Evidence;
|
|
54
|
+
export declare function conceptAnchorEvidence(anchor: ConceptAnchorRow, matchedTokenCount: number, queryTokenCount?: number): Evidence;
|
|
55
|
+
/**
|
|
56
|
+
* A direct concept cue that is too thin to claim its anchors authoritatively.
|
|
57
|
+
*
|
|
58
|
+
* Bare single-word lexicon entries are good defaults for broad queries. Inside
|
|
59
|
+
* a longer query they are only a hint, so they stay visible but spend from the
|
|
60
|
+
* weak concept_lexicon budget rather than the authoritative concept_anchor one.
|
|
61
|
+
*/
|
|
62
|
+
export declare function conceptCueEvidence(anchor: ConceptAnchorRow, matchedTokenCount: number, queryTokenCount?: number): Evidence;
|
|
63
|
+
/**
|
|
64
|
+
* One-token lexicon entries are broad-query defaults.
|
|
65
|
+
*
|
|
66
|
+
* They remain authoritative when the whole query is that token. Inside a
|
|
67
|
+
* longer query they should act as theme cues.
|
|
68
|
+
*
|
|
69
|
+
* Width is measured in SIGNIFICANT tokens, not raw words. This deliberately
|
|
70
|
+
* REVERSES the earlier carve-out that excluded multi-word phrasings whose
|
|
71
|
+
* normalization reduces to one token ("forgive others" -> `forgive`): at
|
|
72
|
+
* match time such a phrase IS its single stored token, so exempting it let a
|
|
73
|
+
* stopword-heavy phrase evade the thin-cue gate a deliberate bare word must
|
|
74
|
+
* face. Legitimate remembered phrasings stay protected by the two guards this
|
|
75
|
+
* keeps: (i) `queryTokenCount > 1` — a query that itself collapses to the
|
|
76
|
+
* same lone token ("do not be afraid" -> `afraid`) stays authoritative — and
|
|
77
|
+
* (ii) the IDF-share test in isThinBareWordConceptCue, so a collapsed cue
|
|
78
|
+
* that carries real query meaning survives.
|
|
79
|
+
*/
|
|
80
|
+
export declare function isBareWordConceptCue(matchedPhrase: string, queryTokenCount: number): boolean;
|
|
81
|
+
export declare function isThinBareWordConceptCue(matchedPhrase: string, queryTokenCount: number, queryIdfShare: number): boolean;
|
|
82
|
+
/**
|
|
83
|
+
* Curated phrase/hymn alias evidence, concept-target arm (0.13.0/QR-6).
|
|
84
|
+
*
|
|
85
|
+
* Files under the EXISTING concept_anchor family — no new SignalFamily, so
|
|
86
|
+
* the reviewed budgets roster is untouched and the alias claim competes
|
|
87
|
+
* under exactly the authority a curated concept naming already has. Strength
|
|
88
|
+
* is the alias row's weight (the editorial prior that this whole query
|
|
89
|
+
* names this hymn and this hymn names this theme) times the anchor's own
|
|
90
|
+
* curated weight — the anchors keep their reviewed ordering among
|
|
91
|
+
* themselves. No specificity or coverage discount applies because whole-
|
|
92
|
+
* query EQUALITY matching is full-query parity by construction: the query
|
|
93
|
+
* IS the phrase.
|
|
94
|
+
*
|
|
95
|
+
* The label carries the full attribution chain — hymn title, then the theme
|
|
96
|
+
* — because the explanation is the contract (covenant 5) and the chip must
|
|
97
|
+
* say on whose word the connection stands (covenant 6: a curated source
|
|
98
|
+
* names it; the engine adjudicates nothing).
|
|
99
|
+
*/
|
|
100
|
+
export declare function aliasConceptEvidence(alias: CuratedAliasRow, anchor: ConceptAnchorRow): Evidence;
|
|
101
|
+
/**
|
|
102
|
+
* Alias evidence, verse-range arm: the alias names an explicit passage
|
|
103
|
+
* rather than a concept (the schema XOR's other side — for a hymn whose
|
|
104
|
+
* scriptural basis is one passage no curated concept represents). Same
|
|
105
|
+
* family, same authority reasoning; the passage label itself is the result
|
|
106
|
+
* row's reference, so the chip carries only the hymn attribution.
|
|
107
|
+
*/
|
|
108
|
+
export declare function aliasPassageEvidence(alias: CuratedAliasRow, _verse: ScriptureVerse): Evidence;
|
|
29
109
|
/**
|
|
30
110
|
* Evidence for a concept reached one hop away in the curated graph.
|
|
31
111
|
*
|
|
@@ -53,6 +133,39 @@ export declare function crossReferenceEvidence(edge: CrossReferenceRow, maxVotes
|
|
|
53
133
|
* honest about what it is — a hint that people preaching this passage reach
|
|
54
134
|
* for these words, nothing stronger.
|
|
55
135
|
*/
|
|
136
|
+
/**
|
|
137
|
+
* The verse reads this way in another English translation.
|
|
138
|
+
*
|
|
139
|
+
* Distinct from token_overlap, which matches the SHIPPED wording, and from
|
|
140
|
+
* passage_terms, which is what expositors said ABOUT a verse. This is the
|
|
141
|
+
* verse itself, worded differently — the case where somebody learned Jeremiah
|
|
142
|
+
* 29:11 as "plans to prosper you" and the shipped text says "thoughts of
|
|
143
|
+
* peace".
|
|
144
|
+
*
|
|
145
|
+
* Strength saturates on how many query stems the verse accounts for. Matching
|
|
146
|
+
* one stem is weak (many verses contain `plan`); matching several is a strong
|
|
147
|
+
* signal that this is the verse being remembered.
|
|
148
|
+
*
|
|
149
|
+
* The label deliberately does not name a translation. The stored stems are
|
|
150
|
+
* merged across sources, so which translation contributed a given one is not
|
|
151
|
+
* recoverable — and claiming otherwise would be inventing provenance we do
|
|
152
|
+
* not have.
|
|
153
|
+
*/
|
|
154
|
+
export declare function translationVariantEvidence(match: {
|
|
155
|
+
matchedTokens: readonly string[];
|
|
156
|
+
}, queryIdfTotal: number, documentFrequencies: ReadonlyMap<string, number>, documentCount: number): Evidence | null;
|
|
157
|
+
/**
|
|
158
|
+
* Half-saturation constant for the passage_terms PMI factor (0.10.0 stage 5).
|
|
159
|
+
*
|
|
160
|
+
* A match whose mean per-term PMI equals this value scores factor 0.5. The
|
|
161
|
+
* value is calibrated against reviewed data: the G5 admission floor
|
|
162
|
+
* (`eval/budgets.json` distinctiveness.minPmi = 2.0) scores 0.25 — a term
|
|
163
|
+
* that barely cleared admission speaks at quarter volume — and the measured
|
|
164
|
+
* corpus-mean PMI 6.37 scores ≈0.52. Mirrored into `eval/budgets.json`
|
|
165
|
+
* signalBudgets for the G6 reviewed-constants check; value pending J21
|
|
166
|
+
* sign-off, which rides the normal approval flow.
|
|
167
|
+
*/
|
|
168
|
+
export declare const PASSAGE_TERM_PMI_HALF_SATURATION = 6;
|
|
56
169
|
export declare function passageTermEvidence(match: {
|
|
57
170
|
matchedTerms: readonly string[];
|
|
58
171
|
pmiSum: number;
|
package/dist/intents/concept.js
CHANGED
|
@@ -10,6 +10,105 @@
|
|
|
10
10
|
* the source. Where that source is us, the chip says "LH editorial" and the
|
|
11
11
|
* reader can weigh it accordingly.
|
|
12
12
|
*/
|
|
13
|
+
import { significantWords } from '../tokenizer/index.js';
|
|
14
|
+
function conceptSpecificity(matchedTokenCount) {
|
|
15
|
+
return Math.min(1, 0.55 + 0.15 * Math.max(0, matchedTokenCount - 1));
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* How much of the QUERY this concept explains.
|
|
19
|
+
*
|
|
20
|
+
* Specificity alone asks "how much of the concept's phrase matched"; it
|
|
21
|
+
* cannot tell "love" (the whole query) from "love" inside "do justly love
|
|
22
|
+
* mercy walk humbly". Once bare words entered the lexicons those became very
|
|
23
|
+
* different claims, and without this the second one buried Micah 6:8 under
|
|
24
|
+
* God's-love passages — a concept explaining one word of six speaking as
|
|
25
|
+
* loudly as one explaining all of it.
|
|
26
|
+
*
|
|
27
|
+
* Square-rooted rather than linear: a concept that explains half a query is
|
|
28
|
+
* still saying something substantial, and a linear penalty would mute
|
|
29
|
+
* legitimate multi-word matches to chase a single-word failure.
|
|
30
|
+
*/
|
|
31
|
+
function conceptCoverage(matchedTokenCount, queryTokenCount) {
|
|
32
|
+
return queryTokenCount > 0 ? Math.sqrt(Math.min(1, matchedTokenCount / queryTokenCount)) : 1;
|
|
33
|
+
}
|
|
34
|
+
function conceptMatchStrength(anchorWeight, matchedTokenCount, queryTokenCount) {
|
|
35
|
+
// Full-query parity (0.10.0 stage 3): when the concept's normalized phrase
|
|
36
|
+
// accounts for EVERY significant token of the query — count equality
|
|
37
|
+
// implies set equality, because matchConcepts only matches phrases whose
|
|
38
|
+
// normalized tokens all occur among the query tokens — and is at least two
|
|
39
|
+
// tokens wide, specificity is 1. The query IS the phrase; docking it for
|
|
40
|
+
// being stored as a two-token entry made a weight-1.0 anchor worth 28 of
|
|
41
|
+
// its 40 points and let a tapered verbatim rebuke outrank the instituting
|
|
42
|
+
// passage a human curated. One- and zero-token matches keep the graded
|
|
43
|
+
// specificity: parity for a bare word would undo the thin-cue design.
|
|
44
|
+
// Deliberate, measured consequence: a stopword-heavy query that collapses
|
|
45
|
+
// to the same significant tokens as a remembered-phrasing lexicon entry
|
|
46
|
+
// gets parity — there is one tokenizer, and a query whose meaning collapses
|
|
47
|
+
// to two tokens IS a two-token query.
|
|
48
|
+
const parity = matchedTokenCount >= 2 && matchedTokenCount === queryTokenCount;
|
|
49
|
+
return (Math.max(0, Math.min(1, anchorWeight)) *
|
|
50
|
+
(parity ? 1 : conceptSpecificity(matchedTokenCount)) *
|
|
51
|
+
conceptCoverage(matchedTokenCount, queryTokenCount));
|
|
52
|
+
}
|
|
53
|
+
// A bare-word cue that explains less than this share of the query's meaning
|
|
54
|
+
// is too thin to activate a concept's full anchor set authoritatively.
|
|
55
|
+
export const MIN_AUTHORITATIVE_BARE_CUE_IDF_SHARE = 0.2;
|
|
56
|
+
/**
|
|
57
|
+
* One verse, one concept, ONE scored contribution (0.10.0 stage 6).
|
|
58
|
+
*
|
|
59
|
+
* The importer emits one anchor row per (entry × source), and overlapping
|
|
60
|
+
* ranges within a concept emit one row per range — so the same verse could
|
|
61
|
+
* reach the ranker several times for the SAME concept and be summed as if
|
|
62
|
+
* the entries were independent evidence. They are not: two sources naming
|
|
63
|
+
* one verse for one theme is agreement about a single fact, and G7 files
|
|
64
|
+
* agreement as one budget, not two. This is how a duplicated 1 Peter 5:7
|
|
65
|
+
* outscored peace-of-god's own weight-1.0 anchor.
|
|
66
|
+
*
|
|
67
|
+
* Groups by (conceptId, translationCode, verseId); the carrier is chosen
|
|
68
|
+
* deterministically (weight desc → sourceId asc → locator asc → anchor start
|
|
69
|
+
* asc) and provenance is NOT dropped: the surviving row's sourceId becomes
|
|
70
|
+
* the '+'-joined ascending union of the group's sources — the same
|
|
71
|
+
* convention passage_terms already uses — so one chip honestly names every
|
|
72
|
+
* agreeing source (covenant: explanations are contract; sources are named,
|
|
73
|
+
* never adjudicated). Cross-CONCEPT stacking is deliberately untouched: two
|
|
74
|
+
* different matched concepts naming one verse are two different claims.
|
|
75
|
+
* Groups keep first-occurrence order, so unmerged inputs pass through
|
|
76
|
+
* byte-identical and the output is a pure, deterministic function of the
|
|
77
|
+
* input order (the repository's ORDER BY makes that order stable).
|
|
78
|
+
*/
|
|
79
|
+
export function dedupeConceptAnchors(anchors) {
|
|
80
|
+
const groups = new Map();
|
|
81
|
+
const order = [];
|
|
82
|
+
for (const anchor of anchors) {
|
|
83
|
+
// U+0000 as the WRITTEN escape (never the raw byte, which turns a source
|
|
84
|
+
// file git-binary): no id component can contain it, so the joined key
|
|
85
|
+
// cannot collide across components.
|
|
86
|
+
const key = `${anchor.conceptId}\u0000${anchor.translationCode}\u0000${anchor.verseId}`;
|
|
87
|
+
const bucket = groups.get(key);
|
|
88
|
+
if (bucket)
|
|
89
|
+
bucket.push(anchor);
|
|
90
|
+
else {
|
|
91
|
+
groups.set(key, [anchor]);
|
|
92
|
+
order.push(key);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return order.map((key) => {
|
|
96
|
+
const group = groups.get(key);
|
|
97
|
+
if (group.length === 1)
|
|
98
|
+
return group[0];
|
|
99
|
+
const sorted = [...group].sort((a, b) => b.weight - a.weight ||
|
|
100
|
+
(a.sourceId < b.sourceId ? -1 : a.sourceId > b.sourceId ? 1 : 0) ||
|
|
101
|
+
((a.locator ?? '') < (b.locator ?? '')
|
|
102
|
+
? -1
|
|
103
|
+
: (a.locator ?? '') > (b.locator ?? '')
|
|
104
|
+
? 1
|
|
105
|
+
: 0) ||
|
|
106
|
+
a.anchorStartVerseId - b.anchorStartVerseId);
|
|
107
|
+
const carrier = sorted[0];
|
|
108
|
+
const sourceIds = [...new Set(group.map((row) => row.sourceId))].sort();
|
|
109
|
+
return sourceIds.length === 1 ? carrier : { ...carrier, sourceId: sourceIds.join('+') };
|
|
110
|
+
});
|
|
111
|
+
}
|
|
13
112
|
/**
|
|
14
113
|
* Anchor evidence.
|
|
15
114
|
*
|
|
@@ -23,20 +122,116 @@
|
|
|
23
122
|
* evidence that the user meant the curated concept; a four-token match
|
|
24
123
|
* ("be doers of the word") is close to unambiguous.
|
|
25
124
|
*/
|
|
26
|
-
export function conceptAnchorEvidence(anchor, matchedTokenCount) {
|
|
27
|
-
const specificity = Math.min(1, 0.55 + 0.15 * Math.max(0, matchedTokenCount - 1));
|
|
125
|
+
export function conceptAnchorEvidence(anchor, matchedTokenCount, queryTokenCount = matchedTokenCount) {
|
|
28
126
|
return {
|
|
29
127
|
family: 'concept_anchor',
|
|
30
128
|
label: `Theme: ${anchor.conceptLabel}`,
|
|
31
|
-
strength:
|
|
129
|
+
strength: conceptMatchStrength(anchor.weight, matchedTokenCount, queryTokenCount),
|
|
32
130
|
provenance: {
|
|
33
131
|
sourceId: anchor.sourceId,
|
|
34
|
-
|
|
132
|
+
// Rendered through joinedSourceLabel so a stage-6 merged row ('a+b')
|
|
133
|
+
// names every agreeing source; single ids render byte-identically.
|
|
134
|
+
label: joinedSourceLabel(anchor.sourceId),
|
|
35
135
|
...(anchor.locator ? { locator: anchor.locator } : {}),
|
|
36
136
|
weight: anchor.weight,
|
|
37
137
|
},
|
|
38
138
|
};
|
|
39
139
|
}
|
|
140
|
+
/**
|
|
141
|
+
* A direct concept cue that is too thin to claim its anchors authoritatively.
|
|
142
|
+
*
|
|
143
|
+
* Bare single-word lexicon entries are good defaults for broad queries. Inside
|
|
144
|
+
* a longer query they are only a hint, so they stay visible but spend from the
|
|
145
|
+
* weak concept_lexicon budget rather than the authoritative concept_anchor one.
|
|
146
|
+
*/
|
|
147
|
+
export function conceptCueEvidence(anchor, matchedTokenCount, queryTokenCount = matchedTokenCount) {
|
|
148
|
+
return {
|
|
149
|
+
family: 'concept_lexicon',
|
|
150
|
+
label: `Theme cue: ${anchor.conceptLabel}`,
|
|
151
|
+
strength: conceptMatchStrength(anchor.weight, matchedTokenCount, queryTokenCount),
|
|
152
|
+
provenance: {
|
|
153
|
+
sourceId: anchor.sourceId,
|
|
154
|
+
label: joinedSourceLabel(anchor.sourceId),
|
|
155
|
+
...(anchor.locator ? { locator: anchor.locator } : {}),
|
|
156
|
+
weight: anchor.weight,
|
|
157
|
+
},
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* One-token lexicon entries are broad-query defaults.
|
|
162
|
+
*
|
|
163
|
+
* They remain authoritative when the whole query is that token. Inside a
|
|
164
|
+
* longer query they should act as theme cues.
|
|
165
|
+
*
|
|
166
|
+
* Width is measured in SIGNIFICANT tokens, not raw words. This deliberately
|
|
167
|
+
* REVERSES the earlier carve-out that excluded multi-word phrasings whose
|
|
168
|
+
* normalization reduces to one token ("forgive others" -> `forgive`): at
|
|
169
|
+
* match time such a phrase IS its single stored token, so exempting it let a
|
|
170
|
+
* stopword-heavy phrase evade the thin-cue gate a deliberate bare word must
|
|
171
|
+
* face. Legitimate remembered phrasings stay protected by the two guards this
|
|
172
|
+
* keeps: (i) `queryTokenCount > 1` — a query that itself collapses to the
|
|
173
|
+
* same lone token ("do not be afraid" -> `afraid`) stays authoritative — and
|
|
174
|
+
* (ii) the IDF-share test in isThinBareWordConceptCue, so a collapsed cue
|
|
175
|
+
* that carries real query meaning survives.
|
|
176
|
+
*/
|
|
177
|
+
export function isBareWordConceptCue(matchedPhrase, queryTokenCount) {
|
|
178
|
+
return queryTokenCount > 1 && significantWords(matchedPhrase).length <= 1;
|
|
179
|
+
}
|
|
180
|
+
export function isThinBareWordConceptCue(matchedPhrase, queryTokenCount, queryIdfShare) {
|
|
181
|
+
return (isBareWordConceptCue(matchedPhrase, queryTokenCount) &&
|
|
182
|
+
queryIdfShare < MIN_AUTHORITATIVE_BARE_CUE_IDF_SHARE);
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Curated phrase/hymn alias evidence, concept-target arm (0.13.0/QR-6).
|
|
186
|
+
*
|
|
187
|
+
* Files under the EXISTING concept_anchor family — no new SignalFamily, so
|
|
188
|
+
* the reviewed budgets roster is untouched and the alias claim competes
|
|
189
|
+
* under exactly the authority a curated concept naming already has. Strength
|
|
190
|
+
* is the alias row's weight (the editorial prior that this whole query
|
|
191
|
+
* names this hymn and this hymn names this theme) times the anchor's own
|
|
192
|
+
* curated weight — the anchors keep their reviewed ordering among
|
|
193
|
+
* themselves. No specificity or coverage discount applies because whole-
|
|
194
|
+
* query EQUALITY matching is full-query parity by construction: the query
|
|
195
|
+
* IS the phrase.
|
|
196
|
+
*
|
|
197
|
+
* The label carries the full attribution chain — hymn title, then the theme
|
|
198
|
+
* — because the explanation is the contract (covenant 5) and the chip must
|
|
199
|
+
* say on whose word the connection stands (covenant 6: a curated source
|
|
200
|
+
* names it; the engine adjudicates nothing).
|
|
201
|
+
*/
|
|
202
|
+
export function aliasConceptEvidence(alias, anchor) {
|
|
203
|
+
return {
|
|
204
|
+
family: 'concept_anchor',
|
|
205
|
+
label: `Hymn: "${alias.title}" → Theme: ${anchor.conceptLabel}`,
|
|
206
|
+
strength: Math.max(0, Math.min(1, alias.weight)) * Math.max(0, Math.min(1, anchor.weight)),
|
|
207
|
+
provenance: {
|
|
208
|
+
sourceId: alias.sourceId,
|
|
209
|
+
label: sourceLabel(alias.sourceId),
|
|
210
|
+
...(alias.locator ? { locator: alias.locator } : {}),
|
|
211
|
+
weight: alias.weight,
|
|
212
|
+
},
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Alias evidence, verse-range arm: the alias names an explicit passage
|
|
217
|
+
* rather than a concept (the schema XOR's other side — for a hymn whose
|
|
218
|
+
* scriptural basis is one passage no curated concept represents). Same
|
|
219
|
+
* family, same authority reasoning; the passage label itself is the result
|
|
220
|
+
* row's reference, so the chip carries only the hymn attribution.
|
|
221
|
+
*/
|
|
222
|
+
export function aliasPassageEvidence(alias, _verse) {
|
|
223
|
+
return {
|
|
224
|
+
family: 'concept_anchor',
|
|
225
|
+
label: `Hymn: "${alias.title}"`,
|
|
226
|
+
strength: Math.max(0, Math.min(1, alias.weight)),
|
|
227
|
+
provenance: {
|
|
228
|
+
sourceId: alias.sourceId,
|
|
229
|
+
label: sourceLabel(alias.sourceId),
|
|
230
|
+
...(alias.locator ? { locator: alias.locator } : {}),
|
|
231
|
+
weight: alias.weight,
|
|
232
|
+
},
|
|
233
|
+
};
|
|
234
|
+
}
|
|
40
235
|
/**
|
|
41
236
|
* Evidence for a concept reached one hop away in the curated graph.
|
|
42
237
|
*
|
|
@@ -52,7 +247,7 @@ export function relatedConceptEvidence(anchor) {
|
|
|
52
247
|
strength: Math.max(0, Math.min(1, anchor.weight)) * 0.5,
|
|
53
248
|
provenance: {
|
|
54
249
|
sourceId: anchor.sourceId,
|
|
55
|
-
label:
|
|
250
|
+
label: joinedSourceLabel(anchor.sourceId),
|
|
56
251
|
...(anchor.locator ? { locator: anchor.locator } : {}),
|
|
57
252
|
},
|
|
58
253
|
};
|
|
@@ -88,6 +283,64 @@ export function crossReferenceEvidence(edge, maxVotes, fromReference) {
|
|
|
88
283
|
* honest about what it is — a hint that people preaching this passage reach
|
|
89
284
|
* for these words, nothing stronger.
|
|
90
285
|
*/
|
|
286
|
+
/**
|
|
287
|
+
* The verse reads this way in another English translation.
|
|
288
|
+
*
|
|
289
|
+
* Distinct from token_overlap, which matches the SHIPPED wording, and from
|
|
290
|
+
* passage_terms, which is what expositors said ABOUT a verse. This is the
|
|
291
|
+
* verse itself, worded differently — the case where somebody learned Jeremiah
|
|
292
|
+
* 29:11 as "plans to prosper you" and the shipped text says "thoughts of
|
|
293
|
+
* peace".
|
|
294
|
+
*
|
|
295
|
+
* Strength saturates on how many query stems the verse accounts for. Matching
|
|
296
|
+
* one stem is weak (many verses contain `plan`); matching several is a strong
|
|
297
|
+
* signal that this is the verse being remembered.
|
|
298
|
+
*
|
|
299
|
+
* The label deliberately does not name a translation. The stored stems are
|
|
300
|
+
* merged across sources, so which translation contributed a given one is not
|
|
301
|
+
* recoverable — and claiming otherwise would be inventing provenance we do
|
|
302
|
+
* not have.
|
|
303
|
+
*/
|
|
304
|
+
export function translationVariantEvidence(match, queryIdfTotal, documentFrequencies, documentCount) {
|
|
305
|
+
// A SINGLE stem is not evidence. Nearly every verse has some alternate
|
|
306
|
+
// rendering of some common word, so one match says only "this verse exists".
|
|
307
|
+
// Requiring two is what separates "somebody is quoting this verse from
|
|
308
|
+
// another translation" from coincidence — and skipping the first version of
|
|
309
|
+
// this check is what displaced 90% of the top ten on unrelated probes.
|
|
310
|
+
if (match.matchedTokens.length < 2)
|
|
311
|
+
return null;
|
|
312
|
+
// Weighted by rarity, exactly as token_overlap is. Matching `prosper` says
|
|
313
|
+
// far more than matching `help`, and an unweighted count cannot tell them
|
|
314
|
+
// apart.
|
|
315
|
+
const idfSum = match.matchedTokens.reduce((sum, token) => {
|
|
316
|
+
const df = documentFrequencies.get(token) ?? 0;
|
|
317
|
+
return sum + Math.log(1 + documentCount / Math.max(1, df));
|
|
318
|
+
}, 0);
|
|
319
|
+
const coverage = queryIdfTotal > 0 ? Math.min(1, idfSum / queryIdfTotal) : 0;
|
|
320
|
+
if (coverage <= 0)
|
|
321
|
+
return null;
|
|
322
|
+
return {
|
|
323
|
+
family: 'translation_variant',
|
|
324
|
+
label: `Worded this way in another translation: ${match.matchedTokens.slice(0, 3).join(', ')}`,
|
|
325
|
+
strength: Math.max(0, Math.min(1, coverage)),
|
|
326
|
+
provenance: {
|
|
327
|
+
sourceId: 'translation-variants',
|
|
328
|
+
label: 'Cross-translation vocabulary',
|
|
329
|
+
},
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Half-saturation constant for the passage_terms PMI factor (0.10.0 stage 5).
|
|
334
|
+
*
|
|
335
|
+
* A match whose mean per-term PMI equals this value scores factor 0.5. The
|
|
336
|
+
* value is calibrated against reviewed data: the G5 admission floor
|
|
337
|
+
* (`eval/budgets.json` distinctiveness.minPmi = 2.0) scores 0.25 — a term
|
|
338
|
+
* that barely cleared admission speaks at quarter volume — and the measured
|
|
339
|
+
* corpus-mean PMI 6.37 scores ≈0.52. Mirrored into `eval/budgets.json`
|
|
340
|
+
* signalBudgets for the G6 reviewed-constants check; value pending J21
|
|
341
|
+
* sign-off, which rides the normal approval flow.
|
|
342
|
+
*/
|
|
343
|
+
export const PASSAGE_TERM_PMI_HALF_SATURATION = 6.0;
|
|
91
344
|
export function passageTermEvidence(match) {
|
|
92
345
|
const saturating = Math.log1p(match.matchedTerms.length) / Math.log1p(6);
|
|
93
346
|
// Specificity: evidence distilled from a one-verse note is a stronger claim
|
|
@@ -96,12 +349,28 @@ export function passageTermEvidence(match) {
|
|
|
96
349
|
// purpose: diffuse commentary is discounted, never discarded.
|
|
97
350
|
const span = Math.max(1, match.minSpanVerses);
|
|
98
351
|
const specificity = 1 / (1 + 0.25 * Math.log2(span));
|
|
352
|
+
// Distinctiveness (0.10.0 stage 5): the pipeline computed a PMI for every
|
|
353
|
+
// admitted term under G5, and until this factor existed scoring threw that
|
|
354
|
+
// statistic away — every same-count, same-span match tied exactly and
|
|
355
|
+
// canonical book order decided ("propitiation" returned a flat 2.85 pile
|
|
356
|
+
// led by whichever book comes first). Asymptotic on purpose: the earlier
|
|
357
|
+
// `min(1, pmiSum / (terms × 6))` form saturates at 1 for any per-term PMI
|
|
358
|
+
// ≥ 6, and distinctive vocabulary lives ABOVE that line (measured corpus
|
|
359
|
+
// range 2.02–18.54, every stored `propitiation` row 8.52–11.21), so the
|
|
360
|
+
// ties it existed to break survived it byte-identical. This form is
|
|
361
|
+
// strictly monotone in pmiSum — distinct pmiSums never tie, at any
|
|
362
|
+
// magnitude — and bounded below 1. No new adjudication: the statistic was
|
|
363
|
+
// already reviewed data; this only stops discarding it. Zero/absent pmiSum
|
|
364
|
+
// degrades to factor 0 rather than NaN or a negative.
|
|
365
|
+
const pmiSum = Math.max(0, match.pmiSum);
|
|
366
|
+
const halfSaturationMass = Math.max(1, match.matchedTerms.length) * PASSAGE_TERM_PMI_HALF_SATURATION;
|
|
367
|
+
const pmiFactor = pmiSum / (pmiSum + halfSaturationMass);
|
|
99
368
|
return {
|
|
100
369
|
family: 'passage_terms',
|
|
101
370
|
label: match.matchedTerms.length === 1
|
|
102
371
|
? `Preached vocabulary: ${match.matchedTerms[0]}`
|
|
103
372
|
: `Preached vocabulary: ${match.matchedTerms.slice(0, 3).join(', ')}`,
|
|
104
|
-
strength: Math.max(0, Math.min(1, saturating * specificity)),
|
|
373
|
+
strength: Math.max(0, Math.min(1, saturating * specificity * pmiFactor)),
|
|
105
374
|
provenance: {
|
|
106
375
|
sourceId: match.sourceIds,
|
|
107
376
|
label: joinedSourceLabel(match.sourceIds),
|
|
@@ -125,14 +394,62 @@ function sourceLabel(sourceId) {
|
|
|
125
394
|
switch (sourceId) {
|
|
126
395
|
case 'editorial':
|
|
127
396
|
return 'LH editorial';
|
|
397
|
+
case 'hymn-aliases':
|
|
398
|
+
// The QR-6 curated hymn/phrase alias pack: hand-authored rows over
|
|
399
|
+
// public-domain hymns, reviewed like any concept pack (J13). The label
|
|
400
|
+
// says both halves — whose judgment (ours) and what class of source
|
|
401
|
+
// (a public-domain hymn index) — so a reader can weigh it.
|
|
402
|
+
return 'LH editorial (public-domain hymn index)';
|
|
128
403
|
case 'openbible-topics':
|
|
129
404
|
return 'OpenBible topical votes (CC BY)';
|
|
130
405
|
case 'openbible-xrefs':
|
|
131
406
|
return 'OpenBible cross-references (CC BY)';
|
|
407
|
+
case 'openbible-sections':
|
|
408
|
+
// P5.6 (CO-3): pericope grouping provenance. Unreachable until the
|
|
409
|
+
// PR 2 behavior emits grouped results; labeled from day one so the
|
|
410
|
+
// capability PR ships a complete display mapping.
|
|
411
|
+
return 'OpenBible section boundaries (CC BY)';
|
|
412
|
+
case 'tsk':
|
|
413
|
+
return 'Treasury of Scripture Knowledge (public domain)';
|
|
414
|
+
case 'tsk-text':
|
|
415
|
+
// P6.3 (B3): the phrase-keyed TSK module admission. Unreachable in
|
|
416
|
+
// Phase A — no tsk-text row enters cross_references and nothing reads
|
|
417
|
+
// the phrase table at query time — but labeled from day one so the
|
|
418
|
+
// capability ships a complete display mapping (the openbible-sections
|
|
419
|
+
// precedent). Phase B's named-phrase chip composes its own wording;
|
|
420
|
+
// this is the bare provenance label.
|
|
421
|
+
return 'Treasury of Scripture Knowledge (public domain)';
|
|
422
|
+
case 'stepbible-tvtms':
|
|
423
|
+
// P6.4 (B5) S1: the TVTMS versification witness. ZERO SHIPPED BYTES —
|
|
424
|
+
// nothing from it enters the artifact, so no chip can ever cite it;
|
|
425
|
+
// labeled anyway because the completeness contract is "every manifest
|
|
426
|
+
// id renders as prose", with no reachability carve-outs to reason
|
|
427
|
+
// about. CC BY attribution: credit "STEP Bible" (www.STEPBible.org).
|
|
428
|
+
return 'STEP Bible versification data (CC BY)';
|
|
429
|
+
case 'web':
|
|
430
|
+
return 'World English Bible (public domain)';
|
|
431
|
+
case 'torrey':
|
|
432
|
+
return 'Torrey, New Topical Textbook (public domain)';
|
|
433
|
+
case 'nave':
|
|
434
|
+
return "Nave's Topical Bible (public domain)";
|
|
435
|
+
case 'translation-variants':
|
|
436
|
+
return 'Cross-translation vocabulary index';
|
|
132
437
|
case 'maclaren-psalms':
|
|
438
|
+
case 'maclaren-mark':
|
|
133
439
|
return 'Maclaren, Expositions (public domain)';
|
|
440
|
+
case 'clarke':
|
|
441
|
+
return 'Adam Clarke, Commentary (public domain)';
|
|
442
|
+
case 'mhc':
|
|
443
|
+
return 'Matthew Henry, Commentary (public domain)';
|
|
444
|
+
case 'kd':
|
|
445
|
+
return 'Keil & Delitzsch, OT Commentary (public domain)';
|
|
446
|
+
case 'barnes':
|
|
447
|
+
return 'Barnes, Notes on the New Testament (public domain)';
|
|
448
|
+
case 'jfb':
|
|
449
|
+
return 'Jamieson-Fausset-Brown, Commentary (public domain)';
|
|
134
450
|
case 'treasury-of-david-01':
|
|
135
451
|
case 'treasury-of-david-02':
|
|
452
|
+
case 'treasury-of-david-03':
|
|
136
453
|
case 'treasury-of-david-04':
|
|
137
454
|
case 'treasury-of-david-06':
|
|
138
455
|
return 'Spurgeon, Treasury of David (public domain)';
|