@bendyline/gezel-knowledge 1.2.11 → 1.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.d.ts +77 -0
- package/dist/browser.js +1410 -0
- package/dist/index.d.ts +16 -330
- package/dist/index.js +66 -19
- package/dist/registry-xi2R42Av.d.ts +372 -0
- package/package.json +6 -2
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
export { A as AsymmetricHit, B as BGE_SMALL_EN_V15_1, C as CharChunkerOptions, a as ChunkerOptions, E as EMBEDDINGGEMMA_2_512_1, G as GEZEL_BGE_SMALL_EN_V15_1, b as GEZEL_MARKDOWN_CHUNKS_2, c as GEZEL_MULTILINGUAL_E5_SMALL_1, K as KNOWLEDGE_EMBEDDING_PROFILES, d as KNOWLEDGE_VECTOR_FLOORS, M as MARKDOWN_CHUNKS_2, e as MULTILINGUAL_E5_SMALL_1, f as MULTILINGUAL_E5_SMALL_2, g as MarkdownChunk, P as PassageEmbedInput, S as ShardBitIndex, T as TokenChunkerOptions, h as asymmetricTopK, i as buildContextHeader, j as chunkMarkdownProfile, k as knowledgeEmbeddingProfile, l as knowledgeProfileQueryModelBytes, p as passageEmbedInput } from './registry-xi2R42Av.js';
|
|
2
|
+
export { QUERY_STOP_WORDS } from '@bendyline/gezk';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The one document-FTS query. The compiler's seal-time smoke verification
|
|
6
|
+
* and the validator's install-time smoke check MUST agree on semantics
|
|
7
|
+
* (same MATCH, same ranking, same limit interpretation) — the arts-pilot
|
|
8
|
+
* incident was a smoke query that had never been executed until a user's
|
|
9
|
+
* install ran it. Sharing the query is what keeps the two in lockstep:
|
|
10
|
+
* `documentFtsTopIds` takes the raw query and builds its own MATCH.
|
|
11
|
+
*
|
|
12
|
+
* Ranking is NAME-first, because this index is search's exact-name arm
|
|
13
|
+
* (the service fuses it as `docFts` next to the vector and chunk-body arms,
|
|
14
|
+
* which carry descriptive queries):
|
|
15
|
+
*
|
|
16
|
+
* 1. A title equal to the query (case-insensitive) comes first.
|
|
17
|
+
* Plain BM25 cannot promise that: "Quiche" ranked behind "Coronation
|
|
18
|
+
* quiche" and "Lasagna" behind "Matzo lasagna".
|
|
19
|
+
* 2. Then titles the query NAMES (`namedTitleMatches`): every word of the
|
|
20
|
+
* title appears in the query, and at least one is not a function word —
|
|
21
|
+
* "ABBA" in "Who were the members of ABBA?". The most specific name
|
|
22
|
+
* first: summed IDF of the title's words over the title column, a word
|
|
23
|
+
* the asker capitalized mid-sentence counting double. Without this tier
|
|
24
|
+
* the question's function words decided: a vandalised alias reading
|
|
25
|
+
* "…who were the disciples…" put "Romans (group)" first, and ABBA was
|
|
26
|
+
* not in the top 6. Measured on the music catalog, the ABBA article went
|
|
27
|
+
* from absent to first; "What is <title>?" lookups improved alike.
|
|
28
|
+
* 3. Then BM25 with the title weighted 10×, aliases 5×, the summary 1×.
|
|
29
|
+
* FTS5's default `rank` weights every column alike, so in a series of
|
|
30
|
+
* near-identical titles the siblings whose short summary repeats the
|
|
31
|
+
* title's words outranked the page asked for: "1944 Republican Party
|
|
32
|
+
* vice presidential candidate selection" was not in its own top 5 —
|
|
33
|
+
* a dozen sibling selections were — and neither was "1903 Nobel Prize
|
|
34
|
+
* in Literature". Measured on six Wikipedia catalogs (2,000 sampled
|
|
35
|
+
* titles each), title lookup top-1 went from 91–95% to 96–98% and
|
|
36
|
+
* top-5 from 97.8–99.6% to 98.8–100%; 3/1/2 and 5/1/3 helped less.
|
|
37
|
+
* This tier matches the query's content words only: request filler
|
|
38
|
+
* (`QUERY_STOP_WORDS`) ranked a title like "What can you do with X" on
|
|
39
|
+
* "can", "you" and "with" alone.
|
|
40
|
+
*
|
|
41
|
+
* Reader-side, so it applies to every catalog already published — the index
|
|
42
|
+
* itself is unchanged. A recorded smoke query is a document's own title, so
|
|
43
|
+
* tier 1 keeps it first under every tier added since: tiers 1 and 2 read the
|
|
44
|
+
* raw query, and that title holds every word tier 3 matches.
|
|
45
|
+
*
|
|
46
|
+
* The named tier also drives title-assisted routing
|
|
47
|
+
* (`CatalogHandle.titleRouteShards`): the shard holding a page the question
|
|
48
|
+
* names is scanned even when its centroids rank it outside the budget.
|
|
49
|
+
*
|
|
50
|
+
* Throws on invalid FTS5 MATCH syntax — callers decide whether that is a
|
|
51
|
+
* build failure (compiler) or an empty result (user-facing search).
|
|
52
|
+
*/
|
|
53
|
+
|
|
54
|
+
/** Injection-safe FTS5 query over the content words: quoted, OR'd, capped. */
|
|
55
|
+
declare function contentFtsQuery(query: string): string | null;
|
|
56
|
+
/**
|
|
57
|
+
* The chunk-body FTS expression: the content words minus the ones that cost
|
|
58
|
+
* a scan nearly everything and cannot change its order.
|
|
59
|
+
*
|
|
60
|
+
* FTS5's bm25 clamps the IDF of a term found in more than half the rows to
|
|
61
|
+
* 1e-6, yet an OR still ranks every row that term matches. In a 200k-chunk
|
|
62
|
+
* Azure-docs shard "azure" is in 82% of rows, and "Can you tell me how to
|
|
63
|
+
* integrate Azure search with blob storage?" spent ~240 ms per shard ranking
|
|
64
|
+
* it and the function words around it. Next to the vector arm that blew the
|
|
65
|
+
* chat turn's 600 ms knowledge budget, so the catalog answered nothing.
|
|
66
|
+
* Without those terms the same scan takes ~20 ms.
|
|
67
|
+
*
|
|
68
|
+
* Request filler goes first (`contentTokens`), then any term in more than
|
|
69
|
+
* half of `rows`. Neither empties the query: a query of only filler keeps it,
|
|
70
|
+
* and one whose every term is that common keeps its rarest. The title index
|
|
71
|
+
* skips the second step: one row per document is cheap to rank.
|
|
72
|
+
*/
|
|
73
|
+
declare function selectiveFtsQuery(query: string, rows: number, rowsMatching: (phrase: string) => number): string | null;
|
|
74
|
+
/** Numeric order that treats two infinities as equal (plain subtraction gives NaN). */
|
|
75
|
+
declare function compareScores(a: number, b: number): number;
|
|
76
|
+
|
|
77
|
+
export { compareScores, contentFtsQuery, selectiveFtsQuery };
|