@zauso-ai/capstan-db 0.4.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -12,6 +12,8 @@ export { defineEmbedding, openaiEmbeddings } from "./embedding.js";
12
12
  export type { EmbeddingAdapter, EmbeddingConfig, OpenAIEmbeddingOptions } from "./embedding.js";
13
13
  export { cosineDistance, findNearest, hybridSearch } from "./search.js";
14
14
  export type { VectorItem, ScoredResult, HybridItem, HybridSearchOptions } from "./search.js";
15
+ export { setTokenizer, defaultTokenizer } from "./tokenize.js";
16
+ export type { Tokenizer } from "./tokenize.js";
15
17
  export { prepareCreateData, prepareUpdateData } from "./write.js";
16
18
  export type { PrepareWriteOptions } from "./write.js";
17
19
  export type { ModelDefinition, FieldDefinition, RelationDefinition, IndexDefinition, DatabaseConfig, DbProvider, ScalarType, } from "./types.js";
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACnB,wBAAwB,EACxB,oBAAoB,EACpB,yBAAyB,EACzB,kBAAkB,EAClB,uBAAuB,EACvB,sBAAsB,EACtB,2BAA2B,GAC5B,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,cAAc,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpK,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AACrD,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACxI,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACnG,YAAY,EAAE,gBAAgB,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACnE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAChG,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACxE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC7F,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAClE,YAAY,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACtD,YAAY,EACV,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,eAAe,EACf,cAAc,EACd,UAAU,EACV,UAAU,GACX,MAAM,YAAY,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACnB,wBAAwB,EACxB,oBAAoB,EACpB,yBAAyB,EACzB,kBAAkB,EAClB,uBAAuB,EACvB,sBAAsB,EACtB,2BAA2B,GAC5B,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,cAAc,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpK,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AACrD,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACxI,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACnG,YAAY,EAAE,gBAAgB,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACnE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAChG,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACxE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC7F,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC/D,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAClE,YAAY,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACtD,YAAY,EACV,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,eAAe,EACf,cAAc,EACd,UAAU,EACV,UAAU,GACX,MAAM,YAAY,CAAC"}
package/dist/index.js CHANGED
@@ -6,5 +6,6 @@ export { createDatabaseRuntime } from "./runtime.js";
6
6
  export { createCrudRepository, createCrudRuntime, generateCrudRoutes, pluralize } from "./crud.js";
7
7
  export { defineEmbedding, openaiEmbeddings } from "./embedding.js";
8
8
  export { cosineDistance, findNearest, hybridSearch } from "./search.js";
9
+ export { setTokenizer, defaultTokenizer } from "./tokenize.js";
9
10
  export { prepareCreateData, prepareUpdateData } from "./write.js";
10
11
  //# sourceMappingURL=index.js.map
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACnB,wBAAwB,EACxB,oBAAoB,EACpB,yBAAyB,EACzB,kBAAkB,EAClB,uBAAuB,EACvB,sBAAsB,EACtB,2BAA2B,GAC5B,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEnG,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAExE,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACnB,wBAAwB,EACxB,oBAAoB,EACpB,yBAAyB,EACzB,kBAAkB,EAClB,uBAAuB,EACvB,sBAAsB,EACtB,2BAA2B,GAC5B,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEnG,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAExE,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAE/D,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC"}
package/dist/search.d.ts CHANGED
@@ -36,15 +36,26 @@ export interface HybridSearchOptions {
36
36
  keywordWeight?: number;
37
37
  /** Weight for the vector component (0-1). @default 0.7 */
38
38
  vectorWeight?: number;
39
+ /** BM25 tuning: `k1` term-frequency saturation (@default 1.5) and `b`
40
+ * document-length normalisation, 0-1 (@default 0.75). */
41
+ bm25?: {
42
+ k1?: number;
43
+ b?: number;
44
+ };
39
45
  }
40
46
  /**
41
- * Hybrid search combining BM25-style keyword matching with vector cosine
47
+ * Hybrid search combining **Okapi BM25** keyword relevance with vector cosine
42
48
  * similarity.
43
49
  *
44
- * The keyword score is a simple normalised term-frequency overlap (not a
45
- * full BM25 implementation) which is sufficient for re-ranking a
46
- * pre-filtered candidate set. For production-grade keyword search, pair
47
- * with a proper full-text index.
50
+ * The keyword component is BM25 computed over the candidate `items` (which
51
+ * supply the corpus statistics IDF and average document length): term-
52
+ * frequency saturation (`k1`), document-length normalisation (`b`), and
53
+ * probabilistic IDF. BM25 scores are normalised to [0, 1] by the top score in
54
+ * the candidate set, so the keyword component fuses cleanly with the cosine
55
+ * score via the weights.
56
+ *
57
+ * This in-memory scorer is for re-ranking a pre-filtered candidate set; for a
58
+ * large corpus, back it with a dedicated full-text index.
48
59
  *
49
60
  * @returns Items sorted by descending combined score.
50
61
  */
@@ -1 +1 @@
1
- {"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAIA;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAwB/D;AAMD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,MAAM,EAAE,EACf,KAAK,EAAE,UAAU,EAAE,EACnB,CAAC,EAAE,MAAM,GACR,YAAY,EAAE,CAQhB;AAMD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC,+CAA+C;IAC/C,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EAAE,EACrB,KAAK,EAAE,UAAU,EAAE,EACnB,IAAI,CAAC,EAAE,mBAAmB,GACzB,YAAY,EAAE,CA6BhB"}
1
+ {"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAwB/D;AAMD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,MAAM,EAAE,EACf,KAAK,EAAE,UAAU,EAAE,EACnB,CAAC,EAAE,MAAM,GACR,YAAY,EAAE,CAQhB;AAMD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC,+CAA+C;IAC/C,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;6DACyD;IACzD,IAAI,CAAC,EAAE;QAAE,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACpC;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EAAE,EACrB,KAAK,EAAE,UAAU,EAAE,EACnB,IAAI,CAAC,EAAE,mBAAmB,GACzB,YAAY,EAAE,CAkChB"}
package/dist/search.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { tokenize } from "./tokenize.js";
1
2
  // ---------------------------------------------------------------------------
2
3
  // Vector similarity search utilities
3
4
  // ---------------------------------------------------------------------------
@@ -46,13 +47,18 @@ export function findNearest(query, items, k) {
46
47
  return scored.slice(0, k);
47
48
  }
48
49
  /**
49
- * Hybrid search combining BM25-style keyword matching with vector cosine
50
+ * Hybrid search combining **Okapi BM25** keyword relevance with vector cosine
50
51
  * similarity.
51
52
  *
52
- * The keyword score is a simple normalised term-frequency overlap (not a
53
- * full BM25 implementation) which is sufficient for re-ranking a
54
- * pre-filtered candidate set. For production-grade keyword search, pair
55
- * with a proper full-text index.
53
+ * The keyword component is BM25 computed over the candidate `items` (which
54
+ * supply the corpus statistics IDF and average document length): term-
55
+ * frequency saturation (`k1`), document-length normalisation (`b`), and
56
+ * probabilistic IDF. BM25 scores are normalised to [0, 1] by the top score in
57
+ * the candidate set, so the keyword component fuses cleanly with the cosine
58
+ * score via the weights.
59
+ *
60
+ * This in-memory scorer is for re-ranking a pre-filtered candidate set; for a
61
+ * large corpus, back it with a dedicated full-text index.
56
62
  *
57
63
  * @returns Items sorted by descending combined score.
58
64
  */
@@ -60,51 +66,94 @@ export function hybridSearch(query, queryVector, items, opts) {
60
66
  const k = opts?.k ?? 10;
61
67
  const keywordWeight = opts?.keywordWeight ?? 0.3;
62
68
  const vectorWeight = opts?.vectorWeight ?? 0.7;
63
- // Tokenise the query into lower-case terms.
64
- const queryTerms = tokenize(query);
69
+ // Clamp to valid BM25 ranges so an out-of-range option can't make the
70
+ // length-normalisation term negative (which would yield negative scores).
71
+ const k1 = Math.max(0, opts?.bm25?.k1 ?? 1.5);
72
+ const b = Math.min(1, Math.max(0, opts?.bm25?.b ?? 0.75));
73
+ // Tokenise the query into unique terms (via the active tokeniser).
74
+ const queryTerms = [...new Set(tokenize(query))];
65
75
  if (queryTerms.length === 0 && queryVector.length === 0) {
66
76
  return [];
67
77
  }
68
- // Score each item.
69
- const scored = items.map((item) => {
78
+ // BM25 keyword scores for the whole candidate set (normalised to [0, 1]).
79
+ const keywordScores = queryTerms.length > 0 ? bm25Scores(queryTerms, items, k1, b) : null;
80
+ const scored = items.map((item, i) => {
70
81
  const vectorScore = queryVector.length > 0
71
82
  ? 1 - cosineDistance(queryVector, item.vector)
72
83
  : 0;
73
- const keywordScore = queryTerms.length > 0 ? keywordOverlap(queryTerms, item.text) : 0;
84
+ const keywordScore = keywordScores ? keywordScores[i] : 0;
74
85
  return {
75
86
  id: item.id,
76
87
  score: vectorWeight * vectorScore + keywordWeight * keywordScore,
77
88
  };
78
89
  });
79
- scored.sort((a, b) => b.score - a.score);
90
+ scored.sort((a, b2) => b2.score - a.score);
80
91
  return scored.slice(0, k);
81
92
  }
82
93
  // ---------------------------------------------------------------------------
83
94
  // Internal helpers
84
95
  // ---------------------------------------------------------------------------
85
- /** Simple whitespace tokeniser with lower-casing and deduplication. */
86
- function tokenize(text) {
87
- return [
88
- ...new Set(text
89
- .toLowerCase()
90
- .split(/\W+/)
91
- .filter((t) => t.length > 0)),
92
- ];
96
+ /** Term-frequency map for a document (keeps counts), via the active tokeniser. */
97
+ function termCounts(text) {
98
+ const counts = new Map();
99
+ for (const t of tokenize(text)) {
100
+ counts.set(t, (counts.get(t) ?? 0) + 1);
101
+ }
102
+ return counts;
93
103
  }
94
104
  /**
95
- * Normalised keyword overlap score in [0, 1].
105
+ * Okapi BM25 keyword scores for each item, normalised to [0, 1] by the highest
106
+ * score in the set (the best keyword match becomes 1.0, so the component fuses
107
+ * cleanly with cosine similarity). Corpus statistics (IDF, avgdl) are derived
108
+ * from `items`. Returns one score per item, in input order.
96
109
  *
97
- * Counts how many query terms appear in the document text, divided by the
98
- * number of query terms.
110
+ * score(D) = Σ_term IDF(term) · tf·(k1+1) / ( tf + k1·(1 b + b·|D|/avgdl) )
99
111
  */
100
- function keywordOverlap(queryTerms, text) {
101
- const docTokens = new Set(tokenize(text));
102
- let hits = 0;
112
+ function bm25Scores(queryTerms, items, k1, b) {
113
+ const N = items.length;
114
+ if (N === 0)
115
+ return [];
116
+ const docCounts = new Array(N);
117
+ const docLen = new Array(N);
118
+ const df = new Map(); // document frequency per term
119
+ let totalLen = 0;
120
+ for (let i = 0; i < N; i++) {
121
+ const counts = termCounts(items[i].text);
122
+ docCounts[i] = counts;
123
+ let len = 0;
124
+ for (const c of counts.values())
125
+ len += c;
126
+ docLen[i] = len;
127
+ totalLen += len;
128
+ for (const term of counts.keys()) {
129
+ df.set(term, (df.get(term) ?? 0) + 1);
130
+ }
131
+ }
132
+ const avgdl = totalLen / N || 1;
133
+ // Probabilistic IDF, Lucene-style (always >= 0), per query term.
134
+ const idf = new Map();
103
135
  for (const term of queryTerms) {
104
- if (docTokens.has(term)) {
105
- hits++;
136
+ const n = df.get(term) ?? 0;
137
+ idf.set(term, Math.log(1 + (N - n + 0.5) / (n + 0.5)));
138
+ }
139
+ const raw = new Array(N);
140
+ let max = 0;
141
+ for (let i = 0; i < N; i++) {
142
+ const counts = docCounts[i];
143
+ const lenNorm = 1 - b + b * (docLen[i] / avgdl);
144
+ let s = 0;
145
+ for (const term of queryTerms) {
146
+ const tf = counts.get(term) ?? 0;
147
+ if (tf === 0)
148
+ continue;
149
+ s += (idf.get(term) ?? 0) * ((tf * (k1 + 1)) / (tf + k1 * lenNorm));
106
150
  }
151
+ raw[i] = s;
152
+ if (s > max)
153
+ max = s;
107
154
  }
108
- return hits / queryTerms.length;
155
+ if (max === 0)
156
+ return raw; // no query term matched any document
157
+ return raw.map((s) => s / max);
109
158
  }
110
159
  //# sourceMappingURL=search.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,qCAAqC;AACrC,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,CAAW,EAAE,CAAW;IACrD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,2BAA2B,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CACrD,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;QACjB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;QACjB,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;QACf,KAAK,IAAI,EAAE,GAAG,EAAE,CAAC;QACjB,KAAK,IAAI,EAAE,GAAG,EAAE,CAAC;IACnB,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC,wBAAwB;IAEnD,MAAM,UAAU,GAAG,GAAG,GAAG,KAAK,CAAC;IAC/B,OAAO,CAAC,GAAG,UAAU,CAAC;AACxB,CAAC;AAgBD;;;;;;;;;GASG;AACH,MAAM,UAAU,WAAW,CACzB,KAAe,EACf,KAAmB,EACnB,CAAS;IAET,MAAM,MAAM,GAAmB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAClD,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,KAAK,EAAE,CAAC,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;KAC9C,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,CAAC;AAqBD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,WAAqB,EACrB,KAAmB,EACnB,IAA0B;IAE1B,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACxB,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,GAAG,CAAC;IACjD,MAAM,YAAY,GAAG,IAAI,EAAE,YAAY,IAAI,GAAG,CAAC;IAE/C,4CAA4C;IAC5C,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,mBAAmB;IACnB,MAAM,MAAM,GAAmB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAChD,MAAM,WAAW,GACf,WAAW,CAAC,MAAM,GAAG,CAAC;YACpB,CAAC,CAAC,CAAC,GAAG,cAAc,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC;YAC9C,CAAC,CAAC,CAAC,CAAC;QAER,MAAM,YAAY,GAChB,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpE,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,KAAK,EAAE,YAAY,GAAG,WAAW,GAAG,aAAa,GAAG,YAAY;SACjE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,uEAAuE;AACvE,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO;QACL,GAAG,IAAI,GAAG,CACR,IAAI;aACD,WAAW,EAAE;aACb,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAC/B;KACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,SAAS,cAAc,CAAC,UAAoB,EAAE,IAAY;IACxD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,IAAI,EAAE,CAAC;QACT,CAAC;IACH,CAAC;IACD,OAAO,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC;AAClC,CAAC"}
1
+ {"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,8EAA8E;AAC9E,qCAAqC;AACrC,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,CAAW,EAAE,CAAW;IACrD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,2BAA2B,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CACrD,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;QACjB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;QACjB,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;QACf,KAAK,IAAI,EAAE,GAAG,EAAE,CAAC;QACjB,KAAK,IAAI,EAAE,GAAG,EAAE,CAAC;IACnB,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC,wBAAwB;IAEnD,MAAM,UAAU,GAAG,GAAG,GAAG,KAAK,CAAC;IAC/B,OAAO,CAAC,GAAG,UAAU,CAAC;AACxB,CAAC;AAgBD;;;;;;;;;GASG;AACH,MAAM,UAAU,WAAW,CACzB,KAAe,EACf,KAAmB,EACnB,CAAS;IAET,MAAM,MAAM,GAAmB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAClD,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,KAAK,EAAE,CAAC,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;KAC9C,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,CAAC;AAwBD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,WAAqB,EACrB,KAAmB,EACnB,IAA0B;IAE1B,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACxB,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,GAAG,CAAC;IACjD,MAAM,YAAY,GAAG,IAAI,EAAE,YAAY,IAAI,GAAG,CAAC;IAC/C,sEAAsE;IACtE,0EAA0E;IAC1E,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,IAAI,GAAG,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IAE1D,mEAAmE;IACnE,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,0EAA0E;IAC1E,MAAM,aAAa,GACjB,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEtE,MAAM,MAAM,GAAmB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACnD,MAAM,WAAW,GACf,WAAW,CAAC,MAAM,GAAG,CAAC;YACpB,CAAC,CAAC,CAAC,GAAG,cAAc,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC;YAC9C,CAAC,CAAC,CAAC,CAAC;QACR,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAE3D,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,KAAK,EAAE,YAAY,GAAG,WAAW,GAAG,aAAa,GAAG,YAAY;SACjE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC3C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,kFAAkF;AAClF,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,UAAU,CACjB,UAAoB,EACpB,KAAmB,EACnB,EAAU,EACV,CAAS;IAET,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvB,MAAM,SAAS,GAA0B,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;IACtD,MAAM,MAAM,GAAa,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;IACtC,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,8BAA8B;IACpE,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC;QAC1C,SAAS,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;QACtB,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;YAAE,GAAG,IAAI,CAAC,CAAC;QAC1C,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;QAChB,QAAQ,IAAI,GAAG,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YACjC,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,IAAI,CAAC,CAAC;IAEhC,iEAAiE;IACjE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,GAAG,GAAa,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAE,GAAG,KAAK,CAAC,CAAC;QACjD,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,EAAE,KAAK,CAAC;gBAAE,SAAS;YACvB,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;QACD,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACX,IAAI,CAAC,GAAG,GAAG;YAAE,GAAG,GAAG,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC,CAAC,qCAAqC;IAChE,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;AACjC,CAAC"}
@@ -0,0 +1,29 @@
1
+ /** Maps text to an ordered list of lower-case terms (repeats kept). */
2
+ export type Tokenizer = (text: string) => string[];
3
+ /**
4
+ * Default tokeniser: Unicode word segmentation via `Intl.Segmenter`. Segments
5
+ * CJK/Japanese/Thai with ICU dictionaries and splits space-delimited languages
6
+ * on word boundaries; keeps numbers; lower-cases. Falls back to a Unicode regex
7
+ * split when the runtime has no `Intl.Segmenter`.
8
+ */
9
+ export declare const defaultTokenizer: Tokenizer;
10
+ /**
11
+ * Override the tokeniser used by keyword search (`hybridSearch`) — e.g. to plug
12
+ * in a dictionary segmenter like jieba for Chinese:
13
+ *
14
+ * ```ts
15
+ * import { cut } from "@node-rs/jieba";
16
+ * setTokenizer((text) => cut(text.toLowerCase(), true));
17
+ * ```
18
+ *
19
+ * Call with no argument to restore the default `Intl.Segmenter` tokeniser.
20
+ */
21
+ export declare function setTokenizer(tokenizer?: Tokenizer): void;
22
+ /**
23
+ * Tokenise text with the active tokeniser (default: `Intl.Segmenter`). Empty
24
+ * tokens are dropped here so a custom tokeniser that emits `""` (e.g. a naive
25
+ * `split(" ")` on runs of whitespace) can't poison BM25 term counts, document
26
+ * lengths, or let a blank query self-match.
27
+ */
28
+ export declare function tokenize(text: string): string[];
29
+ //# sourceMappingURL=tokenize.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../src/tokenize.ts"],"names":[],"mappings":"AAUA,uEAAuE;AACvE,MAAM,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;AAoBnD;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAiB9B,CAAC;AAIF;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAAC,SAAS,GAAE,SAA4B,GAAG,IAAI,CAE1E;AAED;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAE/C"}
@@ -0,0 +1,76 @@
1
+ // ---------------------------------------------------------------------------
2
+ // Tokenisation for keyword search (BM25 in hybridSearch)
3
+ // ---------------------------------------------------------------------------
4
+ //
5
+ // The default tokeniser uses the runtime's built-in `Intl.Segmenter` (ICU word
6
+ // segmentation), which segments CJK / Japanese / Thai by dictionary
7
+ // (机器学习 → 机器 / 学习) as well as space-delimited languages — no extra
8
+ // dependency. It is pluggable: call `setTokenizer` to swap in a dictionary
9
+ // segmenter such as jieba for higher-quality Chinese segmentation.
10
+ // Keep any segment containing a letter or number; drop whitespace/punctuation.
11
+ // `Intl.Segmenter`'s `isWordLike` is unusable here — it reports `false` for
12
+ // numbers and alphanumerics ("2024", "gpt", "v2"), which we must keep.
13
+ const WORDY = /[\p{L}\p{N}]/u;
14
+ // Runs of non-(letter|number|underscore) — the word/part separator. Used to
15
+ // split intra-word punctuation out of a segment (config.yaml -> config/yaml)
16
+ // and as the whole-text tokeniser when `Intl.Segmenter` is unavailable.
17
+ const FALLBACK_SPLIT = /[^\p{L}\p{N}_]+/u;
18
+ const segmenter = (() => {
19
+ try {
20
+ return new Intl.Segmenter(undefined, { granularity: "word" });
21
+ }
22
+ catch {
23
+ return null;
24
+ }
25
+ })();
26
+ /**
27
+ * Default tokeniser: Unicode word segmentation via `Intl.Segmenter`. Segments
28
+ * CJK/Japanese/Thai with ICU dictionaries and splits space-delimited languages
29
+ * on word boundaries; keeps numbers; lower-cases. Falls back to a Unicode regex
30
+ * split when the runtime has no `Intl.Segmenter`.
31
+ */
32
+ export const defaultTokenizer = (text) => {
33
+ const lower = text.toLowerCase();
34
+ if (!segmenter) {
35
+ return lower.split(FALLBACK_SPLIT).filter((t) => t.length > 0);
36
+ }
37
+ const out = [];
38
+ for (const { segment } of segmenter.segment(lower)) {
39
+ if (!WORDY.test(segment))
40
+ continue; // skip whitespace / punctuation segments
41
+ // Intl.Segmenter keeps intra-word `.` / `'` (e.g. "config.yaml", "user's",
42
+ // "3.14"); split those out so code, filenames, and versions stay searchable
43
+ // by part. Underscore is kept (identifier-like), and CJK / word segments
44
+ // have no such punctuation, so they pass through unchanged.
45
+ for (const part of segment.split(FALLBACK_SPLIT)) {
46
+ if (part.length > 0)
47
+ out.push(part);
48
+ }
49
+ }
50
+ return out;
51
+ };
52
+ let active = defaultTokenizer;
53
+ /**
54
+ * Override the tokeniser used by keyword search (`hybridSearch`) — e.g. to plug
55
+ * in a dictionary segmenter like jieba for Chinese:
56
+ *
57
+ * ```ts
58
+ * import { cut } from "@node-rs/jieba";
59
+ * setTokenizer((text) => cut(text.toLowerCase(), true));
60
+ * ```
61
+ *
62
+ * Call with no argument to restore the default `Intl.Segmenter` tokeniser.
63
+ */
64
+ export function setTokenizer(tokenizer = defaultTokenizer) {
65
+ active = tokenizer;
66
+ }
67
+ /**
68
+ * Tokenise text with the active tokeniser (default: `Intl.Segmenter`). Empty
69
+ * tokens are dropped here so a custom tokeniser that emits `""` (e.g. a naive
70
+ * `split(" ")` on runs of whitespace) can't poison BM25 term counts, document
71
+ * lengths, or let a blank query self-match.
72
+ */
73
+ export function tokenize(text) {
74
+ return active(text).filter((t) => t.length > 0);
75
+ }
76
+ //# sourceMappingURL=tokenize.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tokenize.js","sourceRoot":"","sources":["../src/tokenize.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,yDAAyD;AACzD,8EAA8E;AAC9E,EAAE;AACF,+EAA+E;AAC/E,oEAAoE;AACpE,mEAAmE;AACnE,2EAA2E;AAC3E,mEAAmE;AAKnE,+EAA+E;AAC/E,4EAA4E;AAC5E,uEAAuE;AACvE,MAAM,KAAK,GAAG,eAAe,CAAC;AAE9B,4EAA4E;AAC5E,6EAA6E;AAC7E,wEAAwE;AACxE,MAAM,cAAc,GAAG,kBAAkB,CAAC;AAE1C,MAAM,SAAS,GAA0B,CAAC,GAAG,EAAE;IAC7C,IAAI,CAAC;QACH,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;IAChE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC,CAAC,EAAE,CAAC;AAEL;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAc,CAAC,IAAI,EAAE,EAAE;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,SAAS,CAAC,yCAAyC;QAC7E,2EAA2E;QAC3E,4EAA4E;QAC5E,yEAAyE;QACzE,4DAA4D;QAC5D,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;YACjD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,IAAI,MAAM,GAAc,gBAAgB,CAAC;AAEzC;;;;;;;;;;GAUG;AACH,MAAM,UAAU,YAAY,CAAC,YAAuB,gBAAgB;IAClE,MAAM,GAAG,SAAS,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAClD,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zauso-ai/capstan-db",
3
- "version": "0.4.0",
3
+ "version": "0.5.1",
4
4
  "type": "module",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",