@nebutra/knowledge-rag 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +676 -0
- package/README.md +151 -0
- package/dist/chunker.d.ts +21 -0
- package/dist/chunker.d.ts.map +1 -0
- package/dist/chunker.js +103 -0
- package/dist/cli.d.ts +7 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +26 -0
- package/dist/embedder.d.ts +20 -0
- package/dist/embedder.d.ts.map +1 -0
- package/dist/embedder.js +106 -0
- package/dist/errors.d.ts +19 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +33 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +54 -0
- package/dist/keyword.d.ts +43 -0
- package/dist/keyword.d.ts.map +1 -0
- package/dist/keyword.js +114 -0
- package/dist/pipeline.d.ts +4 -0
- package/dist/pipeline.d.ts.map +1 -0
- package/dist/pipeline.js +224 -0
- package/dist/reranker.d.ts +17 -0
- package/dist/reranker.d.ts.map +1 -0
- package/dist/reranker.js +48 -0
- package/dist/scoring.d.ts +16 -0
- package/dist/scoring.d.ts.map +1 -0
- package/dist/scoring.js +64 -0
- package/dist/stores/memory.d.ts +16 -0
- package/dist/stores/memory.d.ts.map +1 -0
- package/dist/stores/memory.js +43 -0
- package/dist/stores/pgvector.d.ts +31 -0
- package/dist/stores/pgvector.d.ts.map +1 -0
- package/dist/stores/pgvector.js +97 -0
- package/dist/tool.d.ts +30 -0
- package/dist/tool.d.ts.map +1 -0
- package/dist/tool.js +48 -0
- package/dist/types.d.ts +101 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +27 -0
- package/package.json +61 -0
package/dist/keyword.js
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// =============================================================================
|
|
2
|
+
// @nebutra/knowledge-rag — Keyword leg (wraps @nebutra/search)
|
|
3
|
+
// =============================================================================
|
|
4
|
+
// Rather than reinventing a keyword index, this WRAPS the existing
|
|
5
|
+
// provider-agnostic @nebutra/search abstraction (Meilisearch / Typesense /
|
|
6
|
+
// Algolia / pgvector-BM25). Each chunk is indexed as a SearchDocument carrying
|
|
7
|
+
// tenantId; every query filters by { tenantId } so the keyword leg has the
|
|
8
|
+
// same tenant-isolation guarantee as the vector leg.
|
|
9
|
+
//
|
|
10
|
+
// @nebutra/search is imported lazily so the zero-config path (keyword
|
|
11
|
+
// disabled, no search backend) never needs it at module load.
|
|
12
|
+
// =============================================================================
|
|
13
|
+
import { KnowledgeRagError } from "./errors";
|
|
14
|
+
/**
|
|
15
|
+
* KeywordIndex backed by @nebutra/search. Construct via
|
|
16
|
+
* `SearchKeywordIndex.tryCreate(indexName)` which returns `null` when no
|
|
17
|
+
* search backend is configured (so the pipeline can degrade to vector-only).
|
|
18
|
+
*/
|
|
19
|
+
export class SearchKeywordIndex {
|
|
20
|
+
// biome-ignore lint/suspicious/noExplicitAny: external provider type imported lazily
|
|
21
|
+
provider;
|
|
22
|
+
indexName;
|
|
23
|
+
// biome-ignore lint/suspicious/noExplicitAny: external provider type imported lazily
|
|
24
|
+
constructor(provider, indexName) {
|
|
25
|
+
this.provider = provider;
|
|
26
|
+
this.indexName = indexName;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Returns a live keyword index, or `null` when no @nebutra/search backend
|
|
30
|
+
* is genuinely reachable. The default provider auto-detects to Meilisearch
|
|
31
|
+
* even with no env; it only fails on the first network call. So we probe
|
|
32
|
+
* with a real createIndex/search call and degrade to vector-only on any
|
|
33
|
+
* failure — the zero-config path must NEVER break because a search server
|
|
34
|
+
* happens to be absent.
|
|
35
|
+
*/
|
|
36
|
+
static async tryCreate(indexName) {
|
|
37
|
+
try {
|
|
38
|
+
const mod = await import("@nebutra/search");
|
|
39
|
+
const search = await mod.getSearch();
|
|
40
|
+
try {
|
|
41
|
+
await search.createIndex(indexName, {
|
|
42
|
+
primaryKey: "id",
|
|
43
|
+
filterableAttributes: ["tenantId", "docId"],
|
|
44
|
+
searchableAttributes: ["text"],
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
// createIndex may fail because the index already exists (fine) OR
|
|
49
|
+
// because the backend is unreachable (not fine). Disambiguate with a
|
|
50
|
+
// cheap reachability probe.
|
|
51
|
+
try {
|
|
52
|
+
await search.search(indexName, { query: "", hitsPerPage: 1 });
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return null; // backend not reachable — degrade to vector-only
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return new SearchKeywordIndex(search, indexName);
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
async index(chunks) {
|
|
65
|
+
if (chunks.length === 0)
|
|
66
|
+
return;
|
|
67
|
+
await this.provider.indexDocuments(this.indexName, chunks.map((c) => ({
|
|
68
|
+
id: c.id,
|
|
69
|
+
tenantId: c.tenantId,
|
|
70
|
+
docId: c.docId,
|
|
71
|
+
text: c.text,
|
|
72
|
+
})));
|
|
73
|
+
}
|
|
74
|
+
async search(tenantId, query, topK) {
|
|
75
|
+
try {
|
|
76
|
+
const result = await this.provider.search(this.indexName, {
|
|
77
|
+
query,
|
|
78
|
+
tenantId,
|
|
79
|
+
filters: { tenantId },
|
|
80
|
+
hitsPerPage: Math.min(100, Math.max(1, topK)),
|
|
81
|
+
});
|
|
82
|
+
return (result.hits ?? [])
|
|
83
|
+
.filter((h) => h.doc.tenantId === tenantId)
|
|
84
|
+
.map((h) => ({
|
|
85
|
+
chunkId: h.doc.id,
|
|
86
|
+
docId: h.doc.docId,
|
|
87
|
+
tenantId: h.doc.tenantId,
|
|
88
|
+
score: h.score,
|
|
89
|
+
}));
|
|
90
|
+
}
|
|
91
|
+
catch (err) {
|
|
92
|
+
throw new KnowledgeRagError(`Keyword search failed: ${err?.message ?? String(err)}`, {
|
|
93
|
+
code: "E_KEYWORD_SEARCH",
|
|
94
|
+
cause: err,
|
|
95
|
+
suggestion: "Check the @nebutra/search backend (MEILISEARCH_URL / TYPESENSE_URL / ALGOLIA_APP_ID), or set { disableKeyword: true } for vector-only retrieval.",
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
async deleteByDoc(docId, tenantId) {
|
|
100
|
+
await this.provider.deleteByFilter(this.indexName, { docId, tenantId });
|
|
101
|
+
}
|
|
102
|
+
async health() {
|
|
103
|
+
try {
|
|
104
|
+
await this.provider.search(this.indexName, { query: "", hitsPerPage: 1 });
|
|
105
|
+
return { ok: true, detail: `@nebutra/search: index "${this.indexName}" reachable` };
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
return {
|
|
109
|
+
ok: false,
|
|
110
|
+
detail: `@nebutra/search unreachable: ${err?.message ?? String(err)}`,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAoBA,OAAO,EAML,KAAK,YAAY,EACjB,KAAK,kBAAkB,EAIxB,MAAM,SAAS,CAAC;AAmOjB,2DAA2D;AAC3D,wBAAgB,kBAAkB,CAAC,MAAM,GAAE,kBAAuB,GAAG,YAAY,CAEhF"}
|
package/dist/pipeline.js
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
// =============================================================================
|
|
2
|
+
// @nebutra/knowledge-rag — Pipeline
|
|
3
|
+
// =============================================================================
|
|
4
|
+
// ingest: validate → chunk → embed → vector upsert + keyword index
|
|
5
|
+
// query: validate → embed query → vector leg + keyword leg → hybrid blend
|
|
6
|
+
// → optional rerank → topK
|
|
7
|
+
// delete: scoped by (docId, tenantId) across both legs
|
|
8
|
+
//
|
|
9
|
+
// tenantId is required on every input and threaded into every store call.
|
|
10
|
+
// Tenant isolation is enforced by the stores; the pipeline never widens scope.
|
|
11
|
+
// =============================================================================
|
|
12
|
+
import { RecursiveCharChunker } from "./chunker";
|
|
13
|
+
import { LocalHashEmbedder } from "./embedder";
|
|
14
|
+
import { KnowledgeRagError } from "./errors";
|
|
15
|
+
import { SearchKeywordIndex } from "./keyword";
|
|
16
|
+
import { IdentityReranker } from "./reranker";
|
|
17
|
+
import { hybridBlend, normalizeScores } from "./scoring";
|
|
18
|
+
import { InMemoryVectorStore } from "./stores/memory";
|
|
19
|
+
import { IngestDocumentSchema, QueryInputSchema, } from "./types";
|
|
20
|
+
const DEFAULT_INDEX = "knowledge_rag";
|
|
21
|
+
const DEFAULT_VECTOR_WEIGHT = 0.6;
|
|
22
|
+
const DEFAULT_TOP_K = 8;
|
|
23
|
+
class KnowledgeRagPipeline {
|
|
24
|
+
chunker;
|
|
25
|
+
embedder;
|
|
26
|
+
store;
|
|
27
|
+
reranker;
|
|
28
|
+
indexName;
|
|
29
|
+
vectorWeight;
|
|
30
|
+
disableKeyword;
|
|
31
|
+
keyword = null;
|
|
32
|
+
keywordResolved = false;
|
|
33
|
+
constructor(config = {}) {
|
|
34
|
+
this.chunker = config.chunker ?? new RecursiveCharChunker({ size: 800, overlap: 100 });
|
|
35
|
+
this.embedder = config.embedder ?? new LocalHashEmbedder(256);
|
|
36
|
+
this.store = config.vectorStore ?? new InMemoryVectorStore();
|
|
37
|
+
this.reranker = config.reranker ?? new IdentityReranker();
|
|
38
|
+
this.indexName = config.index ?? DEFAULT_INDEX;
|
|
39
|
+
this.vectorWeight = config.vectorWeight ?? DEFAULT_VECTOR_WEIGHT;
|
|
40
|
+
this.disableKeyword = config.disableKeyword ?? false;
|
|
41
|
+
}
|
|
42
|
+
/** Lazily resolve the keyword leg; null when no search backend / disabled. */
|
|
43
|
+
async keywordIndex() {
|
|
44
|
+
if (this.disableKeyword)
|
|
45
|
+
return null;
|
|
46
|
+
if (this.keywordResolved)
|
|
47
|
+
return this.keyword;
|
|
48
|
+
this.keyword = await SearchKeywordIndex.tryCreate(this.indexName);
|
|
49
|
+
this.keywordResolved = true;
|
|
50
|
+
return this.keyword;
|
|
51
|
+
}
|
|
52
|
+
async ingest(doc) {
|
|
53
|
+
const parsed = IngestDocumentSchema.safeParse(doc);
|
|
54
|
+
if (!parsed.success) {
|
|
55
|
+
throw new KnowledgeRagError(`Invalid ingest document: ${parsed.error.issues.map((i) => i.path.join(".") + " " + i.message).join("; ")}`, {
|
|
56
|
+
code: "E_INGEST_INPUT",
|
|
57
|
+
suggestion: "ingest() requires { id, text, tenantId } all non-empty. tenantId is mandatory — every record is tenant-scoped.",
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
const { id, text, tenantId, meta = {} } = parsed.data;
|
|
61
|
+
const pieces = this.chunker.split(text);
|
|
62
|
+
if (pieces.length === 0) {
|
|
63
|
+
return { docId: id, tenantId, chunks: 0 };
|
|
64
|
+
}
|
|
65
|
+
const vectors = await this.embedder.embed(pieces);
|
|
66
|
+
const chunks = pieces.map((t, ordinal) => ({
|
|
67
|
+
id: `${id}::${ordinal}`,
|
|
68
|
+
docId: id,
|
|
69
|
+
tenantId,
|
|
70
|
+
text: t,
|
|
71
|
+
ordinal,
|
|
72
|
+
embedding: vectors[ordinal],
|
|
73
|
+
meta,
|
|
74
|
+
}));
|
|
75
|
+
await this.store.upsert(chunks);
|
|
76
|
+
const kw = await this.keywordIndex();
|
|
77
|
+
if (kw)
|
|
78
|
+
await kw.index(chunks);
|
|
79
|
+
return { docId: id, tenantId, chunks: chunks.length };
|
|
80
|
+
}
|
|
81
|
+
async query(input) {
|
|
82
|
+
const parsed = QueryInputSchema.safeParse(input);
|
|
83
|
+
if (!parsed.success) {
|
|
84
|
+
throw new KnowledgeRagError(`Invalid query input: ${parsed.error.issues.map((i) => i.path.join(".") + " " + i.message).join("; ")}`, {
|
|
85
|
+
code: "E_QUERY_INPUT",
|
|
86
|
+
suggestion: "query() requires { query, tenantId } non-empty; topK is 1..100.",
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
const { query, tenantId } = parsed.data;
|
|
90
|
+
const topK = parsed.data.topK ?? DEFAULT_TOP_K;
|
|
91
|
+
const fetchK = Math.max(topK * 4, topK);
|
|
92
|
+
const [queryVec] = await this.embedder.embed([query]);
|
|
93
|
+
const vectorHits = await this.store.queryByVector(tenantId, queryVec, fetchK);
|
|
94
|
+
const kw = await this.keywordIndex();
|
|
95
|
+
const keywordHits = kw ? await kw.search(tenantId, query, fetchK) : [];
|
|
96
|
+
// Merge candidate set keyed by chunkId. Defence in depth: drop anything
|
|
97
|
+
// whose tenantId doesn't match the requested tenant.
|
|
98
|
+
const byId = new Map();
|
|
99
|
+
for (const h of vectorHits) {
|
|
100
|
+
if (h.chunk.tenantId !== tenantId)
|
|
101
|
+
continue;
|
|
102
|
+
byId.set(h.chunk.id, { chunk: h.chunk, vec: h.score, kw: 0 });
|
|
103
|
+
}
|
|
104
|
+
for (const h of keywordHits) {
|
|
105
|
+
if (h.tenantId !== tenantId)
|
|
106
|
+
continue;
|
|
107
|
+
const existing = byId.get(h.chunkId);
|
|
108
|
+
if (existing)
|
|
109
|
+
existing.kw = h.score;
|
|
110
|
+
// Keyword-only hits with no embedding context are skipped: the vector
|
|
111
|
+
// store is the source of truth for chunk bodies in this pipeline.
|
|
112
|
+
}
|
|
113
|
+
const entries = [...byId.values()];
|
|
114
|
+
if (entries.length === 0)
|
|
115
|
+
return [];
|
|
116
|
+
const normVec = normalizeScores(entries.map((e) => e.vec));
|
|
117
|
+
const normKw = normalizeScores(entries.map((e) => e.kw));
|
|
118
|
+
const hasKeyword = keywordHits.length > 0;
|
|
119
|
+
const ranked = entries.map((e, i) => {
|
|
120
|
+
const v = normVec[i];
|
|
121
|
+
const k = normKw[i];
|
|
122
|
+
const blended = hasKeyword ? hybridBlend(v, k, this.vectorWeight) : v;
|
|
123
|
+
return {
|
|
124
|
+
chunk: e.chunk,
|
|
125
|
+
score: blended,
|
|
126
|
+
scores: { vector: v, keyword: k },
|
|
127
|
+
source: hasKeyword ? "hybrid" : "vector",
|
|
128
|
+
};
|
|
129
|
+
});
|
|
130
|
+
ranked.sort((a, b) => b.score - a.score);
|
|
131
|
+
const reranked = await this.reranker.rerank(query, ranked);
|
|
132
|
+
return reranked.slice(0, topK);
|
|
133
|
+
}
|
|
134
|
+
async deleteByDoc(docId, tenantId) {
|
|
135
|
+
if (!docId || !tenantId) {
|
|
136
|
+
throw new KnowledgeRagError("deleteByDoc requires docId and tenantId", {
|
|
137
|
+
code: "E_DELETE_INPUT",
|
|
138
|
+
suggestion: "Pass both docId and tenantId — deletion is always tenant-scoped.",
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
const removed = await this.store.deleteByDoc(docId, tenantId);
|
|
142
|
+
const kw = await this.keywordIndex();
|
|
143
|
+
if (kw) {
|
|
144
|
+
try {
|
|
145
|
+
await kw.deleteByDoc(docId, tenantId);
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
// Keyword cleanup is best-effort; vector store is authoritative.
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return removed;
|
|
152
|
+
}
|
|
153
|
+
async doctor() {
|
|
154
|
+
const start = Date.now();
|
|
155
|
+
const components = [];
|
|
156
|
+
components.push({
|
|
157
|
+
name: `embedder:${this.embedder.name}`,
|
|
158
|
+
ok: true,
|
|
159
|
+
detail: this.embedder.name === "local-hash"
|
|
160
|
+
? "zero-config deterministic local embedder (no network)"
|
|
161
|
+
: "provider embedder configured",
|
|
162
|
+
});
|
|
163
|
+
try {
|
|
164
|
+
const h = await withTimeout(this.store.health(), 1500);
|
|
165
|
+
components.push({ name: `vector-store:${this.store.name}`, ok: h.ok, detail: h.detail });
|
|
166
|
+
}
|
|
167
|
+
catch (err) {
|
|
168
|
+
components.push({
|
|
169
|
+
name: `vector-store:${this.store.name}`,
|
|
170
|
+
ok: false,
|
|
171
|
+
detail: `health timed out: ${err.message}`,
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
if (this.disableKeyword) {
|
|
175
|
+
components.push({
|
|
176
|
+
name: "keyword-index",
|
|
177
|
+
ok: true,
|
|
178
|
+
detail: "disabled by config (vector-only retrieval)",
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
try {
|
|
183
|
+
const kw = await withTimeout(this.keywordIndex(), 1500);
|
|
184
|
+
if (kw) {
|
|
185
|
+
const h = await withTimeout(kw.health(), 1000);
|
|
186
|
+
components.push({ name: "keyword-index:@nebutra/search", ok: h.ok, detail: h.detail });
|
|
187
|
+
}
|
|
188
|
+
else {
|
|
189
|
+
components.push({
|
|
190
|
+
name: "keyword-index",
|
|
191
|
+
ok: true,
|
|
192
|
+
detail: "no @nebutra/search backend detected — degraded to vector-only",
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
catch (err) {
|
|
197
|
+
components.push({
|
|
198
|
+
name: "keyword-index",
|
|
199
|
+
ok: false,
|
|
200
|
+
detail: `probe failed: ${err.message}`,
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
return {
|
|
205
|
+
ok: components.every((c) => c.ok),
|
|
206
|
+
checkedAt: new Date().toISOString(),
|
|
207
|
+
durationMs: Date.now() - start,
|
|
208
|
+
components,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
function withTimeout(p, ms) {
|
|
213
|
+
return Promise.race([
|
|
214
|
+
p,
|
|
215
|
+
new Promise((_, reject) => setTimeout(() => reject(new KnowledgeRagError(`timeout after ${ms}ms`, {
|
|
216
|
+
code: "E_HEALTH_TIMEOUT",
|
|
217
|
+
suggestion: "External dependency is slow/unreachable; check its URL/credentials.",
|
|
218
|
+
})), ms)),
|
|
219
|
+
]);
|
|
220
|
+
}
|
|
221
|
+
/** Synchronous factory — fully usable with zero config. */
|
|
222
|
+
export function createKnowledgeRag(config = {}) {
|
|
223
|
+
return new KnowledgeRagPipeline(config);
|
|
224
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { RankedChunk, Reranker } from "./types";
|
|
2
|
+
/** No-op reranker — keeps the hybrid-blended order. */
|
|
3
|
+
export declare class IdentityReranker implements Reranker {
|
|
4
|
+
readonly name = "identity";
|
|
5
|
+
rerank(_query: string, candidates: RankedChunk[]): Promise<RankedChunk[]>;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Re-orders candidates by blending the hybrid score with lexical
|
|
9
|
+
* query-term coverage. Deterministic, no external calls.
|
|
10
|
+
*/
|
|
11
|
+
export declare class LexicalOverlapReranker implements Reranker {
|
|
12
|
+
readonly name = "lexical-overlap";
|
|
13
|
+
private readonly weight;
|
|
14
|
+
constructor(weight?: number);
|
|
15
|
+
rerank(query: string, candidates: RankedChunk[]): Promise<RankedChunk[]>;
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=reranker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reranker.d.ts","sourceRoot":"","sources":["../src/reranker.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAErD,uDAAuD;AACvD,qBAAa,gBAAiB,YAAW,QAAQ;IAC/C,QAAQ,CAAC,IAAI,cAAc;IAErB,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;CAGhF;AAED;;;GAGG;AACH,qBAAa,sBAAuB,YAAW,QAAQ;IACrD,QAAQ,CAAC,IAAI,qBAAqB;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBAEpB,MAAM,SAAM;IAKlB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;CAmB/E"}
|
package/dist/reranker.js
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// =============================================================================
|
|
2
|
+
// @nebutra/knowledge-rag — Rerankers
|
|
3
|
+
// =============================================================================
|
|
4
|
+
// The Reranker interface is the seam for a cross-encoder / LLM reranker. The
|
|
5
|
+
// default IdentityReranker preserves blended order (zero-config, no network).
|
|
6
|
+
// A LexicalOverlapReranker provides a lightweight, dependency-free boost based
|
|
7
|
+
// on query/term overlap — useful before a real cross-encoder is wired in.
|
|
8
|
+
// =============================================================================
|
|
9
|
+
/** No-op reranker — keeps the hybrid-blended order. */
|
|
10
|
+
export class IdentityReranker {
|
|
11
|
+
name = "identity";
|
|
12
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
13
|
+
async rerank(_query, candidates) {
|
|
14
|
+
return candidates;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Re-orders candidates by blending the hybrid score with lexical
|
|
19
|
+
* query-term coverage. Deterministic, no external calls.
|
|
20
|
+
*/
|
|
21
|
+
export class LexicalOverlapReranker {
|
|
22
|
+
name = "lexical-overlap";
|
|
23
|
+
weight;
|
|
24
|
+
constructor(weight = 0.3) {
|
|
25
|
+
this.weight = Math.min(1, Math.max(0, weight));
|
|
26
|
+
}
|
|
27
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
28
|
+
async rerank(query, candidates) {
|
|
29
|
+
const terms = new Set(query
|
|
30
|
+
.toLowerCase()
|
|
31
|
+
.split(/[^a-z0-9]+/)
|
|
32
|
+
.filter(Boolean));
|
|
33
|
+
if (terms.size === 0)
|
|
34
|
+
return candidates;
|
|
35
|
+
const rescored = candidates.map((c) => {
|
|
36
|
+
const text = c.chunk.text.toLowerCase();
|
|
37
|
+
let hit = 0;
|
|
38
|
+
for (const t of terms)
|
|
39
|
+
if (text.includes(t))
|
|
40
|
+
hit++;
|
|
41
|
+
const coverage = hit / terms.size;
|
|
42
|
+
const blended = (1 - this.weight) * c.score + this.weight * coverage;
|
|
43
|
+
return { ...c, score: blended };
|
|
44
|
+
});
|
|
45
|
+
rescored.sort((a, b) => b.score - a.score);
|
|
46
|
+
return rescored;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cosine similarity in [-1, 1]. Returns 0 for a zero-magnitude vector
|
|
3
|
+
* (never NaN). Throws on dimension mismatch.
|
|
4
|
+
*/
|
|
5
|
+
export declare function cosineSimilarity(a: readonly number[], b: readonly number[]): number;
|
|
6
|
+
/**
|
|
7
|
+
* Min-max normalise to [0, 1]. All-equal input → all 1 (avoids divide by
|
|
8
|
+
* zero and keeps every candidate eligible). Empty input → empty.
|
|
9
|
+
*/
|
|
10
|
+
export declare function normalizeScores(scores: readonly number[]): number[];
|
|
11
|
+
/**
|
|
12
|
+
* Weighted hybrid blend. `vectorWeight` is clamped to [0, 1]; the keyword
|
|
13
|
+
* leg receives `1 - vectorWeight`.
|
|
14
|
+
*/
|
|
15
|
+
export declare function hybridBlend(vectorScore: number, keywordScore: number, vectorWeight: number): number;
|
|
16
|
+
//# sourceMappingURL=scoring.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scoring.d.ts","sourceRoot":"","sources":["../src/scoring.ts"],"names":[],"mappings":"AASA;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAoBnF;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAWnE;AAOD;;;GAGG;AACH,wBAAgB,WAAW,CACzB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,GACnB,MAAM,CAGR"}
|
package/dist/scoring.js
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// =============================================================================
|
|
2
|
+
// @nebutra/knowledge-rag — Scoring math
|
|
3
|
+
// =============================================================================
|
|
4
|
+
// Pure functions: cosine similarity, min-max normalisation, weighted hybrid
|
|
5
|
+
// blend of the vector and keyword retrieval legs.
|
|
6
|
+
// =============================================================================
|
|
7
|
+
import { KnowledgeRagError } from "./errors";
|
|
8
|
+
/**
|
|
9
|
+
* Cosine similarity in [-1, 1]. Returns 0 for a zero-magnitude vector
|
|
10
|
+
* (never NaN). Throws on dimension mismatch.
|
|
11
|
+
*/
|
|
12
|
+
export function cosineSimilarity(a, b) {
|
|
13
|
+
if (a.length !== b.length) {
|
|
14
|
+
throw new KnowledgeRagError(`Vector dimension mismatch: ${a.length} vs ${b.length}`, {
|
|
15
|
+
code: "E_DIM_MISMATCH",
|
|
16
|
+
suggestion: "Ensure all chunks were embedded with the same embedder/model as the query. Re-ingest after switching embedder.",
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
let dot = 0;
|
|
20
|
+
let na = 0;
|
|
21
|
+
let nb = 0;
|
|
22
|
+
for (let i = 0; i < a.length; i++) {
|
|
23
|
+
const x = a[i];
|
|
24
|
+
const y = b[i];
|
|
25
|
+
dot += x * y;
|
|
26
|
+
na += x * x;
|
|
27
|
+
nb += y * y;
|
|
28
|
+
}
|
|
29
|
+
if (na === 0 || nb === 0)
|
|
30
|
+
return 0;
|
|
31
|
+
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Min-max normalise to [0, 1]. All-equal input → all 1 (avoids divide by
|
|
35
|
+
* zero and keeps every candidate eligible). Empty input → empty.
|
|
36
|
+
*/
|
|
37
|
+
export function normalizeScores(scores) {
|
|
38
|
+
if (scores.length === 0)
|
|
39
|
+
return [];
|
|
40
|
+
let min = Number.POSITIVE_INFINITY;
|
|
41
|
+
let max = Number.NEGATIVE_INFINITY;
|
|
42
|
+
for (const s of scores) {
|
|
43
|
+
if (s < min)
|
|
44
|
+
min = s;
|
|
45
|
+
if (s > max)
|
|
46
|
+
max = s;
|
|
47
|
+
}
|
|
48
|
+
const range = max - min;
|
|
49
|
+
if (range === 0)
|
|
50
|
+
return scores.map(() => 1);
|
|
51
|
+
return scores.map((s) => (s - min) / range);
|
|
52
|
+
}
|
|
53
|
+
/** Clamp a number into [lo, hi]. */
|
|
54
|
+
function clamp(x, lo, hi) {
|
|
55
|
+
return Math.min(hi, Math.max(lo, x));
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Weighted hybrid blend. `vectorWeight` is clamped to [0, 1]; the keyword
|
|
59
|
+
* leg receives `1 - vectorWeight`.
|
|
60
|
+
*/
|
|
61
|
+
export function hybridBlend(vectorScore, keywordScore, vectorWeight) {
|
|
62
|
+
const w = clamp(vectorWeight, 0, 1);
|
|
63
|
+
return w * vectorScore + (1 - w) * keywordScore;
|
|
64
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { KnowledgeChunk, VectorStore } from "../types";
|
|
2
|
+
export declare class InMemoryVectorStore implements VectorStore {
|
|
3
|
+
readonly name = "in-memory";
|
|
4
|
+
private readonly base;
|
|
5
|
+
upsert(chunks: KnowledgeChunk[]): Promise<void>;
|
|
6
|
+
queryByVector(tenantId: string, vector: readonly number[], topK: number): Promise<Array<{
|
|
7
|
+
chunk: KnowledgeChunk;
|
|
8
|
+
score: number;
|
|
9
|
+
}>>;
|
|
10
|
+
deleteByDoc(docId: string, tenantId: string): Promise<number>;
|
|
11
|
+
health(): Promise<{
|
|
12
|
+
ok: boolean;
|
|
13
|
+
detail: string;
|
|
14
|
+
}>;
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=memory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../src/stores/memory.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAE5D,qBAAa,mBAAoB,YAAW,WAAW;IACrD,QAAQ,CAAC,IAAI,eAAe;IAC5B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA6C;IAE5D,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM/C,aAAa,CACjB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,KAAK,CAAC;QAAE,KAAK,EAAE,cAAc,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAUrD,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAY7D,MAAM,IAAI,OAAO,CAAC;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAGzD"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// =============================================================================
|
|
2
|
+
// @nebutra/knowledge-rag — InMemory vector store (default, zero-config)
|
|
3
|
+
// =============================================================================
|
|
4
|
+
// Storage mechanics + tenant isolation are delegated to the neutral
|
|
5
|
+
// `@nebutra/tenant-store` lower layer (composite-key partition, tenant-checked
|
|
6
|
+
// reads/writes/deletes). This class only adds vector-specific behaviour
|
|
7
|
+
// (similarity scan, delete-by-doc). queryByVector NEVER reads outside the
|
|
8
|
+
// requested tenant — guaranteed structurally by InMemoryTenantStore.
|
|
9
|
+
// =============================================================================
|
|
10
|
+
import { InMemoryTenantStore } from "@nebutra/tenant-store";
|
|
11
|
+
import { cosineSimilarity } from "../scoring";
|
|
12
|
+
export class InMemoryVectorStore {
|
|
13
|
+
name = "in-memory";
|
|
14
|
+
base = new InMemoryTenantStore();
|
|
15
|
+
async upsert(chunks) {
|
|
16
|
+
for (const chunk of chunks) {
|
|
17
|
+
await this.base.write(chunk.tenantId, chunk.id, chunk);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
async queryByVector(tenantId, vector, topK) {
|
|
21
|
+
const chunks = await this.base.listByTenant(tenantId);
|
|
22
|
+
const scored = chunks.map((chunk) => ({
|
|
23
|
+
chunk,
|
|
24
|
+
score: cosineSimilarity(vector, chunk.embedding),
|
|
25
|
+
}));
|
|
26
|
+
scored.sort((a, b) => b.score - a.score);
|
|
27
|
+
return scored.slice(0, Math.max(0, topK));
|
|
28
|
+
}
|
|
29
|
+
async deleteByDoc(docId, tenantId) {
|
|
30
|
+
const chunks = await this.base.listByTenant(tenantId);
|
|
31
|
+
let removed = 0;
|
|
32
|
+
for (const chunk of chunks) {
|
|
33
|
+
if (chunk.docId === docId && (await this.base.delete(tenantId, chunk.id))) {
|
|
34
|
+
removed++;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return removed;
|
|
38
|
+
}
|
|
39
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
40
|
+
async health() {
|
|
41
|
+
return { ok: true, detail: `in-memory: ${this.base.size()} chunk(s)` };
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { KnowledgeChunk, VectorStore } from "../types";
|
|
2
|
+
export interface PgvectorExecutor {
|
|
3
|
+
/** Tagged-template raw query, e.g. Prisma's `$queryRaw`. */
|
|
4
|
+
query<T = unknown>(sql: string, params: unknown[]): Promise<T[]>;
|
|
5
|
+
}
|
|
6
|
+
export interface PgvectorStoreOptions {
|
|
7
|
+
executor: PgvectorExecutor;
|
|
8
|
+
/** Table name. Default: "knowledge_rag_chunk". */
|
|
9
|
+
table?: string;
|
|
10
|
+
/** Embedding dimension the table column was created with. Default 1536. */
|
|
11
|
+
embeddingDim?: number;
|
|
12
|
+
}
|
|
13
|
+
export declare class PgvectorStore implements VectorStore {
|
|
14
|
+
readonly name = "pgvector";
|
|
15
|
+
private readonly exec;
|
|
16
|
+
private readonly table;
|
|
17
|
+
private readonly dim;
|
|
18
|
+
constructor(options: PgvectorStoreOptions);
|
|
19
|
+
private vecLiteral;
|
|
20
|
+
upsert(chunks: KnowledgeChunk[]): Promise<void>;
|
|
21
|
+
queryByVector(tenantId: string, vector: readonly number[], topK: number): Promise<Array<{
|
|
22
|
+
chunk: KnowledgeChunk;
|
|
23
|
+
score: number;
|
|
24
|
+
}>>;
|
|
25
|
+
deleteByDoc(docId: string, tenantId: string): Promise<number>;
|
|
26
|
+
health(): Promise<{
|
|
27
|
+
ok: boolean;
|
|
28
|
+
detail: string;
|
|
29
|
+
}>;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=pgvector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pgvector.d.ts","sourceRoot":"","sources":["../../src/stores/pgvector.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAE5D,MAAM,WAAW,gBAAgB;IAC/B,4DAA4D;IAC5D,KAAK,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;CAClE;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,kDAAkD;IAClD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2EAA2E;IAC3E,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,qBAAa,aAAc,YAAW,WAAW;IAC/C,QAAQ,CAAC,IAAI,cAAc;IAC3B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;gBAEjB,OAAO,EAAE,oBAAoB;IAazC,OAAO,CAAC,UAAU;IAUZ,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB/C,aAAa,CACjB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,KAAK,CAAC;QAAE,KAAK,EAAE,cAAc,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAiCrD,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAW7D,MAAM,IAAI,OAAO,CAAC;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAWzD"}
|