@nebutra/knowledge-rag 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/embedder.d.ts +0 -1
- package/dist/embedder.d.ts.map +1 -1
- package/dist/embedder.js +2 -44
- package/dist/scoring.d.ts.map +1 -1
- package/dist/scoring.js +2 -17
- package/package.json +13 -7
package/dist/embedder.d.ts
CHANGED
package/dist/embedder.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,qBAAa,iBAAkB,YAAW,QAAQ;IAChD,QAAQ,CAAC,IAAI,gBAAgB;IAC7B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;gBAEjB,GAAG,SAAM;IAWf,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAGlD;AAED;;;;GAIG;AACH,qBAAa,gBAAiB,YAAW,QAAQ;IAC/C,QAAQ,CAAC,IAAI,oBAAoB;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;gBAE/B,KAAK,CAAC,EAAE,MAAM;IAIpB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA0BlD"}
|
package/dist/embedder.js
CHANGED
|
@@ -8,22 +8,8 @@
|
|
|
8
8
|
// - ProviderEmbedder: wraps @nebutra/agents `embedMany()` (Vercel AI SDK
|
|
9
9
|
// fallback chain) when real embeddings are configured.
|
|
10
10
|
// =============================================================================
|
|
11
|
+
import { embedTextLocal } from "@nebutra/local-embedding";
|
|
11
12
|
import { KnowledgeRagError } from "./errors";
|
|
12
|
-
/** FNV-1a 32-bit hash — fast, deterministic, well-distributed. */
|
|
13
|
-
function fnv1a(input) {
|
|
14
|
-
let h = 0x811c9dc5;
|
|
15
|
-
for (let i = 0; i < input.length; i++) {
|
|
16
|
-
h ^= input.charCodeAt(i);
|
|
17
|
-
h = Math.imul(h, 0x01000193);
|
|
18
|
-
}
|
|
19
|
-
return h >>> 0;
|
|
20
|
-
}
|
|
21
|
-
function tokenize(text) {
|
|
22
|
-
return text
|
|
23
|
-
.toLowerCase()
|
|
24
|
-
.split(/[^a-z0-9]+/)
|
|
25
|
-
.filter((t) => t.length > 0);
|
|
26
|
-
}
|
|
27
13
|
export class LocalHashEmbedder {
|
|
28
14
|
name = "local-hash";
|
|
29
15
|
dim;
|
|
@@ -38,35 +24,7 @@ export class LocalHashEmbedder {
|
|
|
38
24
|
}
|
|
39
25
|
// eslint-disable-next-line @typescript-eslint/require-await
|
|
40
26
|
async embed(texts) {
|
|
41
|
-
return texts.map((
|
|
42
|
-
}
|
|
43
|
-
embedOne(text) {
|
|
44
|
-
const vec = new Array(this.dim).fill(0);
|
|
45
|
-
const tokens = tokenize(text);
|
|
46
|
-
for (const tok of tokens) {
|
|
47
|
-
// Signed feature hashing reduces collision bias.
|
|
48
|
-
const h = fnv1a(tok);
|
|
49
|
-
const idx = h % this.dim;
|
|
50
|
-
const sign = (h & 1) === 0 ? 1 : -1;
|
|
51
|
-
vec[idx] += sign;
|
|
52
|
-
// Character bigrams add sub-word locality (typo / morphology robustness).
|
|
53
|
-
for (let i = 0; i < tok.length - 1; i++) {
|
|
54
|
-
const bg = fnv1a(`#${tok.slice(i, i + 2)}`);
|
|
55
|
-
vec[bg % this.dim] += (bg & 1) === 0 ? 0.5 : -0.5;
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
let norm = 0;
|
|
59
|
-
for (const v of vec)
|
|
60
|
-
norm += v * v;
|
|
61
|
-
norm = Math.sqrt(norm);
|
|
62
|
-
if (norm === 0) {
|
|
63
|
-
// Empty / symbol-only text — deterministic unit vector on axis 0.
|
|
64
|
-
vec[0] = 1;
|
|
65
|
-
return vec;
|
|
66
|
-
}
|
|
67
|
-
for (let i = 0; i < vec.length; i++)
|
|
68
|
-
vec[i] /= norm;
|
|
69
|
-
return vec;
|
|
27
|
+
return texts.map((text) => embedTextLocal(text, { dimensions: this.dim }));
|
|
70
28
|
}
|
|
71
29
|
}
|
|
72
30
|
/**
|
package/dist/scoring.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scoring.d.ts","sourceRoot":"","sources":["../src/scoring.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"scoring.d.ts","sourceRoot":"","sources":["../src/scoring.ts"],"names":[],"mappings":"AAUA;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CASnF;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAWnE;AAED;;;GAGG;AACH,wBAAgB,WAAW,CACzB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,GACnB,MAAM,CAGR"}
|
package/dist/scoring.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// Pure functions: cosine similarity, min-max normalisation, weighted hybrid
|
|
5
5
|
// blend of the vector and keyword retrieval legs.
|
|
6
6
|
// =============================================================================
|
|
7
|
+
import { clamp, cosineSimilarity as primitiveCosineSimilarity } from "@nebutra/ai-primitives";
|
|
7
8
|
import { KnowledgeRagError } from "./errors";
|
|
8
9
|
/**
|
|
9
10
|
* Cosine similarity in [-1, 1]. Returns 0 for a zero-magnitude vector
|
|
@@ -16,19 +17,7 @@ export function cosineSimilarity(a, b) {
|
|
|
16
17
|
suggestion: "Ensure all chunks were embedded with the same embedder/model as the query. Re-ingest after switching embedder.",
|
|
17
18
|
});
|
|
18
19
|
}
|
|
19
|
-
|
|
20
|
-
let na = 0;
|
|
21
|
-
let nb = 0;
|
|
22
|
-
for (let i = 0; i < a.length; i++) {
|
|
23
|
-
const x = a[i];
|
|
24
|
-
const y = b[i];
|
|
25
|
-
dot += x * y;
|
|
26
|
-
na += x * x;
|
|
27
|
-
nb += y * y;
|
|
28
|
-
}
|
|
29
|
-
if (na === 0 || nb === 0)
|
|
30
|
-
return 0;
|
|
31
|
-
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
20
|
+
return primitiveCosineSimilarity(a, b, { onMismatch: "throw" });
|
|
32
21
|
}
|
|
33
22
|
/**
|
|
34
23
|
* Min-max normalise to [0, 1]. All-equal input → all 1 (avoids divide by
|
|
@@ -50,10 +39,6 @@ export function normalizeScores(scores) {
|
|
|
50
39
|
return scores.map(() => 1);
|
|
51
40
|
return scores.map((s) => (s - min) / range);
|
|
52
41
|
}
|
|
53
|
-
/** Clamp a number into [lo, hi]. */
|
|
54
|
-
function clamp(x, lo, hi) {
|
|
55
|
-
return Math.min(hi, Math.max(lo, x));
|
|
56
|
-
}
|
|
57
42
|
/**
|
|
58
43
|
* Weighted hybrid blend. `vectorWeight` is clamped to [0, 1]; the keyword
|
|
59
44
|
* leg receives `1 - vectorWeight`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nebutra/knowledge-rag",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"private": false,
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -16,7 +16,11 @@
|
|
|
16
16
|
"gaps": [
|
|
17
17
|
"pgvector store is interface-only (Prisma snippet in README, not migrated)",
|
|
18
18
|
"reranker interface ships with identity impl only"
|
|
19
|
-
]
|
|
19
|
+
],
|
|
20
|
+
"featureId": "knowledge-rag",
|
|
21
|
+
"category": "ai",
|
|
22
|
+
"surface": "semantic-index",
|
|
23
|
+
"summary": "Knowledge retrieval grammar with zero-config local embedder and pluggable vector/keyword/rerank ports"
|
|
20
24
|
},
|
|
21
25
|
"main": "./dist/index.js",
|
|
22
26
|
"types": "./dist/index.d.ts",
|
|
@@ -32,11 +36,13 @@
|
|
|
32
36
|
],
|
|
33
37
|
"dependencies": {
|
|
34
38
|
"zod": "^4.3.6",
|
|
35
|
-
"@nebutra/
|
|
36
|
-
"@nebutra/
|
|
37
|
-
"@nebutra/
|
|
38
|
-
"@nebutra/
|
|
39
|
-
"@nebutra/
|
|
39
|
+
"@nebutra/ai-primitives": "0.1.1",
|
|
40
|
+
"@nebutra/agents": "1.1.1",
|
|
41
|
+
"@nebutra/errors": "0.1.1",
|
|
42
|
+
"@nebutra/local-embedding": "0.1.1",
|
|
43
|
+
"@nebutra/logger": "0.1.1",
|
|
44
|
+
"@nebutra/search": "0.1.1",
|
|
45
|
+
"@nebutra/tenant-store": "0.2.1"
|
|
40
46
|
},
|
|
41
47
|
"devDependencies": {
|
|
42
48
|
"typescript": "^5.9.3",
|