@gnldev/rag 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +40 -0
- package/dist/chunk.d.ts +22 -0
- package/dist/chunk.js +127 -0
- package/dist/chunk.js.map +1 -0
- package/dist/graph-rag.d.ts +46 -0
- package/dist/graph-rag.js +152 -0
- package/dist/graph-rag.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -0
- package/dist/postgres-vector-store.d.ts +50 -0
- package/dist/postgres-vector-store.js +165 -0
- package/dist/postgres-vector-store.js.map +1 -0
- package/dist/rag-tool.d.ts +42 -0
- package/dist/rag-tool.js +30 -0
- package/dist/rag-tool.js.map +1 -0
- package/dist/rerank.d.ts +8 -0
- package/dist/rerank.js +38 -0
- package/dist/rerank.js.map +1 -0
- package/dist/semantic-memory.d.ts +41 -0
- package/dist/semantic-memory.js +90 -0
- package/dist/semantic-memory.js.map +1 -0
- package/dist/vector-store.d.ts +72 -0
- package/dist/vector-store.js +101 -0
- package/dist/vector-store.js.map +1 -0
- package/package.json +73 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
export interface VectorDoc {
|
|
2
|
+
id: string;
|
|
3
|
+
text: string;
|
|
4
|
+
metadata?: Record<string, unknown>;
|
|
5
|
+
/** 7.2: optional collection/namespace split — isolated data sets within the same store. */
|
|
6
|
+
namespace?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface VectorItem extends VectorDoc {
|
|
9
|
+
embedding: number[];
|
|
10
|
+
}
|
|
11
|
+
export interface VectorMatch extends VectorDoc {
|
|
12
|
+
score: number;
|
|
13
|
+
}
|
|
14
|
+
/** 7.2: query narrowing/blending options (all optional → old behavior if not given). */
|
|
15
|
+
export interface QueryOptions {
|
|
16
|
+
/** Only items in this namespace (matches item.namespace from upsert). All items if not given. */
|
|
17
|
+
namespace?: string;
|
|
18
|
+
/** Metadata SHALLOW equality filter: EVERY given key must match item.metadata with the SAME value. */
|
|
19
|
+
filter?: Record<string, unknown>;
|
|
20
|
+
/** Score threshold: matches whose final score falls BELOW this value are FILTERED OUT (cosine ~0..1). */
|
|
21
|
+
minScore?: number;
|
|
22
|
+
/** Hybrid search: if given, the keyword score (BM25-lite) is blended with the vector score. */
|
|
23
|
+
text?: string;
|
|
24
|
+
/** Hybrid weight w (0..1): final = (1-w)·vector + w·keyword. Default 0 → vector only (old behavior). */
|
|
25
|
+
keywordWeight?: number;
|
|
26
|
+
}
|
|
27
|
+
/** 7.2: delete condition — id list and/or metadata filter and/or namespace. */
|
|
28
|
+
export interface DeleteWhere {
|
|
29
|
+
ids?: string[];
|
|
30
|
+
filter?: Record<string, unknown>;
|
|
31
|
+
namespace?: string;
|
|
32
|
+
}
|
|
33
|
+
export interface VectorStore {
|
|
34
|
+
upsert(items: VectorItem[]): Promise<void>;
|
|
35
|
+
/** 7.2: opts is backward compatible — `query(embedding, topK)` behaves identically to before if not given. */
|
|
36
|
+
query(embedding: number[], topK: number, opts?: QueryOptions): Promise<VectorMatch[]>;
|
|
37
|
+
/** 7.2 (optional): delete by id/filter/namespace, returns the number deleted. */
|
|
38
|
+
delete?(where: DeleteWhere): Promise<number>;
|
|
39
|
+
}
|
|
40
|
+
/** text → embedding function. Wired to the AI SDK `embed` in prod; faked in tests. */
|
|
41
|
+
export type Embed = (text: string) => Promise<number[]>;
|
|
42
|
+
/**
|
|
43
|
+
* Simple tokenization: lowercase, then keep runs of letters and digits.
|
|
44
|
+
*
|
|
45
|
+
* Unicode-aware on purpose. The previous character class was `[^a-z0-9çğıöşü]`, i.e. ASCII plus the
|
|
46
|
+
* six Turkish letters — so `Grüße`, `français` and `mañana` were each split at the accent and scored
|
|
47
|
+
* against fragments, silently, on any non-English corpus. @gnldev/evals' tokenizer one package over
|
|
48
|
+
* already used \p{L}/\p{N}; this is the same reading. ASCII behaviour is unchanged.
|
|
49
|
+
*/
|
|
50
|
+
export declare function tokenize(s: string): string[];
|
|
51
|
+
/**
|
|
52
|
+
* BM25-lite keyword score (0..1): the ratio of query terms MATCHED in the document.
|
|
53
|
+
* Not full BM25 (no IDF/length normalization) — deliberately simple, "lite": a cheap,
|
|
54
|
+
* deterministic keyword signal meant to be blended with the vector score. 0 if the query is empty.
|
|
55
|
+
*/
|
|
56
|
+
export declare function keywordScore(query: string, doc: string): number;
|
|
57
|
+
/**
|
|
58
|
+
* Metadata shallow equality: does EVERY key in filter exist in item.metadata with the same value.
|
|
59
|
+
*
|
|
60
|
+
* Exported so the other in-repo `VectorStore` implementations narrow the same way. GraphRag used to
|
|
61
|
+
* implement none of this — the second copy of a filtering rule is where the copies start to differ.
|
|
62
|
+
*/
|
|
63
|
+
export declare function matchesFilter(metadata: Record<string, unknown> | undefined, filter?: Record<string, unknown>): boolean;
|
|
64
|
+
/** In-memory vector store (cosine similarity). The pgvector adapter implements the same interface for prod. */
|
|
65
|
+
export declare class InMemoryVectorStore implements VectorStore {
|
|
66
|
+
private items;
|
|
67
|
+
upsert(items: VectorItem[]): Promise<void>;
|
|
68
|
+
query(embedding: number[], topK: number, opts?: QueryOptions): Promise<VectorMatch[]>;
|
|
69
|
+
delete(where: DeleteWhere): Promise<number>;
|
|
70
|
+
}
|
|
71
|
+
/** Embeds documents and writes them to the store (namespace carried over via VectorDoc). */
|
|
72
|
+
export declare function indexDocuments(store: VectorStore, embed: Embed, docs: VectorDoc[]): Promise<void>;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { cosineSimilarity } from 'ai';
|
|
2
|
+
// ── 7.2 Hybrid helpers (pure, testable) ──────────────────────────
|
|
3
|
+
/**
|
|
4
|
+
* Simple tokenization: lowercase, then keep runs of letters and digits.
|
|
5
|
+
*
|
|
6
|
+
* Unicode-aware on purpose. The previous character class was `[^a-z0-9çğıöşü]`, i.e. ASCII plus the
|
|
7
|
+
* six Turkish letters — so `Grüße`, `français` and `mañana` were each split at the accent and scored
|
|
8
|
+
* against fragments, silently, on any non-English corpus. @gnldev/evals' tokenizer one package over
|
|
9
|
+
* already used \p{L}/\p{N}; this is the same reading. ASCII behaviour is unchanged.
|
|
10
|
+
*/
|
|
11
|
+
export function tokenize(s) {
|
|
12
|
+
return (s ?? '').toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* BM25-lite keyword score (0..1): the ratio of query terms MATCHED in the document.
|
|
16
|
+
* Not full BM25 (no IDF/length normalization) — deliberately simple, "lite": a cheap,
|
|
17
|
+
* deterministic keyword signal meant to be blended with the vector score. 0 if the query is empty.
|
|
18
|
+
*/
|
|
19
|
+
export function keywordScore(query, doc) {
|
|
20
|
+
const q = tokenize(query);
|
|
21
|
+
if (q.length === 0)
|
|
22
|
+
return 0;
|
|
23
|
+
const d = new Set(tokenize(doc));
|
|
24
|
+
let hit = 0;
|
|
25
|
+
for (const t of q)
|
|
26
|
+
if (d.has(t))
|
|
27
|
+
hit++;
|
|
28
|
+
return hit / q.length;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Metadata shallow equality: does EVERY key in filter exist in item.metadata with the same value.
|
|
32
|
+
*
|
|
33
|
+
* Exported so the other in-repo `VectorStore` implementations narrow the same way. GraphRag used to
|
|
34
|
+
* implement none of this — the second copy of a filtering rule is where the copies start to differ.
|
|
35
|
+
*/
|
|
36
|
+
export function matchesFilter(metadata, filter) {
|
|
37
|
+
if (!filter)
|
|
38
|
+
return true;
|
|
39
|
+
const m = metadata ?? {};
|
|
40
|
+
for (const [k, v] of Object.entries(filter)) {
|
|
41
|
+
if (m[k] !== v)
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
/** In-memory vector store (cosine similarity). The pgvector adapter implements the same interface for prod. */
|
|
47
|
+
export class InMemoryVectorStore {
|
|
48
|
+
items = [];
|
|
49
|
+
async upsert(items) {
|
|
50
|
+
for (const it of items) {
|
|
51
|
+
const i = this.items.findIndex((x) => x.id === it.id);
|
|
52
|
+
if (i >= 0)
|
|
53
|
+
this.items[i] = it;
|
|
54
|
+
else
|
|
55
|
+
this.items.push(it);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
async query(embedding, topK, opts) {
|
|
59
|
+
const w = opts?.keywordWeight ?? 0;
|
|
60
|
+
const hybrid = w > 0 && !!opts?.text;
|
|
61
|
+
const out = [];
|
|
62
|
+
for (const it of this.items) {
|
|
63
|
+
// 7.2: namespace + metadata narrowing (BEFORE score computation — no wasted work).
|
|
64
|
+
if (opts?.namespace !== undefined && it.namespace !== opts.namespace)
|
|
65
|
+
continue;
|
|
66
|
+
if (!matchesFilter(it.metadata, opts?.filter))
|
|
67
|
+
continue;
|
|
68
|
+
const vec = cosineSimilarity(embedding, it.embedding);
|
|
69
|
+
// 7.2: hybrid → (1-w)·vector + w·keyword; otherwise pure vector (old behavior).
|
|
70
|
+
const score = hybrid ? (1 - w) * vec + w * keywordScore(opts.text, it.text) : vec;
|
|
71
|
+
if (opts?.minScore !== undefined && score < opts.minScore)
|
|
72
|
+
continue;
|
|
73
|
+
out.push({ id: it.id, text: it.text, metadata: it.metadata, namespace: it.namespace, score });
|
|
74
|
+
}
|
|
75
|
+
return out.sort((a, b) => b.score - a.score).slice(0, topK);
|
|
76
|
+
}
|
|
77
|
+
async delete(where) {
|
|
78
|
+
const before = this.items.length;
|
|
79
|
+
const ids = where.ids ? new Set(where.ids) : undefined;
|
|
80
|
+
this.items = this.items.filter((it) => {
|
|
81
|
+
// Should it be deleted? ALL given conditions must match (ids ∧ filter ∧ namespace).
|
|
82
|
+
if (ids && !ids.has(it.id))
|
|
83
|
+
return true;
|
|
84
|
+
if (where.namespace !== undefined && it.namespace !== where.namespace)
|
|
85
|
+
return true;
|
|
86
|
+
if (where.filter && !matchesFilter(it.metadata, where.filter))
|
|
87
|
+
return true;
|
|
88
|
+
// If no condition was given (empty where), delete NOTHING (safe side).
|
|
89
|
+
if (!ids && where.namespace === undefined && !where.filter)
|
|
90
|
+
return true;
|
|
91
|
+
return false; // delete
|
|
92
|
+
});
|
|
93
|
+
return before - this.items.length;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/** Embeds documents and writes them to the store (namespace carried over via VectorDoc). */
|
|
97
|
+
export async function indexDocuments(store, embed, docs) {
|
|
98
|
+
const items = await Promise.all(docs.map(async (d) => ({ ...d, embedding: await embed(d.text) })));
|
|
99
|
+
await store.upsert(items);
|
|
100
|
+
}
|
|
101
|
+
//# sourceMappingURL=vector-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-store.js","sourceRoot":"","sources":["../src/vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAC;AAgDtC,oEAAoE;AAEpE;;;;;;;GAOG;AACH,MAAM,UAAU,QAAQ,CAAC,CAAS;IAChC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC;AAChE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,GAAW;IACrD,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7B,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,CAAC;QAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,GAAG,EAAE,CAAC;IACvC,OAAO,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC;AACxB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,QAA6C,EAAE,MAAgC;IAC3G,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACzB,MAAM,CAAC,GAAG,QAAQ,IAAI,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;IAC/B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,+GAA+G;AAC/G,MAAM,OAAO,mBAAmB;IACtB,KAAK,GAAiB,EAAE,CAAC;IAEjC,KAAK,CAAC,MAAM,CAAC,KAAmB;QAC9B,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YACtD,IAAI,CAAC,IAAI,CAAC;gBAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;;gBAC1B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,SAAmB,EAAE,IAAY,EAAE,IAAmB;QAChE,MAAM,CAAC,GAAG,IAAI,EAAE,aAAa,IAAI,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC;QACrC,MAAM,GAAG,GAAkB,EAAE,CAAC;QAC9B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC5B,mFAAmF;YACnF,IAAI,IAAI,EAAE,SAAS,KAAK,SAAS,IAAI,EAAE,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS;gBAAE,SAAS;YAC/E,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC;gBAAE,SAAS;YACxD,MAAM,GAAG,GAAG,gBAAgB,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC;YACtD,gFAAgF;YAChF,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,YAAY,CAAC,IAAK,CAAC,IAAK,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;YACpF,IAAI,IAAI,EAAE,QAAQ,KAAK,SAAS,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ;gBAAE,SAAS;YACpE,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAChG,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAkB;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QACjC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACvD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;YACpC,oFAAoF;YACpF,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAAE,OAAO,IAAI,CAAC;YACxC,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;gBAAE,OAAO,IAAI,CAAC;YACnF,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC3E,uEAAuE;YACvE,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,MAAM;gBAAE,OAAO,IAAI,CAAC;YACxE,OAAO,KAAK,CAAC,CAAC,SAAS;QACzB,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IACpC,CAAC;CACF;AAED,4FAA4F;AAC5F,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAkB,EAAE,KAAY,EAAE,IAAiB;IACtF,MAAM,KAAK,GAAiB,MAAM,OAAO,CAAC,GAAG,CAC3C,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAClE,CAAC;IACF,MAAM,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC","sourcesContent":["import { cosineSimilarity } from 'ai';\n\nexport interface VectorDoc {\n id: string;\n text: string;\n metadata?: Record<string, unknown>;\n /** 7.2: optional collection/namespace split — isolated data sets within the same store. */\n namespace?: string;\n}\nexport interface VectorItem extends VectorDoc {\n embedding: number[];\n}\nexport interface VectorMatch extends VectorDoc {\n score: number;\n}\n\n/** 7.2: query narrowing/blending options (all optional → old behavior if not given). */\nexport interface QueryOptions {\n /** Only items in this namespace (matches item.namespace from upsert). All items if not given. */\n namespace?: string;\n /** Metadata SHALLOW equality filter: EVERY given key must match item.metadata with the SAME value. */\n filter?: Record<string, unknown>;\n /** Score threshold: matches whose final score falls BELOW this value are FILTERED OUT (cosine ~0..1). */\n minScore?: number;\n /** Hybrid search: if given, the keyword score (BM25-lite) is blended with the vector score. */\n text?: string;\n /** Hybrid weight w (0..1): final = (1-w)·vector + w·keyword. Default 0 → vector only (old behavior). */\n keywordWeight?: number;\n}\n\n/** 7.2: delete condition — id list and/or metadata filter and/or namespace. */\nexport interface DeleteWhere {\n ids?: string[];\n filter?: Record<string, unknown>;\n namespace?: string;\n}\n\nexport interface VectorStore {\n upsert(items: VectorItem[]): Promise<void>;\n /** 7.2: opts is backward compatible — `query(embedding, topK)` behaves identically to before if not given. */\n query(embedding: number[], topK: number, opts?: QueryOptions): Promise<VectorMatch[]>;\n /** 7.2 (optional): delete by id/filter/namespace, returns the number deleted. */\n delete?(where: DeleteWhere): Promise<number>;\n}\n\n/** text → embedding function. Wired to the AI SDK `embed` in prod; faked in tests. */\nexport type Embed = (text: string) => Promise<number[]>;\n\n// ── 7.2 Hybrid helpers (pure, testable) ──────────────────────────\n\n/**\n * Simple tokenization: lowercase, then keep runs of letters and digits.\n *\n * Unicode-aware on purpose. The previous character class was `[^a-z0-9çğıöşü]`, i.e. ASCII plus the\n * six Turkish letters — so `Grüße`, `français` and `mañana` were each split at the accent and scored\n * against fragments, silently, on any non-English corpus. @gnldev/evals' tokenizer one package over\n * already used \\p{L}/\\p{N}; this is the same reading. ASCII behaviour is unchanged.\n */\nexport function tokenize(s: string): string[] {\n return (s ?? '').toLowerCase().match(/[\\p{L}\\p{N}]+/gu) ?? [];\n}\n\n/**\n * BM25-lite keyword score (0..1): the ratio of query terms MATCHED in the document.\n * Not full BM25 (no IDF/length normalization) — deliberately simple, \"lite\": a cheap,\n * deterministic keyword signal meant to be blended with the vector score. 0 if the query is empty.\n */\nexport function keywordScore(query: string, doc: string): number {\n const q = tokenize(query);\n if (q.length === 0) return 0;\n const d = new Set(tokenize(doc));\n let hit = 0;\n for (const t of q) if (d.has(t)) hit++;\n return hit / q.length;\n}\n\n/**\n * Metadata shallow equality: does EVERY key in filter exist in item.metadata with the same value.\n *\n * Exported so the other in-repo `VectorStore` implementations narrow the same way. GraphRag used to\n * implement none of this — the second copy of a filtering rule is where the copies start to differ.\n */\nexport function matchesFilter(metadata: Record<string, unknown> | undefined, filter?: Record<string, unknown>): boolean {\n if (!filter) return true;\n const m = metadata ?? {};\n for (const [k, v] of Object.entries(filter)) {\n if (m[k] !== v) return false;\n }\n return true;\n}\n\n/** In-memory vector store (cosine similarity). The pgvector adapter implements the same interface for prod. */\nexport class InMemoryVectorStore implements VectorStore {\n private items: VectorItem[] = [];\n\n async upsert(items: VectorItem[]): Promise<void> {\n for (const it of items) {\n const i = this.items.findIndex((x) => x.id === it.id);\n if (i >= 0) this.items[i] = it;\n else this.items.push(it);\n }\n }\n\n async query(embedding: number[], topK: number, opts?: QueryOptions): Promise<VectorMatch[]> {\n const w = opts?.keywordWeight ?? 0;\n const hybrid = w > 0 && !!opts?.text;\n const out: VectorMatch[] = [];\n for (const it of this.items) {\n // 7.2: namespace + metadata narrowing (BEFORE score computation — no wasted work).\n if (opts?.namespace !== undefined && it.namespace !== opts.namespace) continue;\n if (!matchesFilter(it.metadata, opts?.filter)) continue;\n const vec = cosineSimilarity(embedding, it.embedding);\n // 7.2: hybrid → (1-w)·vector + w·keyword; otherwise pure vector (old behavior).\n const score = hybrid ? (1 - w) * vec + w * keywordScore(opts!.text!, it.text) : vec;\n if (opts?.minScore !== undefined && score < opts.minScore) continue;\n out.push({ id: it.id, text: it.text, metadata: it.metadata, namespace: it.namespace, score });\n }\n return out.sort((a, b) => b.score - a.score).slice(0, topK);\n }\n\n async delete(where: DeleteWhere): Promise<number> {\n const before = this.items.length;\n const ids = where.ids ? new Set(where.ids) : undefined;\n this.items = this.items.filter((it) => {\n // Should it be deleted? ALL given conditions must match (ids ∧ filter ∧ namespace).\n if (ids && !ids.has(it.id)) return true;\n if (where.namespace !== undefined && it.namespace !== where.namespace) return true;\n if (where.filter && !matchesFilter(it.metadata, where.filter)) return true;\n // If no condition was given (empty where), delete NOTHING (safe side).\n if (!ids && where.namespace === undefined && !where.filter) return true;\n return false; // delete\n });\n return before - this.items.length;\n }\n}\n\n/** Embeds documents and writes them to the store (namespace carried over via VectorDoc). */\nexport async function indexDocuments(store: VectorStore, embed: Embed, docs: VectorDoc[]): Promise<void> {\n const items: VectorItem[] = await Promise.all(\n docs.map(async (d) => ({ ...d, embedding: await embed(d.text) })),\n );\n await store.upsert(items);\n}\n"]}
|
package/package.json
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@gnldev/rag",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"license": "Apache-2.0",
|
|
5
|
+
"engines": {
|
|
6
|
+
"node": ">=22.13.0"
|
|
7
|
+
},
|
|
8
|
+
"description": "Vector store + RAG tool for AI SDK agents. Used as a tool, it becomes replayable/exactly-once RAG with @gnldev/durable.",
|
|
9
|
+
"keywords": [
|
|
10
|
+
"ai",
|
|
11
|
+
"agent",
|
|
12
|
+
"llm",
|
|
13
|
+
"typescript",
|
|
14
|
+
"ai-sdk",
|
|
15
|
+
"durable",
|
|
16
|
+
"exactly-once",
|
|
17
|
+
"retrieval",
|
|
18
|
+
"embeddings",
|
|
19
|
+
"vector-search"
|
|
20
|
+
],
|
|
21
|
+
"type": "module",
|
|
22
|
+
"main": "./dist/index.js",
|
|
23
|
+
"types": "./dist/index.d.ts",
|
|
24
|
+
"exports": {
|
|
25
|
+
".": {
|
|
26
|
+
"types": "./dist/index.d.ts",
|
|
27
|
+
"default": "./dist/index.js"
|
|
28
|
+
},
|
|
29
|
+
"./postgres": {
|
|
30
|
+
"types": "./dist/postgres-vector-store.d.ts",
|
|
31
|
+
"default": "./dist/postgres-vector-store.js"
|
|
32
|
+
},
|
|
33
|
+
"./package.json": "./package.json"
|
|
34
|
+
},
|
|
35
|
+
"files": [
|
|
36
|
+
"dist"
|
|
37
|
+
],
|
|
38
|
+
"peerDependencies": {
|
|
39
|
+
"ai": "^7.0.0",
|
|
40
|
+
"pg": "^8.0.0",
|
|
41
|
+
"zod": "^3.25.76 || ^4.1.8",
|
|
42
|
+
"@gnldev/durable": "^0.1.0"
|
|
43
|
+
},
|
|
44
|
+
"peerDependenciesMeta": {
|
|
45
|
+
"pg": {
|
|
46
|
+
"optional": true
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
"devDependencies": {
|
|
50
|
+
"ai": "^7.0.0",
|
|
51
|
+
"pg": "^8.0.0",
|
|
52
|
+
"zod": "^3.25.0",
|
|
53
|
+
"@gnldev/durable": "0.1.0"
|
|
54
|
+
},
|
|
55
|
+
"author": "Karaca Yılmaz (https://gnl.dev)",
|
|
56
|
+
"homepage": "https://gnl.dev",
|
|
57
|
+
"bugs": {
|
|
58
|
+
"url": "https://github.com/Karaca7/gnldev/issues"
|
|
59
|
+
},
|
|
60
|
+
"repository": {
|
|
61
|
+
"type": "git",
|
|
62
|
+
"url": "git+https://github.com/Karaca7/gnldev.git",
|
|
63
|
+
"directory": "packages/rag"
|
|
64
|
+
},
|
|
65
|
+
"publishConfig": {
|
|
66
|
+
"access": "public"
|
|
67
|
+
},
|
|
68
|
+
"scripts": {
|
|
69
|
+
"build": "tsc -p tsconfig.json",
|
|
70
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
71
|
+
"test": "vitest run"
|
|
72
|
+
}
|
|
73
|
+
}
|