@askdb/rag 0.2.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +10 -0
- package/README.md +76 -0
- package/bin/askdb-rag.js +2 -0
- package/dist/bin.d.ts +3 -0
- package/dist/bin.d.ts.map +1 -0
- package/dist/bin.js +7 -0
- package/dist/bin.js.map +1 -0
- package/dist/chunker/chunker.d.ts +36 -0
- package/dist/chunker/chunker.d.ts.map +1 -0
- package/dist/chunker/chunker.js +426 -0
- package/dist/chunker/chunker.js.map +1 -0
- package/dist/chunker/index.d.ts +10 -0
- package/dist/chunker/index.d.ts.map +1 -0
- package/dist/chunker/index.js +14 -0
- package/dist/chunker/index.js.map +1 -0
- package/dist/chunker/options.d.ts +24 -0
- package/dist/chunker/options.d.ts.map +1 -0
- package/dist/chunker/options.js +3 -0
- package/dist/chunker/options.js.map +1 -0
- package/dist/chunker/sources.d.ts +30 -0
- package/dist/chunker/sources.d.ts.map +1 -0
- package/dist/chunker/sources.js +68 -0
- package/dist/chunker/sources.js.map +1 -0
- package/dist/cli.d.ts +11 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +332 -0
- package/dist/cli.js.map +1 -0
- package/dist/embedders/ai-sdk.d.ts +22 -0
- package/dist/embedders/ai-sdk.d.ts.map +1 -0
- package/dist/embedders/ai-sdk.js +36 -0
- package/dist/embedders/ai-sdk.js.map +1 -0
- package/dist/embedders/openai.d.ts +21 -0
- package/dist/embedders/openai.d.ts.map +1 -0
- package/dist/embedders/openai.js +24 -0
- package/dist/embedders/openai.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +13 -0
- package/dist/index.js.map +1 -0
- package/dist/indexer/hash.d.ts +3 -0
- package/dist/indexer/hash.d.ts.map +1 -0
- package/dist/indexer/hash.js +6 -0
- package/dist/indexer/hash.js.map +1 -0
- package/dist/indexer/index.d.ts +72 -0
- package/dist/indexer/index.d.ts.map +1 -0
- package/dist/indexer/index.js +196 -0
- package/dist/indexer/index.js.map +1 -0
- package/dist/indexer/lock-file.d.ts +23 -0
- package/dist/indexer/lock-file.d.ts.map +1 -0
- package/dist/indexer/lock-file.js +33 -0
- package/dist/indexer/lock-file.js.map +1 -0
- package/dist/log-events.d.ts +23 -0
- package/dist/log-events.d.ts.map +1 -0
- package/dist/log-events.js +22 -0
- package/dist/log-events.js.map +1 -0
- package/dist/stores/file.d.ts +42 -0
- package/dist/stores/file.d.ts.map +1 -0
- package/dist/stores/file.js +127 -0
- package/dist/stores/file.js.map +1 -0
- package/dist/stores/memory.d.ts +34 -0
- package/dist/stores/memory.d.ts.map +1 -0
- package/dist/stores/memory.js +118 -0
- package/dist/stores/memory.js.map +1 -0
- package/dist/stores/pgvector.d.ts +41 -0
- package/dist/stores/pgvector.d.ts.map +1 -0
- package/dist/stores/pgvector.js +176 -0
- package/dist/stores/pgvector.js.map +1 -0
- package/dist/types.d.ts +89 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +9 -0
- package/dist/types.js.map +1 -0
- package/package.json +100 -0
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
export function createMemoryStore() {
|
|
2
|
+
const records = new Map();
|
|
3
|
+
let dimensions;
|
|
4
|
+
const upsert = async (input) => {
|
|
5
|
+
for (const r of input) {
|
|
6
|
+
const f32 = toFloat32(r.vector);
|
|
7
|
+
if (dimensions === undefined)
|
|
8
|
+
dimensions = f32.length;
|
|
9
|
+
else if (dimensions !== f32.length) {
|
|
10
|
+
throw new Error(`Vector dimension mismatch: expected ${dimensions}, got ${f32.length} for id="${r.id}".`);
|
|
11
|
+
}
|
|
12
|
+
const existing = records.get(r.id);
|
|
13
|
+
records.set(r.id, {
|
|
14
|
+
id: r.id,
|
|
15
|
+
vector: f32,
|
|
16
|
+
payload: r.payload,
|
|
17
|
+
hash: r.hash ?? existing?.hash,
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
const query = async (vector, k, filter) => {
|
|
22
|
+
const q = toFloat32(vector);
|
|
23
|
+
const qNorm = norm(q);
|
|
24
|
+
if (qNorm === 0)
|
|
25
|
+
return [];
|
|
26
|
+
const out = [];
|
|
27
|
+
for (const r of records.values()) {
|
|
28
|
+
if (filter && !matchesFilter(r.payload, filter))
|
|
29
|
+
continue;
|
|
30
|
+
const rNorm = norm(r.vector);
|
|
31
|
+
if (rNorm === 0)
|
|
32
|
+
continue;
|
|
33
|
+
const score = dot(q, r.vector) / (qNorm * rNorm);
|
|
34
|
+
out.push({ id: r.id, score, payload: r.payload });
|
|
35
|
+
}
|
|
36
|
+
// Stable sort: score desc, then id asc for deterministic ties.
|
|
37
|
+
out.sort((a, b) => b.score !== a.score ? b.score - a.score : a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
38
|
+
return out.slice(0, Math.max(0, k));
|
|
39
|
+
};
|
|
40
|
+
const del = async (ids) => {
|
|
41
|
+
for (const id of ids)
|
|
42
|
+
records.delete(id);
|
|
43
|
+
};
|
|
44
|
+
const hashesByPrefix = async (prefix) => {
|
|
45
|
+
const out = {};
|
|
46
|
+
for (const r of records.values()) {
|
|
47
|
+
if (r.hash && r.id.startsWith(prefix))
|
|
48
|
+
out[r.id] = r.hash;
|
|
49
|
+
}
|
|
50
|
+
return out;
|
|
51
|
+
};
|
|
52
|
+
return {
|
|
53
|
+
upsert,
|
|
54
|
+
query,
|
|
55
|
+
delete: del,
|
|
56
|
+
hashesByPrefix,
|
|
57
|
+
size() {
|
|
58
|
+
return records.size;
|
|
59
|
+
},
|
|
60
|
+
snapshot() {
|
|
61
|
+
return {
|
|
62
|
+
dimensions,
|
|
63
|
+
records: Array.from(records.values()).map((r) => ({
|
|
64
|
+
id: r.id,
|
|
65
|
+
vector: r.vector,
|
|
66
|
+
payload: r.payload,
|
|
67
|
+
hash: r.hash,
|
|
68
|
+
})),
|
|
69
|
+
};
|
|
70
|
+
},
|
|
71
|
+
restore(snap) {
|
|
72
|
+
records.clear();
|
|
73
|
+
dimensions = snap.dimensions;
|
|
74
|
+
for (const r of snap.records) {
|
|
75
|
+
records.set(r.id, {
|
|
76
|
+
id: r.id,
|
|
77
|
+
vector: toFloat32(r.vector),
|
|
78
|
+
payload: r.payload,
|
|
79
|
+
hash: r.hash,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
setHash(id, hash) {
|
|
84
|
+
const r = records.get(id);
|
|
85
|
+
if (r)
|
|
86
|
+
r.hash = hash;
|
|
87
|
+
},
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
function toFloat32(v) {
|
|
91
|
+
return v instanceof Float32Array ? v : Float32Array.from(v);
|
|
92
|
+
}
|
|
93
|
+
function dot(a, b) {
|
|
94
|
+
let s = 0;
|
|
95
|
+
const len = Math.min(a.length, b.length);
|
|
96
|
+
for (let i = 0; i < len; i++)
|
|
97
|
+
s += a[i] * b[i];
|
|
98
|
+
return s;
|
|
99
|
+
}
|
|
100
|
+
function norm(v) {
|
|
101
|
+
let s = 0;
|
|
102
|
+
for (let i = 0; i < v.length; i++)
|
|
103
|
+
s += v[i] * v[i];
|
|
104
|
+
return Math.sqrt(s);
|
|
105
|
+
}
|
|
106
|
+
function matchesFilter(payload, filter) {
|
|
107
|
+
if (filter.schemaId !== undefined && payload.schemaId !== filter.schemaId)
|
|
108
|
+
return false;
|
|
109
|
+
if (filter.types && filter.types.length > 0 && !filter.types.includes(payload.type))
|
|
110
|
+
return false;
|
|
111
|
+
if (filter.refs && filter.refs.length > 0) {
|
|
112
|
+
const set = new Set(payload.refs);
|
|
113
|
+
if (!filter.refs.some((r) => set.has(r)))
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
return true;
|
|
117
|
+
}
|
|
118
|
+
//# sourceMappingURL=memory.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../src/stores/memory.ts"],"names":[],"mappings":"AAmCA,MAAM,UAAU,iBAAiB;IAQ/B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAChD,IAAI,UAA8B,CAAC;IAEnC,MAAM,MAAM,GAAG,KAAK,EAAE,KAAqB,EAAiB,EAAE;QAC5D,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAChC,IAAI,UAAU,KAAK,SAAS;gBAAE,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC;iBACjD,IAAI,UAAU,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CACb,uCAAuC,UAAU,SAAS,GAAG,CAAC,MAAM,YAAY,CAAC,CAAC,EAAE,IAAI,CACzF,CAAC;YACJ,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBAChB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,QAAQ,EAAE,IAAI;aAC/B,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,KAAK,EACjB,MAAgB,EAChB,CAAS,EACT,MAAe,EACS,EAAE;QAC1B,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC3B,MAAM,GAAG,GAA2D,EAAE,CAAC;QACvE,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACjC,IAAI,MAAM,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC;gBAAE,SAAS;YAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC7B,IAAI,KAAK,KAAK,CAAC;gBAAE,SAAS;YAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;YACjD,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,+DAA+D;QAC/D,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAChB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACjF,CAAC;QACF,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtC,CAAC,CAAC;IAEF,MAAM,GAAG,GAAG,KAAK,EAAE,GAAa,EAAiB,EAAE;QACjD,KAAK,MAAM,EAAE,IAAI,GAAG;YAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC3C,CAAC,CAAC;IAEF,MAAM,cAAc,GAAG,KAAK,EAAE,MAAc,EAAmC,EAAE;QAC/E,MAAM,GAAG,GAA2B,EAAE,CAAC;QACvC,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACjC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;gBAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;QAC5D,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IAEF,OAAO;QACL,MAAM;QACN,KAAK;QACL,MAAM,EAAE,GAAG;QACX,cAAc;QACd,IAAI;YACF,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;QACD,QAAQ;YACN,OAAO;gBACL,UAAU;gBACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAChD,EAAE,EAAE,CAAC,CAAC,EAAE;oBACR,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,IAAI,EAAE,CAAC,CAAC,IAAI;iBACb,CAAC,CAAC;aACJ,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,IAAI;YACV,OAAO,CAAC,KAAK,EAAE,CAAC;YAChB,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;YAC7B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBAChB,EAAE,EAAE,CAAC,CAAC,EAAE;oBACR,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;oBAC3B,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,IAAI,EAAE,CAAC,CAAC,IAAI;iBACb,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,CAAC,EAAE,EAAE,IAAI;YACd,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC1B,IAAI,CAAC;gBAAE,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC;QACvB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAA0B;IAC3C,OAAO,CAAC,YAAY,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,GAAG,CAAC,CAAe,EAAE,CAAe;IAC3C,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;QAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,IAAI,CAAC,CAAe;IAC3B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,aAAa,CAAC,OAAqB,EAAE,MAAc;IAC1D,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IACxF,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAClG,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;IACzD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { VectorStore } from "../types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Minimal `pg`-compatible client shape so this module can stay free of a
|
|
4
|
+
* hard dependency on the `pg` package. Consumers pass either a `pg.Pool`,
|
|
5
|
+
* `pg.Client`, or any object exposing `query(text, params)`.
|
|
6
|
+
*/
|
|
7
|
+
export type PgClient = {
|
|
8
|
+
query: (text: string, params?: unknown[]) => Promise<{
|
|
9
|
+
rows: unknown[];
|
|
10
|
+
}>;
|
|
11
|
+
};
|
|
12
|
+
export type PgvectorIndexStrategy = "ivfflat" | "hnsw" | "none";
|
|
13
|
+
export type CreatePgvectorStoreOptions = {
|
|
14
|
+
/**
|
|
15
|
+
* Either a connection string (the adapter lazy-loads `pg`) or a pre-built
|
|
16
|
+
* client. When a client is supplied, the adapter never imports `pg`.
|
|
17
|
+
*/
|
|
18
|
+
connectionString?: string;
|
|
19
|
+
client?: PgClient;
|
|
20
|
+
/** Table name to read/write. Default `"askdb_rag_chunks"`. */
|
|
21
|
+
table?: string;
|
|
22
|
+
/** Embedding dimensions. Required — pgvector columns are dimension-typed. */
|
|
23
|
+
dimensions: number;
|
|
24
|
+
/** Index strategy hint, surfaced via the documented DDL helper. Default `"hnsw"`. */
|
|
25
|
+
indexStrategy?: PgvectorIndexStrategy;
|
|
26
|
+
};
|
|
27
|
+
export type PgvectorStore = VectorStore & {
|
|
28
|
+
/** Documented DDL the consumer must run before first use. */
|
|
29
|
+
setupSql(): string;
|
|
30
|
+
/** Close any pool the adapter built internally. No-op when an external client was supplied. */
|
|
31
|
+
close(): Promise<void>;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* pgvector adapter. Follows the same `VectorStore` interface as the other
|
|
35
|
+
* stores; cosine similarity via the `<=>` operator. Index creation is the
|
|
36
|
+
* consumer's responsibility — `setupSql()` returns documented DDL but is
|
|
37
|
+
* **not** auto-run (so accidental `CREATE EXTENSION` in production migrations
|
|
38
|
+
* doesn't surprise anyone).
|
|
39
|
+
*/
|
|
40
|
+
export declare function createPgvectorStore(options: CreatePgvectorStoreOptions): PgvectorStore;
|
|
41
|
+
//# sourceMappingURL=pgvector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pgvector.d.ts","sourceRoot":"","sources":["../../src/stores/pgvector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAMV,WAAW,EACZ,MAAM,aAAa,CAAC;AAErB;;;;GAIG;AACH,MAAM,MAAM,QAAQ,GAAG;IACrB,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC;QAAE,IAAI,EAAE,OAAO,EAAE,CAAA;KAAE,CAAC,CAAC;CAC3E,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;AAEhE,MAAM,MAAM,0BAA0B,GAAG;IACvC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,QAAQ,CAAC;IAClB,8DAA8D;IAC9D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,6EAA6E;IAC7E,UAAU,EAAE,MAAM,CAAC;IACnB,qFAAqF;IACrF,aAAa,CAAC,EAAE,qBAAqB,CAAC;CACvC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,WAAW,GAAG;IACxC,6DAA6D;IAC7D,QAAQ,IAAI,MAAM,CAAC;IACnB,+FAA+F;IAC/F,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAIF;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,0BAA0B,GAClC,aAAa,CAsLf"}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
const DEFAULT_TABLE = "askdb_rag_chunks";
|
|
2
|
+
/**
|
|
3
|
+
* pgvector adapter. Follows the same `VectorStore` interface as the other
|
|
4
|
+
* stores; cosine similarity via the `<=>` operator. Index creation is the
|
|
5
|
+
* consumer's responsibility — `setupSql()` returns documented DDL but is
|
|
6
|
+
* **not** auto-run (so accidental `CREATE EXTENSION` in production migrations
|
|
7
|
+
* doesn't surprise anyone).
|
|
8
|
+
*/
|
|
9
|
+
export function createPgvectorStore(options) {
|
|
10
|
+
const table = options.table ?? DEFAULT_TABLE;
|
|
11
|
+
const dimensions = options.dimensions;
|
|
12
|
+
const indexStrategy = options.indexStrategy ?? "hnsw";
|
|
13
|
+
if (!Number.isInteger(dimensions) || dimensions <= 0) {
|
|
14
|
+
throw new Error(`pgvector store requires positive integer dimensions; got ${dimensions}`);
|
|
15
|
+
}
|
|
16
|
+
let internalPool;
|
|
17
|
+
let client = options.client;
|
|
18
|
+
const getClient = async () => {
|
|
19
|
+
if (client)
|
|
20
|
+
return client;
|
|
21
|
+
if (!options.connectionString) {
|
|
22
|
+
throw new Error("createPgvectorStore: pass either `client` or `connectionString`.");
|
|
23
|
+
}
|
|
24
|
+
// Lazy-load `pg` so the package stays usable without it for the other stores.
|
|
25
|
+
const pgMod =
|
|
26
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
27
|
+
(await import("pg"));
|
|
28
|
+
const pool = new pgMod.Pool({ connectionString: options.connectionString });
|
|
29
|
+
internalPool = pool;
|
|
30
|
+
client = pool;
|
|
31
|
+
return pool;
|
|
32
|
+
};
|
|
33
|
+
const upsert = async (records) => {
|
|
34
|
+
if (records.length === 0)
|
|
35
|
+
return;
|
|
36
|
+
const c = await getClient();
|
|
37
|
+
// Single round-trip with parameterised UNNEST.
|
|
38
|
+
const ids = records.map((r) => r.id);
|
|
39
|
+
const types = records.map((r) => r.payload.type);
|
|
40
|
+
const texts = records.map((r) => r.payload.text);
|
|
41
|
+
const schemaIds = records.map((r) => r.payload.schemaId);
|
|
42
|
+
const refs = records.map((r) => JSON.stringify(r.payload.refs));
|
|
43
|
+
const sensitives = records.map((r) => r.payload.sensitive);
|
|
44
|
+
const vectors = records.map((r) => formatVector(r.vector));
|
|
45
|
+
const sql = `
|
|
46
|
+
INSERT INTO ${quoteIdent(table)} (id, type, text, schema_id, refs, sensitive, embedding)
|
|
47
|
+
SELECT
|
|
48
|
+
UNNEST($1::text[]),
|
|
49
|
+
UNNEST($2::text[]),
|
|
50
|
+
UNNEST($3::text[]),
|
|
51
|
+
UNNEST($4::text[]),
|
|
52
|
+
UNNEST($5::jsonb[]),
|
|
53
|
+
UNNEST($6::boolean[]),
|
|
54
|
+
UNNEST($7::vector[])
|
|
55
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
56
|
+
type = EXCLUDED.type,
|
|
57
|
+
text = EXCLUDED.text,
|
|
58
|
+
schema_id = EXCLUDED.schema_id,
|
|
59
|
+
refs = EXCLUDED.refs,
|
|
60
|
+
sensitive = EXCLUDED.sensitive,
|
|
61
|
+
embedding = EXCLUDED.embedding
|
|
62
|
+
`;
|
|
63
|
+
await c.query(sql, [ids, types, texts, schemaIds, refs, sensitives, vectors]);
|
|
64
|
+
};
|
|
65
|
+
const query = async (vector, k, filter) => {
|
|
66
|
+
const c = await getClient();
|
|
67
|
+
const wheres = [];
|
|
68
|
+
const params = [formatVector(vector), Math.max(0, k)];
|
|
69
|
+
let pIdx = 3;
|
|
70
|
+
if (filter?.schemaId !== undefined) {
|
|
71
|
+
wheres.push(`schema_id = $${pIdx++}`);
|
|
72
|
+
params.push(filter.schemaId);
|
|
73
|
+
}
|
|
74
|
+
if (filter?.types && filter.types.length > 0) {
|
|
75
|
+
wheres.push(`type = ANY($${pIdx++}::text[])`);
|
|
76
|
+
params.push(filter.types);
|
|
77
|
+
}
|
|
78
|
+
if (filter?.refs && filter.refs.length > 0) {
|
|
79
|
+
wheres.push(`refs ?| $${pIdx++}::text[]`);
|
|
80
|
+
params.push(filter.refs);
|
|
81
|
+
}
|
|
82
|
+
const whereSql = wheres.length > 0 ? `WHERE ${wheres.join(" AND ")}` : "";
|
|
83
|
+
const sql = `
|
|
84
|
+
SELECT
|
|
85
|
+
id,
|
|
86
|
+
type,
|
|
87
|
+
text,
|
|
88
|
+
schema_id,
|
|
89
|
+
refs,
|
|
90
|
+
sensitive,
|
|
91
|
+
1 - (embedding <=> $1::vector) AS score
|
|
92
|
+
FROM ${quoteIdent(table)}
|
|
93
|
+
${whereSql}
|
|
94
|
+
ORDER BY embedding <=> $1::vector ASC
|
|
95
|
+
LIMIT $2
|
|
96
|
+
`;
|
|
97
|
+
const result = await c.query(sql, params);
|
|
98
|
+
return result.rows.map((row) => {
|
|
99
|
+
const r = row;
|
|
100
|
+
const payload = {
|
|
101
|
+
id: r.id,
|
|
102
|
+
type: r.type,
|
|
103
|
+
text: r.text,
|
|
104
|
+
schemaId: r.schema_id,
|
|
105
|
+
refs: Array.isArray(r.refs) ? r.refs : [],
|
|
106
|
+
sensitive: r.sensitive,
|
|
107
|
+
};
|
|
108
|
+
return { id: r.id, score: r.score, payload };
|
|
109
|
+
});
|
|
110
|
+
};
|
|
111
|
+
const del = async (ids) => {
|
|
112
|
+
if (ids.length === 0)
|
|
113
|
+
return;
|
|
114
|
+
const c = await getClient();
|
|
115
|
+
await c.query(`DELETE FROM ${quoteIdent(table)} WHERE id = ANY($1::text[])`, [ids]);
|
|
116
|
+
};
|
|
117
|
+
const setupSql = () => {
|
|
118
|
+
const indexClause = indexStrategy === "hnsw"
|
|
119
|
+
? `CREATE INDEX IF NOT EXISTS ${quoteIdent(`${table}_embedding_hnsw`)} ON ${quoteIdent(table)} USING hnsw (embedding vector_cosine_ops);`
|
|
120
|
+
: indexStrategy === "ivfflat"
|
|
121
|
+
? `CREATE INDEX IF NOT EXISTS ${quoteIdent(`${table}_embedding_ivfflat`)} ON ${quoteIdent(table)} USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);`
|
|
122
|
+
: "";
|
|
123
|
+
return [
|
|
124
|
+
`CREATE EXTENSION IF NOT EXISTS vector;`,
|
|
125
|
+
`CREATE TABLE IF NOT EXISTS ${quoteIdent(table)} (`,
|
|
126
|
+
` id text PRIMARY KEY,`,
|
|
127
|
+
` type text NOT NULL,`,
|
|
128
|
+
` text text NOT NULL,`,
|
|
129
|
+
` schema_id text NOT NULL,`,
|
|
130
|
+
` refs jsonb NOT NULL DEFAULT '[]'::jsonb,`,
|
|
131
|
+
` sensitive boolean NOT NULL DEFAULT false,`,
|
|
132
|
+
` embedding vector(${dimensions}) NOT NULL`,
|
|
133
|
+
`);`,
|
|
134
|
+
`CREATE INDEX IF NOT EXISTS ${quoteIdent(`${table}_schema_id`)} ON ${quoteIdent(table)} (schema_id);`,
|
|
135
|
+
`CREATE INDEX IF NOT EXISTS ${quoteIdent(`${table}_type`)} ON ${quoteIdent(table)} (type);`,
|
|
136
|
+
`CREATE INDEX IF NOT EXISTS ${quoteIdent(`${table}_refs`)} ON ${quoteIdent(table)} USING gin (refs);`,
|
|
137
|
+
indexClause,
|
|
138
|
+
]
|
|
139
|
+
.filter(Boolean)
|
|
140
|
+
.join("\n");
|
|
141
|
+
};
|
|
142
|
+
const close = async () => {
|
|
143
|
+
if (internalPool) {
|
|
144
|
+
await internalPool.end();
|
|
145
|
+
internalPool = undefined;
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
const hashesByPrefix = async (prefix) => {
|
|
149
|
+
// pgvector store doesn't persist hashes itself — the indexer relies on
|
|
150
|
+
// `schema.lock.json`. Returning empty here means the indexer falls back
|
|
151
|
+
// to its file-based hash bookkeeping, which is the intended path.
|
|
152
|
+
void prefix;
|
|
153
|
+
return {};
|
|
154
|
+
};
|
|
155
|
+
return {
|
|
156
|
+
upsert,
|
|
157
|
+
query,
|
|
158
|
+
delete: del,
|
|
159
|
+
hashesByPrefix,
|
|
160
|
+
setupSql,
|
|
161
|
+
close,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
function quoteIdent(name) {
|
|
165
|
+
// Conservative identifier quoting — only allow alnum + underscore.
|
|
166
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
|
|
167
|
+
throw new Error(`Unsafe identifier for pgvector store: ${name}`);
|
|
168
|
+
}
|
|
169
|
+
return `"${name}"`;
|
|
170
|
+
}
|
|
171
|
+
function formatVector(v) {
|
|
172
|
+
// pgvector text format is `[v1,v2,...]`.
|
|
173
|
+
const arr = v instanceof Float32Array ? Array.from(v) : v;
|
|
174
|
+
return `[${arr.join(",")}]`;
|
|
175
|
+
}
|
|
176
|
+
//# sourceMappingURL=pgvector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pgvector.js","sourceRoot":"","sources":["../../src/stores/pgvector.ts"],"names":[],"mappings":"AA0CA,MAAM,aAAa,GAAG,kBAAkB,CAAC;AAEzC;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CACjC,OAAmC;IAEnC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,aAAa,CAAC;IAC7C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IACtC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,MAAM,CAAC;IAEtD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,4DAA4D,UAAU,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,IAAI,YAAsD,CAAC;IAC3D,IAAI,MAAM,GAAyB,OAAO,CAAC,MAAM,CAAC;IAElD,MAAM,SAAS,GAAG,KAAK,IAAuB,EAAE;QAC9C,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAC1B,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,kEAAkE,CACnE,CAAC;QACJ,CAAC;QACD,8EAA8E;QAC9E,MAAM,KAAK;QACT,iEAAiE;QACjE,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAiE,CAAC;QACvF,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAEzE,CAAC;QACF,YAAY,GAAG,IAAI,CAAC;QACpB,MAAM,GAAG,IAAI,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,KAAK,EAAE,OAAuB,EAAiB,EAAE;QAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACjC,MAAM,CAAC,GAAG,MAAM,SAAS,EAAE,CAAC;QAC5B,+CAA+C;QAC/C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAChE,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAE3D,MAAM,GAAG,GAAG;oBACI,UAAU,CAAC,KAAK,CAAC;;;;;;;;;;;;;;;;KAgBhC,CAAC;QACF,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IAChF,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,KAAK,EACjB,MAAgB,EAChB,CAAS,EACT,MAAe,EACS,EAAE;QAC1B,MAAM,CAAC,GAAG,MAAM,SAAS,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAc,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjE,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,MAAM,EAAE,QAAQ,KAAK,SAAS,EAAE,CAAC;YACnC,MAAM,CAAC,IAAI,CAAC,gBAAgB,IAAI,EAAE,EAAE,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,MAAM,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,WAAW,CAAC,CAAC;YAC9C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QACD,IAAI,MAAM,EAAE,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,UAAU,CAAC,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,MAAM,GAAG,GAAG;;;;;;;;;aASH,UAAU,CAAC,KAAK,CAAC;QACtB,QAAQ;;;KAGX,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC1C,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC7B,MAAM,CAAC,GAAG,GAQT,CAAC;YACF,MAAM,OAAO,GAAiB;gBAC5B,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,CAAC,CAAC,SAAS;gBACrB,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;gBACzC,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC;YACF,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,GAAG,GAAG,KAAK,EAAE,GAAa,EAAiB,EAAE;QACjD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC7B,MAAM,CAAC,GAAG,MAAM,SAAS,EAAE,CAAC;QAC5B,MAAM,CAAC,CAAC,KAAK,CACX,eAAe,UAAU,CAAC,KAAK,CAAC,6BAA6B,EAC7D,CAAC,GAAG,CAAC,CACN,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,GAAW,EAAE;QAC5B,MAAM,WAAW,GACf,aAAa,KAAK,MAAM;YACtB,CAAC,CAAC,8BAA8B,UAAU,CAAC,GAAG,KAAK,iBAAiB,CAAC,OAAO,UAAU,CAAC,KAAK,CAAC,4CAA4C;YACzI,CAAC,CAAC,aAAa,KAAK,SAAS;gBAC3B,CAAC,CAAC,8BAA8B,UAAU,CAAC,GAAG,KAAK,oBAAoB,CAAC,OAAO,UAAU,CAAC,KAAK,CAAC,kEAAkE;gBAClK,CAAC,CAAC,EAAE,CAAC;QACX,OAAO;YACL,wCAAwC;YACxC,8BAA8B,UAAU,CAAC,KAAK,CAAC,IAAI;YACnD,wBAAwB;YACxB,uBAAuB;YACvB,uBAAuB;YACvB,4BAA4B;YAC5B,4CAA4C;YAC5C,6CAA6C;YAC7C,sBAAsB,UAAU,YAAY;YAC5C,IAAI;YACJ,8BAA8B,UAAU,CAAC,GAAG,KAAK,YAAY,CAAC,OAAO,UAAU,CAAC,KAAK,CAAC,eAAe;YACrG,8BAA8B,UAAU,CAAC,GAAG,KAAK,OAAO,CAAC,OAAO,UAAU,CAAC,KAAK,CAAC,UAAU;YAC3F,8BAA8B,UAAU,CAAC,GAAG,KAAK,OAAO,CAAC,OAAO,UAAU,CAAC,KAAK,CAAC,oBAAoB;YACrG,WAAW;SACZ;aACE,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACtC,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,YAAY,CAAC,GAAG,EAAE,CAAC;YACzB,YAAY,GAAG,SAAS,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,cAAc,GAAG,KAAK,EAAE,MAAc,EAAmC,EAAE;QAC/E,uEAAuE;QACvE,wEAAwE;QACxE,kEAAkE;QAClE,KAAK,MAAM,CAAC;QACZ,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC;IAEF,OAAO;QACL,MAAM;QACN,KAAK;QACL,MAAM,EAAE,GAAG;QACX,cAAc;QACd,QAAQ;QACR,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,mEAAmE;IACnE,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,EAAE,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,IAAI,IAAI,GAAG,CAAC;AACrB,CAAC;AAED,SAAS,YAAY,CAAC,CAA0B;IAC9C,yCAAyC;IACzC,MAAM,GAAG,GAAG,CAAC,YAAY,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAC9B,CAAC"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public types for @askdb/rag.
|
|
3
|
+
*
|
|
4
|
+
* The chunker, indexer, and stores all converge on these shapes; nothing else
|
|
5
|
+
* leaks across the package boundary. Embedder and VectorStore are BYO seams —
|
|
6
|
+
* consumers wire their own providers behind these interfaces.
|
|
7
|
+
*/
|
|
8
|
+
/** Chunk type taxonomy from `docs/contracts/schema-v2.md` (Chunking rules). */
|
|
9
|
+
export type ChunkType = "table" | "column" | "cql" | "question" | "concept" | "relationship";
|
|
10
|
+
/**
|
|
11
|
+
* One slice of the v2 artifact, embedded as a single vector.
|
|
12
|
+
*
|
|
13
|
+
* `text` is what gets embedded. `id` is stable across runs (re-embedding is
|
|
14
|
+
* gated on text changes via `schema.lock.json`).
|
|
15
|
+
*/
|
|
16
|
+
export type Chunk = {
|
|
17
|
+
id: string;
|
|
18
|
+
type: ChunkType;
|
|
19
|
+
text: string;
|
|
20
|
+
schemaId: string;
|
|
21
|
+
/** Schema-v2 ids referenced by this chunk (for cross-link filtering). */
|
|
22
|
+
refs: string[];
|
|
23
|
+
/**
|
|
24
|
+
* True when the chunk's source content references a sensitive column.
|
|
25
|
+
* Default chunker excludes such chunks; this metadata flows through anyway
|
|
26
|
+
* so opt-in mode (`includeSensitiveDescribable: true`) can carry it for telemetry.
|
|
27
|
+
*/
|
|
28
|
+
sensitive: boolean;
|
|
29
|
+
};
|
|
30
|
+
/** Payload stored in vector stores alongside the embedding. */
|
|
31
|
+
export type ChunkPayload = {
|
|
32
|
+
id: string;
|
|
33
|
+
type: ChunkType;
|
|
34
|
+
text: string;
|
|
35
|
+
schemaId: string;
|
|
36
|
+
refs: string[];
|
|
37
|
+
sensitive: boolean;
|
|
38
|
+
};
|
|
39
|
+
/** Filter for vector-store queries. Every field is optional. */
|
|
40
|
+
export type Filter = {
|
|
41
|
+
schemaId?: string;
|
|
42
|
+
types?: ChunkType[];
|
|
43
|
+
refs?: string[];
|
|
44
|
+
};
|
|
45
|
+
/** Result of a vector-store similarity query. */
|
|
46
|
+
export type QueryResult = {
|
|
47
|
+
id: string;
|
|
48
|
+
score: number;
|
|
49
|
+
payload: ChunkPayload;
|
|
50
|
+
};
|
|
51
|
+
/** Single record passed to {@link VectorStore.upsert}. */
|
|
52
|
+
export type UpsertRecord = {
|
|
53
|
+
id: string;
|
|
54
|
+
vector: number[];
|
|
55
|
+
payload: ChunkPayload;
|
|
56
|
+
/** Optional content hash. Stores may persist it for `hashesByPrefix` reuse. */
|
|
57
|
+
hash?: string;
|
|
58
|
+
};
|
|
59
|
+
/**
|
|
60
|
+
* BYO embedder seam.
|
|
61
|
+
*
|
|
62
|
+
* Implementations batch as they like; callers pass whatever they want
|
|
63
|
+
* embedded together. No model-specific concerns leak across this seam.
|
|
64
|
+
*/
|
|
65
|
+
export type Embedder = (texts: string[]) => Promise<number[][]>;
|
|
66
|
+
/**
|
|
67
|
+
* BYO vector-store seam.
|
|
68
|
+
*
|
|
69
|
+
* Adapters: in-memory (default), file-backed, pgvector. Adding more is a
|
|
70
|
+
* pure adapter concern — nothing else in the package needs to know.
|
|
71
|
+
*/
|
|
72
|
+
export type VectorStore = {
|
|
73
|
+
upsert(records: UpsertRecord[]): Promise<void>;
|
|
74
|
+
query(vector: number[], k: number, filter?: Filter): Promise<QueryResult[]>;
|
|
75
|
+
delete(ids: string[]): Promise<void>;
|
|
76
|
+
/**
|
|
77
|
+
* Optional fast path used by the indexer to skip re-embedding unchanged
|
|
78
|
+
* chunks. Returns `chunkId → contentHash` for ids whose stored hash starts
|
|
79
|
+
* with `prefix`. Implementations without persistent hash storage can omit it.
|
|
80
|
+
*/
|
|
81
|
+
hashesByPrefix?(prefix: string): Promise<Record<string, string>>;
|
|
82
|
+
};
|
|
83
|
+
/** Retriever shape consumed by `@askdb/core` `ask({ retriever })`. */
|
|
84
|
+
export type Retriever = (params: {
|
|
85
|
+
question: string;
|
|
86
|
+
k?: number;
|
|
87
|
+
filter?: Filter;
|
|
88
|
+
}) => Promise<QueryResult[]>;
|
|
89
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,+EAA+E;AAC/E,MAAM,MAAM,SAAS,GACjB,OAAO,GACP,QAAQ,GACR,KAAK,GACL,UAAU,GACV,SAAS,GACT,cAAc,CAAC;AAEnB;;;;;GAKG;AACH,MAAM,MAAM,KAAK,GAAG;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,yEAAyE;IACzE,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;;;OAIG;IACH,SAAS,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,+DAA+D;AAC/D,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,SAAS,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,gEAAgE;AAChE,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAEF,iDAAiD;AACjD,MAAM,MAAM,WAAW,GAAG;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,0DAA0D;AAC1D,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,YAAY,CAAC;IACtB,+EAA+E;IAC/E,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAEhE;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrC;;;;OAIG;IACH,cAAc,CAAC,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAClE,CAAC;AAEF,sEAAsE;AACtE,MAAM,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public types for @askdb/rag.
|
|
3
|
+
*
|
|
4
|
+
* The chunker, indexer, and stores all converge on these shapes; nothing else
|
|
5
|
+
* leaks across the package boundary. Embedder and VectorStore are BYO seams —
|
|
6
|
+
* consumers wire their own providers behind these interfaces.
|
|
7
|
+
*/
|
|
8
|
+
export {};
|
|
9
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG"}
|
package/package.json
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@askdb/rag",
|
|
3
|
+
"version": "0.2.0-beta.1",
|
|
4
|
+
"description": "AskDB RAG layer: deterministic chunker over Schema v2, BYO embedder + vector store (in-memory, file-backed, pgvector), and an optional retriever wired into @askdb/core ask().",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"askdb",
|
|
7
|
+
"rag",
|
|
8
|
+
"retrieval",
|
|
9
|
+
"embeddings",
|
|
10
|
+
"schema-v2",
|
|
11
|
+
"pgvector",
|
|
12
|
+
"ai"
|
|
13
|
+
],
|
|
14
|
+
"homepage": "https://github.com/Ygilany/AskDB#readme",
|
|
15
|
+
"bugs": {
|
|
16
|
+
"url": "https://github.com/Ygilany/AskDB/issues"
|
|
17
|
+
},
|
|
18
|
+
"repository": {
|
|
19
|
+
"type": "git",
|
|
20
|
+
"url": "git+https://github.com/Ygilany/AskDB.git",
|
|
21
|
+
"directory": "packages/rag"
|
|
22
|
+
},
|
|
23
|
+
"license": "Apache-2.0",
|
|
24
|
+
"author": "Yahya Gilany",
|
|
25
|
+
"type": "module",
|
|
26
|
+
"main": "./dist/index.js",
|
|
27
|
+
"types": "./dist/index.d.ts",
|
|
28
|
+
"exports": {
|
|
29
|
+
".": {
|
|
30
|
+
"types": "./dist/index.d.ts",
|
|
31
|
+
"import": "./dist/index.js"
|
|
32
|
+
},
|
|
33
|
+
"./stores/memory": {
|
|
34
|
+
"types": "./dist/stores/memory.d.ts",
|
|
35
|
+
"import": "./dist/stores/memory.js"
|
|
36
|
+
},
|
|
37
|
+
"./stores/file": {
|
|
38
|
+
"types": "./dist/stores/file.d.ts",
|
|
39
|
+
"import": "./dist/stores/file.js"
|
|
40
|
+
},
|
|
41
|
+
"./stores/pgvector": {
|
|
42
|
+
"types": "./dist/stores/pgvector.d.ts",
|
|
43
|
+
"import": "./dist/stores/pgvector.js"
|
|
44
|
+
},
|
|
45
|
+
"./embedders/openai": {
|
|
46
|
+
"types": "./dist/embedders/openai.d.ts",
|
|
47
|
+
"import": "./dist/embedders/openai.js"
|
|
48
|
+
},
|
|
49
|
+
"./embedders/ai-sdk": {
|
|
50
|
+
"types": "./dist/embedders/ai-sdk.d.ts",
|
|
51
|
+
"import": "./dist/embedders/ai-sdk.js"
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
"bin": {
|
|
55
|
+
"askdb-rag": "./bin/askdb-rag.js"
|
|
56
|
+
},
|
|
57
|
+
"files": [
|
|
58
|
+
"bin",
|
|
59
|
+
"dist",
|
|
60
|
+
"README.md",
|
|
61
|
+
"LICENSE",
|
|
62
|
+
"NOTICE"
|
|
63
|
+
],
|
|
64
|
+
"engines": {
|
|
65
|
+
"node": ">=20"
|
|
66
|
+
},
|
|
67
|
+
"dependencies": {
|
|
68
|
+
"@askdb/core": "0.5.0-beta.0",
|
|
69
|
+
"@askdb/config": "0.3.0-beta.1"
|
|
70
|
+
},
|
|
71
|
+
"peerDependencies": {
|
|
72
|
+
"@ai-sdk/openai": ">=1",
|
|
73
|
+
"ai": ">=4",
|
|
74
|
+
"pg": ">=8"
|
|
75
|
+
},
|
|
76
|
+
"peerDependenciesMeta": {
|
|
77
|
+
"@ai-sdk/openai": {
|
|
78
|
+
"optional": true
|
|
79
|
+
},
|
|
80
|
+
"ai": {
|
|
81
|
+
"optional": true
|
|
82
|
+
},
|
|
83
|
+
"pg": {
|
|
84
|
+
"optional": true
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
"devDependencies": {
|
|
88
|
+
"@ai-sdk/openai": "^1.3.21",
|
|
89
|
+
"@types/pg": "^8.11.12",
|
|
90
|
+
"ai": "^4.3.19",
|
|
91
|
+
"pg": "^8.14.1",
|
|
92
|
+
"typescript": "^5.7.3",
|
|
93
|
+
"vitest": "^3.0.9"
|
|
94
|
+
},
|
|
95
|
+
"scripts": {
|
|
96
|
+
"build": "tsc -p tsconfig.build.json",
|
|
97
|
+
"lint": "tsc -p tsconfig.build.json --noEmit",
|
|
98
|
+
"test": "vitest run --config ../../vitest.config.ts"
|
|
99
|
+
}
|
|
100
|
+
}
|