@juspay/neurolink 9.89.0 → 9.91.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/adapters/tts/googleTTSHandler.d.ts +10 -0
- package/dist/adapters/tts/googleTTSHandler.js +27 -18
- package/dist/agent/directTools.js +1 -1
- package/dist/browser/neurolink.min.js +387 -365
- package/dist/core/baseProvider.d.ts +37 -3
- package/dist/core/baseProvider.js +167 -44
- package/dist/core/modules/GenerationHandler.js +7 -1
- package/dist/core/modules/ToolsManager.js +5 -0
- package/dist/core/toolDedup.js +4 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/lib/adapters/tts/googleTTSHandler.d.ts +10 -0
- package/dist/lib/adapters/tts/googleTTSHandler.js +27 -18
- package/dist/lib/agent/directTools.js +1 -1
- package/dist/lib/core/baseProvider.d.ts +37 -3
- package/dist/lib/core/baseProvider.js +167 -44
- package/dist/lib/core/modules/GenerationHandler.js +7 -1
- package/dist/lib/core/modules/ToolsManager.js +5 -0
- package/dist/lib/core/toolDedup.js +4 -1
- package/dist/lib/index.d.ts +1 -1
- package/dist/lib/index.js +1 -1
- package/dist/lib/mcp/toolRegistry.js +7 -2
- package/dist/lib/neurolink.d.ts +31 -1
- package/dist/lib/neurolink.js +151 -26
- package/dist/lib/rag/index.d.ts +1 -0
- package/dist/lib/rag/index.js +2 -0
- package/dist/lib/rag/stores/chroma.d.ts +90 -0
- package/dist/lib/rag/stores/chroma.js +281 -0
- package/dist/lib/rag/stores/index.d.ts +21 -0
- package/dist/lib/rag/stores/index.js +22 -0
- package/dist/lib/rag/stores/pgvector.d.ts +95 -0
- package/dist/lib/rag/stores/pgvector.js +400 -0
- package/dist/lib/rag/stores/pinecone.d.ts +85 -0
- package/dist/lib/rag/stores/pinecone.js +159 -0
- package/dist/lib/server/routes/agentRoutes.js +25 -2
- package/dist/lib/tools/toolDiscovery.d.ts +48 -0
- package/dist/lib/tools/toolDiscovery.js +231 -0
- package/dist/lib/tools/toolGate.d.ts +16 -0
- package/dist/lib/tools/toolGate.js +51 -0
- package/dist/lib/tools/toolPolicy.d.ts +40 -0
- package/dist/lib/tools/toolPolicy.js +194 -0
- package/dist/lib/types/config.d.ts +43 -3
- package/dist/lib/types/index.d.ts +3 -0
- package/dist/lib/types/index.js +3 -0
- package/dist/lib/types/providers.d.ts +8 -0
- package/dist/lib/types/rag.d.ts +30 -0
- package/dist/lib/types/toolResolution.d.ts +73 -0
- package/dist/lib/types/toolResolution.js +11 -0
- package/dist/lib/types/vectorStoreChroma.d.ts +67 -0
- package/dist/lib/types/vectorStoreChroma.js +12 -0
- package/dist/lib/types/vectorStorePinecone.d.ts +48 -0
- package/dist/lib/types/vectorStorePinecone.js +12 -0
- package/dist/mcp/toolRegistry.js +7 -2
- package/dist/neurolink.d.ts +31 -1
- package/dist/neurolink.js +151 -26
- package/dist/rag/index.d.ts +1 -0
- package/dist/rag/index.js +2 -0
- package/dist/rag/stores/chroma.d.ts +90 -0
- package/dist/rag/stores/chroma.js +280 -0
- package/dist/rag/stores/index.d.ts +21 -0
- package/dist/rag/stores/index.js +21 -0
- package/dist/rag/stores/pgvector.d.ts +95 -0
- package/dist/rag/stores/pgvector.js +399 -0
- package/dist/rag/stores/pinecone.d.ts +85 -0
- package/dist/rag/stores/pinecone.js +158 -0
- package/dist/server/routes/agentRoutes.js +25 -2
- package/dist/tools/toolDiscovery.d.ts +48 -0
- package/dist/tools/toolDiscovery.js +230 -0
- package/dist/tools/toolGate.d.ts +16 -0
- package/dist/tools/toolGate.js +50 -0
- package/dist/tools/toolPolicy.d.ts +40 -0
- package/dist/tools/toolPolicy.js +193 -0
- package/dist/types/config.d.ts +43 -3
- package/dist/types/index.d.ts +3 -0
- package/dist/types/index.js +3 -0
- package/dist/types/providers.d.ts +8 -0
- package/dist/types/rag.d.ts +30 -0
- package/dist/types/toolResolution.d.ts +73 -0
- package/dist/types/toolResolution.js +10 -0
- package/dist/types/vectorStoreChroma.d.ts +67 -0
- package/dist/types/vectorStoreChroma.js +11 -0
- package/dist/types/vectorStorePinecone.d.ts +48 -0
- package/dist/types/vectorStorePinecone.js +11 -0
- package/package.json +12 -7
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vector Store Adapters
|
|
3
|
+
*
|
|
4
|
+
* Client-injection adapters implementing the `VectorStore` contract
|
|
5
|
+
* (see `src/lib/types/rag.ts`) against real vector database backends.
|
|
6
|
+
* Each adapter accepts an already-constructed vendor client typed as a
|
|
7
|
+
* minimal structural interface — no vendor SDK is a runtime dependency
|
|
8
|
+
* of this package; callers bring their own client instance.
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```typescript
|
|
12
|
+
* import { PineconeVectorStore } from '@juspay/neurolink/rag';
|
|
13
|
+
* import { Pinecone } from '@pinecone-database/pinecone';
|
|
14
|
+
*
|
|
15
|
+
* const client = new Pinecone({ apiKey: '...' });
|
|
16
|
+
* const store = new PineconeVectorStore(client.index('my-index'));
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
export * from "./pinecone.js";
|
|
20
|
+
export * from "./pgvector.js";
|
|
21
|
+
export * from "./chroma.js";
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vector Store Adapters
|
|
3
|
+
*
|
|
4
|
+
* Client-injection adapters implementing the `VectorStore` contract
|
|
5
|
+
* (see `src/lib/types/rag.ts`) against real vector database backends.
|
|
6
|
+
* Each adapter accepts an already-constructed vendor client typed as a
|
|
7
|
+
* minimal structural interface — no vendor SDK is a runtime dependency
|
|
8
|
+
* of this package; callers bring their own client instance.
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```typescript
|
|
12
|
+
* import { PineconeVectorStore } from '@juspay/neurolink/rag';
|
|
13
|
+
* import { Pinecone } from '@pinecone-database/pinecone';
|
|
14
|
+
*
|
|
15
|
+
* const client = new Pinecone({ apiKey: '...' });
|
|
16
|
+
* const store = new PineconeVectorStore(client.index('my-index'));
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
export * from "./pinecone.js";
|
|
20
|
+
export * from "./pgvector.js";
|
|
21
|
+
export * from "./chroma.js";
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* pgvector Vector Store Adapter
|
|
3
|
+
*
|
|
4
|
+
* Client-injection `VectorStore` implementation backed by Postgres +
|
|
5
|
+
* the `pgvector` extension. Callers construct and own the Postgres client
|
|
6
|
+
* (e.g. a `pg.Pool`, or an in-process `@electric-sql/pglite` instance) and
|
|
7
|
+
* pass it in — this module adds zero vendor SDKs as runtime dependencies.
|
|
8
|
+
*
|
|
9
|
+
* Storage model: one table per `indexName`, created lazily on first
|
|
10
|
+
* `upsert()` (`CREATE TABLE IF NOT EXISTS`, so it is safe to call
|
|
11
|
+
* repeatedly). Schema:
|
|
12
|
+
* id text PRIMARY KEY
|
|
13
|
+
* embedding vector(n) -- n fixed to the dimension of the first upsert
|
|
14
|
+
* metadata jsonb NOT NULL DEFAULT '{}'::jsonb
|
|
15
|
+
*
|
|
16
|
+
* Scoring: matches `InMemoryVectorStore`'s cosine-similarity convention
|
|
17
|
+
* (higher is better, 1.0 = identical direction) by converting pgvector's
|
|
18
|
+
* `<=>` cosine *distance* operator (0 = identical) via `score = 1 - distance`.
|
|
19
|
+
*
|
|
20
|
+
* `MetadataFilter` -> SQL: every value that flows into a query is a bound
|
|
21
|
+
* parameter (`$1`, `$2`, ...), including metadata field *names* — nothing
|
|
22
|
+
* from the filter is ever string-concatenated into SQL text. The one thing
|
|
23
|
+
* that IS textually embedded is the derived table name, and only after it
|
|
24
|
+
* passes a strict identifier allow-list (see `IDENTIFIER_RE`), because
|
|
25
|
+
* Postgres has no way to bind an identifier as a query parameter.
|
|
26
|
+
*
|
|
27
|
+
* Operators not expressible without ambiguity are rejected loudly instead of
|
|
28
|
+
* silently no-op'ing:
|
|
29
|
+
* - `$nor` (top-level logical operator) and `$size` (field-level operator)
|
|
30
|
+
* are declared on `MetadataFilter` but (per the reference
|
|
31
|
+
* `InMemoryVectorStore.matchesFilter`) are not actually implemented
|
|
32
|
+
* there either — rather than guess at semantics that would silently
|
|
33
|
+
* diverge from any caller's expectations, this adapter throws.
|
|
34
|
+
* - Any other unrecognized `$foo` key, top-level or field-level, throws
|
|
35
|
+
* for the same reason.
|
|
36
|
+
*
|
|
37
|
+
* Known, documented behavioral differences from `InMemoryVectorStore`:
|
|
38
|
+
* - `$eq` / bare-value equality compares via Postgres jsonb structural
|
|
39
|
+
* equality, whereas the in-memory reference uses JS `!==` (so it can
|
|
40
|
+
* never match on array/object metadata values — two distinct array
|
|
41
|
+
* instances are never `===`). This adapter is strictly more capable
|
|
42
|
+
* for those value types.
|
|
43
|
+
* - `$regex` runs Postgres POSIX regex (`~`) rather than a JS `RegExp`.
|
|
44
|
+
* For non-string or missing fields the in-memory store tests the
|
|
45
|
+
* pattern against `""` (so e.g. `^$` or `.*` can still match); this
|
|
46
|
+
* adapter instead requires the field to be a JSON string, so those
|
|
47
|
+
* edge-case patterns will not match a missing/non-string field here.
|
|
48
|
+
*/
|
|
49
|
+
import type { MetadataFilter, PgClientLike, PgVectorStoreOptions, VectorQueryResult, VectorStore } from "../../types/index.js";
|
|
50
|
+
export declare class PgVectorStore implements VectorStore {
|
|
51
|
+
private readonly client;
|
|
52
|
+
private readonly tablePrefix;
|
|
53
|
+
/** Tables we've already run `CREATE TABLE IF NOT EXISTS` for, this process. */
|
|
54
|
+
private readonly ensuredTables;
|
|
55
|
+
constructor(client: PgClientLike, options?: PgVectorStoreOptions);
|
|
56
|
+
/**
|
|
57
|
+
* Derive and validate the backing table name for an index. Throws on
|
|
58
|
+
* anything that isn't a safe bare SQL identifier — this is the injection
|
|
59
|
+
* guard for `indexName`.
|
|
60
|
+
*/
|
|
61
|
+
private tableName;
|
|
62
|
+
/**
|
|
63
|
+
* Ensure the `vector` extension and the index's table exist. Safe to call
|
|
64
|
+
* repeatedly (`IF NOT EXISTS` throughout). `dimension` is fixed by the
|
|
65
|
+
* first upsert into a fresh index; it is a validated positive integer
|
|
66
|
+
* embedded directly in DDL (Postgres cannot bind type modifiers either).
|
|
67
|
+
*/
|
|
68
|
+
private ensureTable;
|
|
69
|
+
/**
|
|
70
|
+
* Upsert vectors into an index, creating its table on first use.
|
|
71
|
+
* Mirrors `InMemoryVectorStore.upsert` (metadata defaults to `{}`).
|
|
72
|
+
*/
|
|
73
|
+
upsert(indexName: string, items: Array<{
|
|
74
|
+
id: string;
|
|
75
|
+
vector: number[];
|
|
76
|
+
metadata?: Record<string, unknown>;
|
|
77
|
+
}>): Promise<void>;
|
|
78
|
+
/**
|
|
79
|
+
* Query by cosine similarity. Returns `[]` if the index's table doesn't
|
|
80
|
+
* exist yet (mirrors `InMemoryVectorStore.query` on an unknown index)
|
|
81
|
+
* rather than throwing.
|
|
82
|
+
*/
|
|
83
|
+
query(params: {
|
|
84
|
+
indexName: string;
|
|
85
|
+
queryVector: number[];
|
|
86
|
+
topK?: number;
|
|
87
|
+
filter?: MetadataFilter;
|
|
88
|
+
includeVectors?: boolean;
|
|
89
|
+
}): Promise<VectorQueryResult[]>;
|
|
90
|
+
/**
|
|
91
|
+
* Delete vectors by id. No-op if the index's table doesn't exist yet
|
|
92
|
+
* (mirrors `InMemoryVectorStore.delete`).
|
|
93
|
+
*/
|
|
94
|
+
delete(indexName: string, ids: string[]): Promise<void>;
|
|
95
|
+
}
|
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* pgvector Vector Store Adapter
|
|
3
|
+
*
|
|
4
|
+
* Client-injection `VectorStore` implementation backed by Postgres +
|
|
5
|
+
* the `pgvector` extension. Callers construct and own the Postgres client
|
|
6
|
+
* (e.g. a `pg.Pool`, or an in-process `@electric-sql/pglite` instance) and
|
|
7
|
+
* pass it in — this module adds zero vendor SDKs as runtime dependencies.
|
|
8
|
+
*
|
|
9
|
+
* Storage model: one table per `indexName`, created lazily on first
|
|
10
|
+
* `upsert()` (`CREATE TABLE IF NOT EXISTS`, so it is safe to call
|
|
11
|
+
* repeatedly). Schema:
|
|
12
|
+
* id text PRIMARY KEY
|
|
13
|
+
* embedding vector(n) -- n fixed to the dimension of the first upsert
|
|
14
|
+
* metadata jsonb NOT NULL DEFAULT '{}'::jsonb
|
|
15
|
+
*
|
|
16
|
+
* Scoring: matches `InMemoryVectorStore`'s cosine-similarity convention
|
|
17
|
+
* (higher is better, 1.0 = identical direction) by converting pgvector's
|
|
18
|
+
* `<=>` cosine *distance* operator (0 = identical) via `score = 1 - distance`.
|
|
19
|
+
*
|
|
20
|
+
* `MetadataFilter` -> SQL: every value that flows into a query is a bound
|
|
21
|
+
* parameter (`$1`, `$2`, ...), including metadata field *names* — nothing
|
|
22
|
+
* from the filter is ever string-concatenated into SQL text. The one thing
|
|
23
|
+
* that IS textually embedded is the derived table name, and only after it
|
|
24
|
+
* passes a strict identifier allow-list (see `IDENTIFIER_RE`), because
|
|
25
|
+
* Postgres has no way to bind an identifier as a query parameter.
|
|
26
|
+
*
|
|
27
|
+
* Operators not expressible without ambiguity are rejected loudly instead of
|
|
28
|
+
* silently no-op'ing:
|
|
29
|
+
* - `$nor` (top-level logical operator) and `$size` (field-level operator)
|
|
30
|
+
* are declared on `MetadataFilter` but (per the reference
|
|
31
|
+
* `InMemoryVectorStore.matchesFilter`) are not actually implemented
|
|
32
|
+
* there either — rather than guess at semantics that would silently
|
|
33
|
+
* diverge from any caller's expectations, this adapter throws.
|
|
34
|
+
* - Any other unrecognized `$foo` key, top-level or field-level, throws
|
|
35
|
+
* for the same reason.
|
|
36
|
+
*
|
|
37
|
+
* Known, documented behavioral differences from `InMemoryVectorStore`:
|
|
38
|
+
* - `$eq` / bare-value equality compares via Postgres jsonb structural
|
|
39
|
+
* equality, whereas the in-memory reference uses JS `!==` (so it can
|
|
40
|
+
* never match on array/object metadata values — two distinct array
|
|
41
|
+
* instances are never `===`). This adapter is strictly more capable
|
|
42
|
+
* for those value types.
|
|
43
|
+
* - `$regex` runs Postgres POSIX regex (`~`) rather than a JS `RegExp`.
|
|
44
|
+
* For non-string or missing fields the in-memory store tests the
|
|
45
|
+
* pattern against `""` (so e.g. `^$` or `.*` can still match); this
|
|
46
|
+
* adapter instead requires the field to be a JSON string, so those
|
|
47
|
+
* edge-case patterns will not match a missing/non-string field here.
|
|
48
|
+
*/
|
|
49
|
+
// ============================================================================
|
|
50
|
+
// Identifier safety
|
|
51
|
+
// ============================================================================
|
|
52
|
+
/**
|
|
53
|
+
* Strict allow-list for anything that becomes part of a SQL identifier
|
|
54
|
+
* (table name). Letters/digits/underscore only, must not start with a
|
|
55
|
+
* digit — this is what stands between an attacker-controlled `indexName`
|
|
56
|
+
* and SQL injection, since Postgres cannot bind identifiers as parameters.
|
|
57
|
+
*/
|
|
58
|
+
const IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]{0,62}$/;
|
|
59
|
+
const DEFAULT_TABLE_PREFIX = "neurolink_vs_";
|
|
60
|
+
const MAX_IDENTIFIER_LENGTH = 63; // Postgres NAMEDATALEN limit
|
|
61
|
+
// ============================================================================
|
|
62
|
+
// PgVectorStore
|
|
63
|
+
// ============================================================================
|
|
64
|
+
export class PgVectorStore {
|
|
65
|
+
client;
|
|
66
|
+
tablePrefix;
|
|
67
|
+
/** Tables we've already run `CREATE TABLE IF NOT EXISTS` for, this process. */
|
|
68
|
+
ensuredTables = new Set();
|
|
69
|
+
constructor(client, options = {}) {
|
|
70
|
+
this.client = client;
|
|
71
|
+
this.tablePrefix = options.tablePrefix ?? DEFAULT_TABLE_PREFIX;
|
|
72
|
+
if (!IDENTIFIER_RE.test(this.tablePrefix)) {
|
|
73
|
+
throw new Error(`PgVectorStore: invalid tablePrefix "${this.tablePrefix}" — must match ${IDENTIFIER_RE}`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Derive and validate the backing table name for an index. Throws on
|
|
78
|
+
* anything that isn't a safe bare SQL identifier — this is the injection
|
|
79
|
+
* guard for `indexName`.
|
|
80
|
+
*/
|
|
81
|
+
tableName(indexName) {
|
|
82
|
+
if (!IDENTIFIER_RE.test(indexName)) {
|
|
83
|
+
throw new Error(`PgVectorStore: invalid indexName "${indexName}" — must match ${IDENTIFIER_RE} ` +
|
|
84
|
+
`(letters, digits, underscore; must not start with a digit) to be used as a SQL identifier`);
|
|
85
|
+
}
|
|
86
|
+
const table = `${this.tablePrefix}${indexName}`;
|
|
87
|
+
if (table.length > MAX_IDENTIFIER_LENGTH) {
|
|
88
|
+
throw new Error(`PgVectorStore: derived table name "${table}" exceeds Postgres's ${MAX_IDENTIFIER_LENGTH}-character identifier limit`);
|
|
89
|
+
}
|
|
90
|
+
return table;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Ensure the `vector` extension and the index's table exist. Safe to call
|
|
94
|
+
* repeatedly (`IF NOT EXISTS` throughout). `dimension` is fixed by the
|
|
95
|
+
* first upsert into a fresh index; it is a validated positive integer
|
|
96
|
+
* embedded directly in DDL (Postgres cannot bind type modifiers either).
|
|
97
|
+
*/
|
|
98
|
+
async ensureTable(indexName, dimension) {
|
|
99
|
+
const table = this.tableName(indexName);
|
|
100
|
+
if (this.ensuredTables.has(table)) {
|
|
101
|
+
return table;
|
|
102
|
+
}
|
|
103
|
+
if (!Number.isInteger(dimension) || dimension <= 0) {
|
|
104
|
+
throw new Error(`PgVectorStore: invalid embedding dimension ${dimension} — must be a positive integer`);
|
|
105
|
+
}
|
|
106
|
+
await this.client.query(`CREATE EXTENSION IF NOT EXISTS vector`);
|
|
107
|
+
await this.client.query(`CREATE TABLE IF NOT EXISTS "${table}" (
|
|
108
|
+
id text PRIMARY KEY,
|
|
109
|
+
embedding vector(${dimension}) NOT NULL,
|
|
110
|
+
metadata jsonb NOT NULL DEFAULT '{}'::jsonb
|
|
111
|
+
)`);
|
|
112
|
+
this.ensuredTables.add(table);
|
|
113
|
+
return table;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Upsert vectors into an index, creating its table on first use.
|
|
117
|
+
* Mirrors `InMemoryVectorStore.upsert` (metadata defaults to `{}`).
|
|
118
|
+
*/
|
|
119
|
+
async upsert(indexName, items) {
|
|
120
|
+
if (items.length === 0) {
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
const dimension = items[0].vector.length;
|
|
124
|
+
for (const item of items) {
|
|
125
|
+
if (item.vector.length !== dimension) {
|
|
126
|
+
throw new Error(`PgVectorStore.upsert: all vectors in a single upsert batch must share the same ` +
|
|
127
|
+
`dimension (expected ${dimension}, got ${item.vector.length} for id "${item.id}")`);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
const table = await this.ensureTable(indexName, dimension);
|
|
131
|
+
const values = [];
|
|
132
|
+
const rows = items.map((item) => {
|
|
133
|
+
const idParam = pushParam(values, item.id);
|
|
134
|
+
const vectorParam = pushParam(values, toVectorLiteral(item.vector));
|
|
135
|
+
const metadataParam = pushParam(values, JSON.stringify(item.metadata ?? {}));
|
|
136
|
+
return `(${idParam}, ${vectorParam}::vector, ${metadataParam}::jsonb)`;
|
|
137
|
+
});
|
|
138
|
+
await this.client.query(`INSERT INTO "${table}" (id, embedding, metadata)
|
|
139
|
+
VALUES ${rows.join(", ")}
|
|
140
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
141
|
+
embedding = EXCLUDED.embedding,
|
|
142
|
+
metadata = EXCLUDED.metadata`, values);
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Query by cosine similarity. Returns `[]` if the index's table doesn't
|
|
146
|
+
* exist yet (mirrors `InMemoryVectorStore.query` on an unknown index)
|
|
147
|
+
* rather than throwing.
|
|
148
|
+
*/
|
|
149
|
+
async query(params) {
|
|
150
|
+
const { indexName, queryVector, topK = 10, filter, includeVectors = false, } = params;
|
|
151
|
+
const table = this.tableName(indexName);
|
|
152
|
+
const values = [toVectorLiteral(queryVector)];
|
|
153
|
+
const whereSql = filter ? compileFilter(filter, values) : undefined;
|
|
154
|
+
const limitParam = pushParam(values, topK);
|
|
155
|
+
const vectorSelect = includeVectors ? ", embedding::text AS embedding" : "";
|
|
156
|
+
const sql = `
|
|
157
|
+
SELECT id, metadata, (1 - (embedding <=> $1::vector)) AS score${vectorSelect}
|
|
158
|
+
FROM "${table}"
|
|
159
|
+
${whereSql ? `WHERE ${whereSql}` : ""}
|
|
160
|
+
ORDER BY embedding <=> $1::vector ASC
|
|
161
|
+
LIMIT ${limitParam}
|
|
162
|
+
`;
|
|
163
|
+
let rows;
|
|
164
|
+
try {
|
|
165
|
+
const result = await this.client.query(sql, values);
|
|
166
|
+
rows = result.rows;
|
|
167
|
+
}
|
|
168
|
+
catch (err) {
|
|
169
|
+
if (isMissingTableError(err)) {
|
|
170
|
+
return [];
|
|
171
|
+
}
|
|
172
|
+
throw err;
|
|
173
|
+
}
|
|
174
|
+
return rows.map((row) => {
|
|
175
|
+
const metadata = toMetadataObject(row.metadata);
|
|
176
|
+
return {
|
|
177
|
+
id: row.id,
|
|
178
|
+
score: Number(row.score),
|
|
179
|
+
text: metadata.text,
|
|
180
|
+
metadata,
|
|
181
|
+
...(includeVectors && typeof row.embedding === "string"
|
|
182
|
+
? { vector: parseVectorLiteral(row.embedding) }
|
|
183
|
+
: {}),
|
|
184
|
+
};
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Delete vectors by id. No-op if the index's table doesn't exist yet
|
|
189
|
+
* (mirrors `InMemoryVectorStore.delete`).
|
|
190
|
+
*/
|
|
191
|
+
async delete(indexName, ids) {
|
|
192
|
+
if (ids.length === 0) {
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
const table = this.tableName(indexName);
|
|
196
|
+
const values = [];
|
|
197
|
+
const placeholders = ids.map((id) => pushParam(values, id));
|
|
198
|
+
try {
|
|
199
|
+
await this.client.query(`DELETE FROM "${table}" WHERE id IN (${placeholders.join(", ")})`, values);
|
|
200
|
+
}
|
|
201
|
+
catch (err) {
|
|
202
|
+
if (isMissingTableError(err)) {
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
throw err;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
// ============================================================================
|
|
210
|
+
// Row / vector (de)serialization helpers
|
|
211
|
+
// ============================================================================
|
|
212
|
+
/** Postgres numeric/jsonb drivers vary in whether values arrive pre-parsed. */
|
|
213
|
+
function toMetadataObject(value) {
|
|
214
|
+
if (value && typeof value === "object") {
|
|
215
|
+
return value;
|
|
216
|
+
}
|
|
217
|
+
if (typeof value === "string") {
|
|
218
|
+
try {
|
|
219
|
+
return JSON.parse(value);
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
return {};
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return {};
|
|
226
|
+
}
|
|
227
|
+
function toVectorLiteral(vector) {
|
|
228
|
+
return `[${vector.join(",")}]`;
|
|
229
|
+
}
|
|
230
|
+
/** Parse pgvector's `[1,2,3]` text representation back into numbers. */
|
|
231
|
+
function parseVectorLiteral(text) {
|
|
232
|
+
const trimmed = text.trim().replace(/^\[/, "").replace(/\]$/, "");
|
|
233
|
+
if (trimmed.length === 0) {
|
|
234
|
+
return [];
|
|
235
|
+
}
|
|
236
|
+
return trimmed.split(",").map((v) => Number(v));
|
|
237
|
+
}
|
|
238
|
+
function isMissingTableError(err) {
|
|
239
|
+
if (!err || typeof err !== "object") {
|
|
240
|
+
return false;
|
|
241
|
+
}
|
|
242
|
+
const code = err.code;
|
|
243
|
+
if (code === "42P01") {
|
|
244
|
+
// Postgres SQLSTATE for undefined_table
|
|
245
|
+
return true;
|
|
246
|
+
}
|
|
247
|
+
const message = err.message;
|
|
248
|
+
return typeof message === "string" && /does not exist/i.test(message);
|
|
249
|
+
}
|
|
250
|
+
// ============================================================================
|
|
251
|
+
// Parameter binding helpers
|
|
252
|
+
// ============================================================================
|
|
253
|
+
function pushParam(values, value) {
|
|
254
|
+
values.push(value);
|
|
255
|
+
return `$${values.length}`;
|
|
256
|
+
}
|
|
257
|
+
// ============================================================================
|
|
258
|
+
// MetadataFilter -> SQL translation
|
|
259
|
+
// ============================================================================
|
|
260
|
+
/** Field-level operators this adapter translates to SQL. */
|
|
261
|
+
const KNOWN_FIELD_OPS = new Set([
|
|
262
|
+
"$eq",
|
|
263
|
+
"$ne",
|
|
264
|
+
"$gt",
|
|
265
|
+
"$gte",
|
|
266
|
+
"$lt",
|
|
267
|
+
"$lte",
|
|
268
|
+
"$in",
|
|
269
|
+
"$nin",
|
|
270
|
+
"$exists",
|
|
271
|
+
"$contains",
|
|
272
|
+
"$regex",
|
|
273
|
+
]);
|
|
274
|
+
/** Logical/field entries are ANDed together, matching the reference semantics. */
|
|
275
|
+
function compileFilter(filter, values) {
|
|
276
|
+
const clauses = [];
|
|
277
|
+
for (const [key, value] of Object.entries(filter)) {
|
|
278
|
+
if (key.startsWith("$")) {
|
|
279
|
+
clauses.push(compileLogicalOperator(key, value, values));
|
|
280
|
+
}
|
|
281
|
+
else {
|
|
282
|
+
clauses.push(compileFieldFilter(key, value, values));
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
return clauses.length > 0 ? clauses.join(" AND ") : "TRUE";
|
|
286
|
+
}
|
|
287
|
+
function compileLogicalOperator(key, value, values) {
|
|
288
|
+
switch (key) {
|
|
289
|
+
case "$and": {
|
|
290
|
+
const subFilters = value;
|
|
291
|
+
if (subFilters.length === 0) {
|
|
292
|
+
return "TRUE";
|
|
293
|
+
}
|
|
294
|
+
return subFilters
|
|
295
|
+
.map((f) => `(${compileFilter(f, values)})`)
|
|
296
|
+
.join(" AND ");
|
|
297
|
+
}
|
|
298
|
+
case "$or": {
|
|
299
|
+
const subFilters = value;
|
|
300
|
+
if (subFilters.length === 0) {
|
|
301
|
+
return "FALSE";
|
|
302
|
+
}
|
|
303
|
+
return `(${subFilters.map((f) => `(${compileFilter(f, values)})`).join(" OR ")})`;
|
|
304
|
+
}
|
|
305
|
+
case "$not":
|
|
306
|
+
return `NOT (${compileFilter(value, values)})`;
|
|
307
|
+
case "$nor":
|
|
308
|
+
case "$size":
|
|
309
|
+
throw new Error(`PgVectorStore: MetadataFilter operator "${key}" is not supported by the pgvector ` +
|
|
310
|
+
`adapter (it is declared on the MetadataFilter type but is also unimplemented in ` +
|
|
311
|
+
`the InMemoryVectorStore reference, so there is no agreed-upon semantics to mirror)`);
|
|
312
|
+
default:
|
|
313
|
+
throw new Error(`PgVectorStore: unrecognized MetadataFilter logical operator "${key}"`);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
function compileFieldFilter(field, value, values) {
|
|
317
|
+
const fieldJson = () => `(metadata -> ${pushParam(values, field)})`;
|
|
318
|
+
const fieldText = () => `(metadata ->> ${pushParam(values, field)})`;
|
|
319
|
+
const fieldExists = () => `(metadata ? ${pushParam(values, field)})`;
|
|
320
|
+
const fieldIsNumber = () => `(jsonb_typeof(metadata -> ${pushParam(values, field)}) = 'number')`;
|
|
321
|
+
const fieldIsString = () => `(jsonb_typeof(metadata -> ${pushParam(values, field)}) = 'string')`;
|
|
322
|
+
if (value !== null && typeof value === "object" && !Array.isArray(value)) {
|
|
323
|
+
const ops = value;
|
|
324
|
+
// Fail loudly on any operator we don't implement (e.g. `$size`, which is
|
|
325
|
+
// declared on MetadataFilter but — like `$nor` — has no agreed-upon
|
|
326
|
+
// semantics because InMemoryVectorStore's matchesFilter doesn't
|
|
327
|
+
// implement it either) instead of silently treating it as a no-op.
|
|
328
|
+
for (const opKey of Object.keys(ops)) {
|
|
329
|
+
if (opKey.startsWith("$") && !KNOWN_FIELD_OPS.has(opKey)) {
|
|
330
|
+
throw new Error(`PgVectorStore: MetadataFilter operator "${opKey}" on field "${field}" is not ` +
|
|
331
|
+
`supported by the pgvector adapter`);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
const clauses = [];
|
|
335
|
+
if ("$eq" in ops) {
|
|
336
|
+
clauses.push(`${fieldJson()} = ${pushParam(values, JSON.stringify(ops.$eq))}::jsonb`);
|
|
337
|
+
}
|
|
338
|
+
if ("$ne" in ops) {
|
|
339
|
+
clauses.push(`${fieldJson()} IS DISTINCT FROM ${pushParam(values, JSON.stringify(ops.$ne))}::jsonb`);
|
|
340
|
+
}
|
|
341
|
+
if ("$gt" in ops) {
|
|
342
|
+
clauses.push(`(${fieldIsNumber()} AND ${fieldText()}::numeric > ${pushParam(values, ops.$gt)})`);
|
|
343
|
+
}
|
|
344
|
+
if ("$gte" in ops) {
|
|
345
|
+
clauses.push(`(${fieldIsNumber()} AND ${fieldText()}::numeric >= ${pushParam(values, ops.$gte)})`);
|
|
346
|
+
}
|
|
347
|
+
if ("$lt" in ops) {
|
|
348
|
+
clauses.push(`(${fieldIsNumber()} AND ${fieldText()}::numeric < ${pushParam(values, ops.$lt)})`);
|
|
349
|
+
}
|
|
350
|
+
if ("$lte" in ops) {
|
|
351
|
+
clauses.push(`(${fieldIsNumber()} AND ${fieldText()}::numeric <= ${pushParam(values, ops.$lte)})`);
|
|
352
|
+
}
|
|
353
|
+
if ("$in" in ops) {
|
|
354
|
+
const list = ops.$in;
|
|
355
|
+
clauses.push(list.length === 0
|
|
356
|
+
? "FALSE"
|
|
357
|
+
: `(${fieldExists()} AND ${fieldJson()} = ANY(ARRAY[${list
|
|
358
|
+
.map((v) => `${pushParam(values, JSON.stringify(v))}::jsonb`)
|
|
359
|
+
.join(", ")}]))`);
|
|
360
|
+
}
|
|
361
|
+
if ("$nin" in ops) {
|
|
362
|
+
const list = ops.$nin;
|
|
363
|
+
clauses.push(list.length === 0
|
|
364
|
+
? "TRUE"
|
|
365
|
+
: `NOT (${fieldExists()} AND ${fieldJson()} = ANY(ARRAY[${list
|
|
366
|
+
.map((v) => `${pushParam(values, JSON.stringify(v))}::jsonb`)
|
|
367
|
+
.join(", ")}]))`);
|
|
368
|
+
}
|
|
369
|
+
if ("$exists" in ops) {
|
|
370
|
+
clauses.push(ops.$exists ? fieldExists() : `NOT ${fieldExists()}`);
|
|
371
|
+
}
|
|
372
|
+
if ("$contains" in ops) {
|
|
373
|
+
clauses.push(`(${fieldIsString()} AND position(${pushParam(values, ops.$contains)} in ${fieldText()}) > 0)`);
|
|
374
|
+
}
|
|
375
|
+
if ("$regex" in ops) {
|
|
376
|
+
const pattern = ops.$regex;
|
|
377
|
+
// Guard mirrors the reference's ReDoS guard: over-length or invalid
|
|
378
|
+
// patterns never match, they don't error the whole query.
|
|
379
|
+
if (pattern.length > 200 || !isValidRegex(pattern)) {
|
|
380
|
+
clauses.push("FALSE");
|
|
381
|
+
}
|
|
382
|
+
else {
|
|
383
|
+
clauses.push(`(${fieldIsString()} AND ${fieldText()} ~ ${pushParam(values, pattern)})`);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
return clauses.length > 0 ? clauses.join(" AND ") : "TRUE";
|
|
387
|
+
}
|
|
388
|
+
// Direct equality (primitive, array, or null filter value).
|
|
389
|
+
return `${fieldJson()} = ${pushParam(values, JSON.stringify(value))}::jsonb`;
|
|
390
|
+
}
|
|
391
|
+
function isValidRegex(pattern) {
|
|
392
|
+
try {
|
|
393
|
+
RegExp(pattern);
|
|
394
|
+
return true;
|
|
395
|
+
}
|
|
396
|
+
catch {
|
|
397
|
+
return false;
|
|
398
|
+
}
|
|
399
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pinecone Vector Store Adapter
|
|
3
|
+
*
|
|
4
|
+
* Implements the `VectorStore` contract (see `src/lib/types/rag.ts`) against
|
|
5
|
+
* a Pinecone index, using client injection: callers construct their own
|
|
6
|
+
* `@pinecone-database/pinecone` client/index and pass it in. No Pinecone SDK
|
|
7
|
+
* is a runtime dependency of this package — `PineconeIndexLike` below is a
|
|
8
|
+
* minimal structural interface satisfied by the real `Index` object (and by
|
|
9
|
+
* hand-written mocks in tests).
|
|
10
|
+
*
|
|
11
|
+
* ## indexName -> namespace mapping
|
|
12
|
+
*
|
|
13
|
+
* The `VectorStore` contract's `indexName` parameter is mapped onto a
|
|
14
|
+
* Pinecone **namespace**, not a Pinecone index. Pinecone ties one client
|
|
15
|
+
* `Index` object to a single physical index (created ahead of time via
|
|
16
|
+
* Pinecone's control-plane API); namespaces are Pinecone's mechanism for
|
|
17
|
+
* partitioning data *within* that one index, and are the closest match to
|
|
18
|
+
* this SDK's per-call `indexName` concept.
|
|
19
|
+
*
|
|
20
|
+
* - If the injected client exposes `.namespace(name)` (as the real
|
|
21
|
+
* `@pinecone-database/pinecone` `Index.namespace()` does), every
|
|
22
|
+
* query/upsert/delete call routes through `client.namespace(indexName)`.
|
|
23
|
+
* - If the injected client does not expose `.namespace` (e.g. a caller
|
|
24
|
+
* already pre-scoped their client to a fixed namespace, or a minimal
|
|
25
|
+
* mock), calls go straight to the injected client and `indexName` has
|
|
26
|
+
* no further effect. Document this for your callers if you construct a
|
|
27
|
+
* `PineconeVectorStore` from a namespace-fixed client.
|
|
28
|
+
*
|
|
29
|
+
* ## Metadata filter translation
|
|
30
|
+
*
|
|
31
|
+
* `MetadataFilter` (Mongo/Sift-style) is translated to Pinecone's native
|
|
32
|
+
* filter DSL, which already shares the same `$op` shape for comparison
|
|
33
|
+
* operators. Supported operators: `$eq`, `$ne`, `$gt`, `$gte`, `$lt`,
|
|
34
|
+
* `$lte`, `$in`, `$nin` (field-level) and `$and` / `$or` (logical,
|
|
35
|
+
* recursively translated). Any other operator — `$not`, `$nor`, `$exists`,
|
|
36
|
+
* `$contains`, `$regex`, `$size` — has no Pinecone equivalent and throws a
|
|
37
|
+
* clear `Error` naming the offending operator/field rather than silently
|
|
38
|
+
* mis-filtering (Pinecone's filter DSL has no negation, existence, string
|
|
39
|
+
* containment, regex, or array-length operators).
|
|
40
|
+
*/
|
|
41
|
+
import type { MetadataFilter, PineconeIndexLike, VectorQueryResult, VectorStore } from "../../types/index.js";
|
|
42
|
+
/**
|
|
43
|
+
* Translate a `MetadataFilter` into Pinecone's native filter object.
|
|
44
|
+
*
|
|
45
|
+
* Throws a descriptive `Error` for any operator Pinecone cannot express
|
|
46
|
+
* ($not, $nor, $exists, $contains, $regex, $size) instead of silently
|
|
47
|
+
* dropping or mis-applying it.
|
|
48
|
+
*/
|
|
49
|
+
export declare function translatePineconeFilter(filter: MetadataFilter): Record<string, unknown>;
|
|
50
|
+
/**
|
|
51
|
+
* Pinecone-backed implementation of the `VectorStore` contract.
|
|
52
|
+
*
|
|
53
|
+
* @example
|
|
54
|
+
* ```typescript
|
|
55
|
+
* import { Pinecone } from '@pinecone-database/pinecone';
|
|
56
|
+
* import { PineconeVectorStore } from '@juspay/neurolink/rag';
|
|
57
|
+
*
|
|
58
|
+
* const client = new Pinecone({ apiKey: process.env.PINECONE_API_KEY! });
|
|
59
|
+
* const store = new PineconeVectorStore(client.index('my-index'));
|
|
60
|
+
*
|
|
61
|
+
* await store.upsert('tenant-a', [{ id: '1', vector: [...], metadata: { text: 'hello' } }]);
|
|
62
|
+
* const results = await store.query({ indexName: 'tenant-a', queryVector: [...], topK: 5 });
|
|
63
|
+
* ```
|
|
64
|
+
*/
|
|
65
|
+
export declare class PineconeVectorStore implements VectorStore {
|
|
66
|
+
private readonly client;
|
|
67
|
+
constructor(client: PineconeIndexLike);
|
|
68
|
+
/** Resolve the Pinecone client scoped to `indexName` (see class docs re: namespace mapping). */
|
|
69
|
+
private resolveClient;
|
|
70
|
+
query(params: {
|
|
71
|
+
indexName: string;
|
|
72
|
+
queryVector: number[];
|
|
73
|
+
topK?: number;
|
|
74
|
+
filter?: MetadataFilter;
|
|
75
|
+
includeVectors?: boolean;
|
|
76
|
+
}): Promise<VectorQueryResult[]>;
|
|
77
|
+
/** Add or update vectors in the namespace mapped from `indexName`. */
|
|
78
|
+
upsert(indexName: string, items: Array<{
|
|
79
|
+
id: string;
|
|
80
|
+
vector: number[];
|
|
81
|
+
metadata?: Record<string, unknown>;
|
|
82
|
+
}>): Promise<void>;
|
|
83
|
+
/** Delete vectors by id from the namespace mapped from `indexName`. */
|
|
84
|
+
delete(indexName: string, ids: string[]): Promise<void>;
|
|
85
|
+
}
|