@devilsdev/rag-connectors 3.5.0 → 3.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.cjs +1360 -188
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +7 -5
package/dist/index.cjs
CHANGED
|
@@ -99,7 +99,7 @@ var require_openai_connector = __commonJS({
|
|
|
99
99
|
"src/connectors/openai-connector.js"(exports2, module2) {
|
|
100
100
|
"use strict";
|
|
101
101
|
var { BaseConnector } = require_base_connector();
|
|
102
|
-
var {
|
|
102
|
+
var { missingPeer } = require_optional_peer();
|
|
103
103
|
var {
|
|
104
104
|
missingApiKeyError,
|
|
105
105
|
raceAbort,
|
|
@@ -121,12 +121,15 @@ var require_openai_connector = __commonJS({
|
|
|
121
121
|
}
|
|
122
122
|
};
|
|
123
123
|
function loadOpenAI() {
|
|
124
|
-
|
|
125
|
-
"openai"
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
124
|
+
try {
|
|
125
|
+
return require("openai");
|
|
126
|
+
} catch (error) {
|
|
127
|
+
throw missingPeer(
|
|
128
|
+
error,
|
|
129
|
+
"openai",
|
|
130
|
+
'OpenAIConnector needs the "openai" package, an optional peer dependency since 3.0.0: npm install openai'
|
|
131
|
+
);
|
|
132
|
+
}
|
|
130
133
|
}
|
|
131
134
|
var OpenAIConnector = class extends BaseConnector {
|
|
132
135
|
/**
|
|
@@ -230,8 +233,25 @@ var require_openai_connector = __commonJS({
|
|
|
230
233
|
_request(signal, fn) {
|
|
231
234
|
return this.withRetry(fn, this.config.maxRetries, this.config.retryDelay, {
|
|
232
235
|
signal
|
|
236
|
+
}).catch((err) => {
|
|
237
|
+
throw this._tagError(err);
|
|
233
238
|
});
|
|
234
239
|
}
|
|
240
|
+
/**
|
|
241
|
+
* Name the provider on an HTTP error from the SDK, as the axios-based
|
|
242
|
+
* connectors do, so a caller such as the CLI can say which key was refused.
|
|
243
|
+
* A custom baseURL never reads OPENAI_API_KEY, so it gets no envVar.
|
|
244
|
+
* @private
|
|
245
|
+
*/
|
|
246
|
+
_tagError(err) {
|
|
247
|
+
if (err && typeof err === "object" && typeof err.status === "number") {
|
|
248
|
+
if (!err.provider) {
|
|
249
|
+
err.provider = this.config.baseURL ? "The OpenAI-compatible API" : "OpenAI";
|
|
250
|
+
}
|
|
251
|
+
if (!this.config.baseURL && !err.envVar) err.envVar = API_KEY_ENV;
|
|
252
|
+
}
|
|
253
|
+
return err;
|
|
254
|
+
}
|
|
235
255
|
/**
|
|
236
256
|
* Generate a chat completion using the provided query and optional RAG context.
|
|
237
257
|
*
|
|
@@ -275,19 +295,24 @@ var require_openai_connector = __commonJS({
|
|
|
275
295
|
this._requireApiKey();
|
|
276
296
|
const client = this._getClient();
|
|
277
297
|
const messages = this._buildMessages(query, context);
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
298
|
+
let stream;
|
|
299
|
+
try {
|
|
300
|
+
stream = await raceAbort(
|
|
301
|
+
client.chat.completions.create(
|
|
302
|
+
{
|
|
303
|
+
model: options.model || this.config.model,
|
|
304
|
+
messages,
|
|
305
|
+
temperature: options.temperature ?? this.config.temperature,
|
|
306
|
+
max_tokens: options.maxTokens ?? this.config.maxTokens,
|
|
307
|
+
stream: true
|
|
308
|
+
},
|
|
309
|
+
signal ? { signal } : void 0
|
|
310
|
+
),
|
|
311
|
+
signal
|
|
312
|
+
);
|
|
313
|
+
} catch (err) {
|
|
314
|
+
throw this._tagError(err);
|
|
315
|
+
}
|
|
291
316
|
for await (const chunk of stream) {
|
|
292
317
|
throwIfAborted(signal);
|
|
293
318
|
const token = chunk.choices[0]?.delta?.content || "";
|
|
@@ -1608,220 +1633,914 @@ var require_qdrant_retriever = __commonJS({
|
|
|
1608
1633
|
}
|
|
1609
1634
|
});
|
|
1610
1635
|
|
|
1611
|
-
// src/retrieval/
|
|
1612
|
-
var
|
|
1613
|
-
"src/retrieval/
|
|
1636
|
+
// src/retrieval/pinecone-retriever.js
|
|
1637
|
+
var require_pinecone_retriever = __commonJS({
|
|
1638
|
+
"src/retrieval/pinecone-retriever.js"(exports2, module2) {
|
|
1614
1639
|
"use strict";
|
|
1615
|
-
var
|
|
1616
|
-
var
|
|
1617
|
-
var
|
|
1618
|
-
var
|
|
1619
|
-
var
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1640
|
+
var crypto = require("crypto");
|
|
1641
|
+
var { throwIfAborted } = require_connector_utils();
|
|
1642
|
+
var API_VERSION = "2026-07";
|
|
1643
|
+
var CONTROL_PLANE = "https://api.pinecone.io";
|
|
1644
|
+
var DEFAULT_NAMESPACE = "rag_documents";
|
|
1645
|
+
var UPSERT_BATCH = 100;
|
|
1646
|
+
var EMBED_UPSERT_BATCH = 96;
|
|
1647
|
+
var REQUEST_BYTES = 2e6 - 1024;
|
|
1648
|
+
var DELETE_BATCH = 1e3;
|
|
1649
|
+
var PAGE = 100;
|
|
1650
|
+
var SEARCH_MAX = 1e4;
|
|
1651
|
+
var PLAIN_ID = /^[\x20-\x7e]{1,512}$/;
|
|
1652
|
+
var FIELDS = ["rag_id", "rag_content", "rag_metadata"];
|
|
1653
|
+
var OURS = { rag_id: { $exists: true } };
|
|
1654
|
+
var PROBE_QUERY = "document";
|
|
1655
|
+
function recordId(id) {
|
|
1656
|
+
const s = String(id);
|
|
1657
|
+
if (PLAIN_ID.test(s)) return s;
|
|
1658
|
+
return `rag-${crypto.createHash("sha256").update(s).digest("hex")}`;
|
|
1628
1659
|
}
|
|
1629
|
-
function
|
|
1660
|
+
function checkVector(vector, id) {
|
|
1630
1661
|
if (!Array.isArray(vector) || vector.length === 0) {
|
|
1631
|
-
throw new Error(`
|
|
1632
|
-
}
|
|
1633
|
-
for (const x of vector) {
|
|
1634
|
-
if (typeof x !== "number" || !Number.isFinite(x)) {
|
|
1635
|
-
throw new Error(
|
|
1636
|
-
`PgVectorRetriever: document "${id}" has a vector with a non-finite value`
|
|
1637
|
-
);
|
|
1638
|
-
}
|
|
1639
|
-
}
|
|
1640
|
-
return `[${vector.join(",")}]`;
|
|
1641
|
-
}
|
|
1642
|
-
function explain(err, table) {
|
|
1643
|
-
const code = err && err.code;
|
|
1644
|
-
if (code === "42P01") {
|
|
1645
|
-
return new Error(
|
|
1646
|
-
`PgVectorRetriever: table ${table} does not exist. Store documents first (rag-pipeline ingest), or create it.`
|
|
1647
|
-
);
|
|
1662
|
+
throw new Error(`PineconeRetriever: document "${id}" has no vector`);
|
|
1648
1663
|
}
|
|
1649
|
-
if (
|
|
1650
|
-
|
|
1651
|
-
`
|
|
1664
|
+
if (!vector.every((x) => typeof x === "number" && Number.isFinite(x))) {
|
|
1665
|
+
throw new Error(
|
|
1666
|
+
`PineconeRetriever: document "${id}" has a vector with a non-finite value`
|
|
1652
1667
|
);
|
|
1653
1668
|
}
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
);
|
|
1669
|
+
}
|
|
1670
|
+
function fromFields(fields, fallbackId) {
|
|
1671
|
+
const m = fields || {};
|
|
1672
|
+
let metadata = {};
|
|
1673
|
+
try {
|
|
1674
|
+
metadata = m.rag_metadata ? JSON.parse(m.rag_metadata) : {};
|
|
1675
|
+
} catch {
|
|
1676
|
+
metadata = {};
|
|
1660
1677
|
}
|
|
1661
|
-
return
|
|
1678
|
+
return {
|
|
1679
|
+
id: m.rag_id !== void 0 ? String(m.rag_id) : fallbackId,
|
|
1680
|
+
content: m.rag_content || "",
|
|
1681
|
+
metadata
|
|
1682
|
+
};
|
|
1662
1683
|
}
|
|
1663
|
-
|
|
1684
|
+
function normalizeHost(host) {
|
|
1685
|
+
let h = String(host);
|
|
1686
|
+
while (h.endsWith("/")) h = h.slice(0, -1);
|
|
1687
|
+
return /^https?:\/\//.test(h) ? h : `https://${h}`;
|
|
1688
|
+
}
|
|
1689
|
+
var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1690
|
+
var PineconeRetriever = class {
|
|
1664
1691
|
/**
|
|
1665
1692
|
* @param {Object} options
|
|
1666
|
-
* @param {
|
|
1667
|
-
* @param {string} [options.
|
|
1668
|
-
* @param {string} [options.
|
|
1669
|
-
* @param {
|
|
1670
|
-
* @param {
|
|
1671
|
-
* @param {
|
|
1672
|
-
* @param {
|
|
1673
|
-
* @param {
|
|
1693
|
+
* @param {string} [options.apiKey] - Defaults to PINECONE_API_KEY
|
|
1694
|
+
* @param {string} [options.index] - The index name, described on first use
|
|
1695
|
+
* @param {string} [options.host] - The index host (from the console), instead of or with `index`
|
|
1696
|
+
* @param {string} [options.namespace='rag_documents'] - A namespace this retriever owns
|
|
1697
|
+
* @param {'auto'|'documents'|'vectors'} [options.api='auto'] - Document index (Documents API) or vector index; auto reads it from the index description
|
|
1698
|
+
* @param {string} [options.vectorField] - The document index's dense_vector field (default: the schema's)
|
|
1699
|
+
* @param {string} [options.textField] - The document index's text field that Pinecone embeds (integrated embedding; default: the schema's, when the index has no dense_vector field)
|
|
1700
|
+
* @param {number} [options.freshnessTimeoutMs=30000] - How long a read waits for this retriever's writes to be indexed; 0 to not wait
|
|
1701
|
+
* @param {string} [options.cacheKey] - Cache identity; change it when the index changes
|
|
1702
|
+
* @param {Function} [options.fetch] - fetch implementation (defaults to the global)
|
|
1674
1703
|
*/
|
|
1675
1704
|
constructor(options = {}) {
|
|
1676
1705
|
const {
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1706
|
+
apiKey = typeof process !== "undefined" && process.env ? process.env.PINECONE_API_KEY : void 0,
|
|
1707
|
+
host,
|
|
1708
|
+
index,
|
|
1709
|
+
namespace = DEFAULT_NAMESPACE,
|
|
1710
|
+
api = "auto",
|
|
1711
|
+
vectorField,
|
|
1712
|
+
textField,
|
|
1713
|
+
freshnessTimeoutMs = 3e4,
|
|
1714
|
+
cacheKey,
|
|
1715
|
+
fetch: fetchImpl = globalThis.fetch
|
|
1685
1716
|
} = options;
|
|
1686
|
-
if (!
|
|
1717
|
+
if (!apiKey) {
|
|
1687
1718
|
throw new Error(
|
|
1688
|
-
"
|
|
1719
|
+
"PineconeRetriever needs an API key: pass apiKey or set PINECONE_API_KEY"
|
|
1689
1720
|
);
|
|
1690
1721
|
}
|
|
1691
|
-
if (
|
|
1692
|
-
throw new Error(
|
|
1722
|
+
if (!host && !index) {
|
|
1723
|
+
throw new Error(
|
|
1724
|
+
"PineconeRetriever needs the index: pass index (its name) or host (the index host)"
|
|
1725
|
+
);
|
|
1693
1726
|
}
|
|
1694
|
-
if (
|
|
1727
|
+
if (typeof namespace !== "string") {
|
|
1728
|
+
throw new Error("PineconeRetriever: namespace must be a string");
|
|
1729
|
+
}
|
|
1730
|
+
if (!["auto", "documents", "vectors"].includes(api)) {
|
|
1695
1731
|
throw new Error(
|
|
1696
|
-
|
|
1732
|
+
'PineconeRetriever: api must be "auto", "documents" or "vectors"'
|
|
1697
1733
|
);
|
|
1698
1734
|
}
|
|
1699
|
-
if (
|
|
1735
|
+
if (vectorField && textField) {
|
|
1700
1736
|
throw new Error(
|
|
1701
|
-
"
|
|
1737
|
+
"PineconeRetriever: pass vectorField (your vectors) or textField (Pinecone embeds the text), not both"
|
|
1702
1738
|
);
|
|
1703
1739
|
}
|
|
1704
|
-
if (
|
|
1740
|
+
if (textField && api === "vectors") {
|
|
1705
1741
|
throw new Error(
|
|
1706
|
-
|
|
1742
|
+
'PineconeRetriever: textField needs a document index (api "documents" or "auto")'
|
|
1707
1743
|
);
|
|
1708
1744
|
}
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
this.dimensions = dimensions;
|
|
1713
|
-
this.createTable = createTable;
|
|
1745
|
+
if (typeof fetchImpl !== "function") {
|
|
1746
|
+
throw new Error("PineconeRetriever needs fetch (Node.js 18 or later)");
|
|
1747
|
+
}
|
|
1714
1748
|
this.index = index;
|
|
1749
|
+
this.namespace = namespace;
|
|
1750
|
+
this.freshnessTimeoutMs = freshnessTimeoutMs;
|
|
1715
1751
|
this.cacheKey = cacheKey;
|
|
1716
|
-
this.
|
|
1717
|
-
this.
|
|
1718
|
-
this.
|
|
1719
|
-
this.
|
|
1752
|
+
this._apiKey = apiKey;
|
|
1753
|
+
this._host = host ? normalizeHost(host) : null;
|
|
1754
|
+
this._api = api;
|
|
1755
|
+
this._vectorField = vectorField;
|
|
1756
|
+
this._textField = textField;
|
|
1757
|
+
this._fetch = fetchImpl === globalThis.fetch ? fetchImpl.bind(globalThis) : fetchImpl;
|
|
1758
|
+
this._resolving = null;
|
|
1759
|
+
this._writeLsn = 0;
|
|
1760
|
+
this._dimension = null;
|
|
1720
1761
|
}
|
|
1721
|
-
/**
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
);
|
|
1731
|
-
}
|
|
1732
|
-
this._pool = new pg.Pool({
|
|
1733
|
-
connectionString: this._connectionString,
|
|
1734
|
-
allowExitOnIdle: true
|
|
1735
|
-
});
|
|
1736
|
-
}
|
|
1737
|
-
return this._pool;
|
|
1762
|
+
/**
|
|
1763
|
+
* Cache identity for createRagPipeline({ cache }): none unless you set
|
|
1764
|
+
* `cacheKey`, since other clients can change the index.
|
|
1765
|
+
* @param {'retriever'} role
|
|
1766
|
+
* @returns {string|null}
|
|
1767
|
+
*/
|
|
1768
|
+
getCacheKey(role) {
|
|
1769
|
+
if (role !== "retriever" || !this.cacheKey) return null;
|
|
1770
|
+
return `pinecone|${this._host || this.index}|${this.namespace}|${this.cacheKey}`;
|
|
1738
1771
|
}
|
|
1739
|
-
|
|
1772
|
+
/**
|
|
1773
|
+
* Whether the index embeds text itself (integrated embedding): then
|
|
1774
|
+
* store() needs no vectors and retrieve() takes the query text, so no
|
|
1775
|
+
* embedder is needed. The CLI checks this before requiring an embedder.
|
|
1776
|
+
* Reads the index description on first use.
|
|
1777
|
+
* @param {{ signal?: AbortSignal }} [options]
|
|
1778
|
+
* @returns {Promise<boolean>}
|
|
1779
|
+
*/
|
|
1780
|
+
async embedsText({ signal } = {}) {
|
|
1781
|
+
await this._resolve(signal);
|
|
1782
|
+
return this._integrated;
|
|
1783
|
+
}
|
|
1784
|
+
/**
|
|
1785
|
+
* One REST call; resolves the parsed body and the response headers.
|
|
1786
|
+
* @private
|
|
1787
|
+
*/
|
|
1788
|
+
async _call(url, method, body, signal) {
|
|
1740
1789
|
throwIfAborted(signal);
|
|
1790
|
+
const headers = {
|
|
1791
|
+
"Api-Key": this._apiKey,
|
|
1792
|
+
"X-Pinecone-Api-Version": API_VERSION,
|
|
1793
|
+
accept: "application/json"
|
|
1794
|
+
};
|
|
1795
|
+
if (body !== void 0) headers["content-type"] = "application/json";
|
|
1796
|
+
let res;
|
|
1741
1797
|
try {
|
|
1742
|
-
|
|
1798
|
+
res = await this._fetch(url, {
|
|
1799
|
+
method,
|
|
1800
|
+
headers,
|
|
1801
|
+
...body !== void 0 && { body: JSON.stringify(body) },
|
|
1802
|
+
...signal && { signal }
|
|
1803
|
+
});
|
|
1743
1804
|
} catch (err) {
|
|
1744
|
-
|
|
1805
|
+
throwIfAborted(signal);
|
|
1806
|
+
throw new Error(
|
|
1807
|
+
`PineconeRetriever: cannot reach Pinecone (${err.message})`
|
|
1808
|
+
);
|
|
1745
1809
|
}
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
async _queryWithEfSearch(text, params, ef, signal) {
|
|
1749
|
-
throwIfAborted(signal);
|
|
1750
|
-
const db = this._db();
|
|
1751
|
-
const pooled = typeof db.connect === "function" && typeof db.end === "function" && db.totalCount !== void 0;
|
|
1752
|
-
const client = pooled ? await db.connect() : db;
|
|
1810
|
+
const text = await res.text();
|
|
1811
|
+
let json;
|
|
1753
1812
|
try {
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
await client.query("COMMIT");
|
|
1758
|
-
return result;
|
|
1759
|
-
} catch (err) {
|
|
1760
|
-
await client.query("ROLLBACK").catch(() => {
|
|
1761
|
-
});
|
|
1762
|
-
throw explain(err, this.table);
|
|
1763
|
-
} finally {
|
|
1764
|
-
if (pooled) client.release();
|
|
1813
|
+
json = text ? JSON.parse(text) : {};
|
|
1814
|
+
} catch {
|
|
1815
|
+
json = {};
|
|
1765
1816
|
}
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
if (!this._ready) {
|
|
1771
|
-
const size = this.dimensions || dimensions;
|
|
1772
|
-
const indexName = `"${this.tableName.split(".").pop()}_embedding_hnsw"`;
|
|
1773
|
-
this._ready = (async () => {
|
|
1774
|
-
await this._query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
1775
|
-
await this._query(
|
|
1776
|
-
`CREATE TABLE IF NOT EXISTS ${this.table} (id text PRIMARY KEY, content text NOT NULL DEFAULT '', metadata jsonb NOT NULL DEFAULT '{}'::jsonb, embedding vector(${size}) NOT NULL)`
|
|
1817
|
+
if (!res.ok) {
|
|
1818
|
+
if (res.status === 401 || res.status === 403) {
|
|
1819
|
+
throw new Error(
|
|
1820
|
+
`PineconeRetriever: Pinecone refused the request (${res.status}); check PINECONE_API_KEY`
|
|
1777
1821
|
);
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
throw err;
|
|
1786
|
-
});
|
|
1822
|
+
}
|
|
1823
|
+
const detail = json.message || json.error && json.error.message || text || res.statusText;
|
|
1824
|
+
const err = new Error(
|
|
1825
|
+
`PineconeRetriever: ${method} ${new URL(url).pathname} failed (${res.status}): ${detail}`
|
|
1826
|
+
);
|
|
1827
|
+
err.status = res.status;
|
|
1828
|
+
throw err;
|
|
1787
1829
|
}
|
|
1788
|
-
return
|
|
1830
|
+
return { json, headers: res.headers };
|
|
1789
1831
|
}
|
|
1790
|
-
/**
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
* you set `cacheKey`: then cached retrieval results are reused until you
|
|
1794
|
-
* change the key or their time to live ends.
|
|
1795
|
-
* @param {'retriever'} role
|
|
1796
|
-
* @returns {string|null}
|
|
1797
|
-
*/
|
|
1798
|
-
getCacheKey(role) {
|
|
1799
|
-
if (role !== "retriever" || !this.cacheKey) return null;
|
|
1800
|
-
return `pgvector|${this.tableName}|${this.cacheKey}`;
|
|
1832
|
+
/** True on a document index that embeds text itself. @private */
|
|
1833
|
+
get _integrated() {
|
|
1834
|
+
return this._api === "documents" && Boolean(this._textField);
|
|
1801
1835
|
}
|
|
1802
1836
|
/**
|
|
1803
|
-
*
|
|
1804
|
-
*
|
|
1805
|
-
*
|
|
1806
|
-
* @
|
|
1837
|
+
* The index's host and kind, from its description (once): a document
|
|
1838
|
+
* index's dense_vector field and dimension or the text field it embeds,
|
|
1839
|
+
* or a vector index's dimension.
|
|
1840
|
+
* @private
|
|
1807
1841
|
*/
|
|
1808
|
-
|
|
1809
|
-
if (
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1842
|
+
_resolve(signal) {
|
|
1843
|
+
if (this._host && this._textField) {
|
|
1844
|
+
this._api = "documents";
|
|
1845
|
+
return Promise.resolve();
|
|
1846
|
+
}
|
|
1847
|
+
if (this._host && this._api !== "auto" && (this._api === "vectors" || this._vectorField)) {
|
|
1848
|
+
return Promise.resolve();
|
|
1849
|
+
}
|
|
1850
|
+
if (!this._resolving) {
|
|
1851
|
+
this._resolving = this._describe(signal).catch((err) => {
|
|
1852
|
+
this._resolving = null;
|
|
1853
|
+
throw err;
|
|
1854
|
+
});
|
|
1855
|
+
}
|
|
1856
|
+
return this._resolving;
|
|
1857
|
+
}
|
|
1858
|
+
/** @private */
|
|
1859
|
+
async _describe(signal) {
|
|
1860
|
+
let info;
|
|
1861
|
+
if (this.index) {
|
|
1862
|
+
({ json: info } = await this._call(
|
|
1863
|
+
`${CONTROL_PLANE}/indexes/${encodeURIComponent(this.index)}`,
|
|
1864
|
+
"GET",
|
|
1865
|
+
void 0,
|
|
1866
|
+
signal
|
|
1867
|
+
));
|
|
1868
|
+
} else {
|
|
1869
|
+
const { json } = await this._call(
|
|
1870
|
+
`${CONTROL_PLANE}/indexes`,
|
|
1871
|
+
"GET",
|
|
1872
|
+
void 0,
|
|
1873
|
+
signal
|
|
1874
|
+
);
|
|
1875
|
+
const host = this._host.replace(/^https?:\/\//, "");
|
|
1876
|
+
info = (json.indexes || []).find((i) => i.host === host);
|
|
1877
|
+
if (!info) {
|
|
1816
1878
|
throw new Error(
|
|
1817
|
-
|
|
1879
|
+
`PineconeRetriever: no index with host ${host} in this project; check the host, or pass index (its name)`
|
|
1818
1880
|
);
|
|
1819
1881
|
}
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1882
|
+
}
|
|
1883
|
+
if (!info.host) {
|
|
1884
|
+
throw new Error(
|
|
1885
|
+
`PineconeRetriever: index "${info.name || this.index}" has no host yet (is it ready?)`
|
|
1886
|
+
);
|
|
1887
|
+
}
|
|
1888
|
+
this._host = this._host || normalizeHost(info.host);
|
|
1889
|
+
const fields = info.schema && info.schema.fields || null;
|
|
1890
|
+
const named = fields ? Object.entries(fields).filter(([name]) => !name.startsWith("_")) : [];
|
|
1891
|
+
const denseFields = named.filter(([, f]) => f && f.type === "dense_vector");
|
|
1892
|
+
const textFields = named.filter(
|
|
1893
|
+
([, f]) => f && f.type === "string" && f.embed
|
|
1894
|
+
);
|
|
1895
|
+
if (this._api === "auto") {
|
|
1896
|
+
this._api = named.length > 0 ? "documents" : "vectors";
|
|
1897
|
+
}
|
|
1898
|
+
if (this._textField) {
|
|
1899
|
+
if (this._api !== "documents" || !textFields.some(([name]) => name === this._textField)) {
|
|
1823
1900
|
throw new Error(
|
|
1824
|
-
`
|
|
1901
|
+
`PineconeRetriever: index "${info.name}" has no text field named "${this._textField}" that it embeds`
|
|
1902
|
+
);
|
|
1903
|
+
}
|
|
1904
|
+
} else if (this._api === "documents") {
|
|
1905
|
+
const field = this._vectorField ? denseFields.find(([name]) => name === this._vectorField) : denseFields[0];
|
|
1906
|
+
if (!field && !this._vectorField && textFields.length > 0) {
|
|
1907
|
+
this._textField = textFields[0][0];
|
|
1908
|
+
return;
|
|
1909
|
+
}
|
|
1910
|
+
if (!field) {
|
|
1911
|
+
throw new Error(
|
|
1912
|
+
`PineconeRetriever: index "${info.name}" has no dense_vector field${this._vectorField ? ` named "${this._vectorField}"` : ""}`
|
|
1913
|
+
);
|
|
1914
|
+
}
|
|
1915
|
+
this._vectorField = field[0];
|
|
1916
|
+
this._indexDimension = field[1].dimension;
|
|
1917
|
+
} else {
|
|
1918
|
+
const values = fields && fields._values;
|
|
1919
|
+
this._indexDimension = info.dimension || values && values.dimension;
|
|
1920
|
+
}
|
|
1921
|
+
}
|
|
1922
|
+
/** A data-plane call; records the log sequence number of writes. @private */
|
|
1923
|
+
async _data(method, path, body, { signal, write = false } = {}) {
|
|
1924
|
+
await this._resolve(signal);
|
|
1925
|
+
const result = await this._call(
|
|
1926
|
+
`${this._host}${path}`,
|
|
1927
|
+
method,
|
|
1928
|
+
body,
|
|
1929
|
+
signal
|
|
1930
|
+
);
|
|
1931
|
+
if (write) {
|
|
1932
|
+
const lsn = Number(result.headers.get("x-pinecone-request-lsn"));
|
|
1933
|
+
if (Number.isFinite(lsn) && lsn > this._writeLsn) this._writeLsn = lsn;
|
|
1934
|
+
}
|
|
1935
|
+
return result;
|
|
1936
|
+
}
|
|
1937
|
+
/** The Documents API path for this namespace. @private */
|
|
1938
|
+
_docs(action) {
|
|
1939
|
+
const ns = encodeURIComponent(this.namespace || "__default__");
|
|
1940
|
+
return `/namespaces/${ns}/documents/${action}`;
|
|
1941
|
+
}
|
|
1942
|
+
/**
|
|
1943
|
+
* A search, repeated until it reflects this retriever's writes (when the
|
|
1944
|
+
* index reports log sequence numbers) or `freshnessTimeoutMs` passes.
|
|
1945
|
+
* `query` is a vector, or the query text on an index that embeds text
|
|
1946
|
+
* itself. Resolves `[{ id, score, fields }]`.
|
|
1947
|
+
* @private
|
|
1948
|
+
*/
|
|
1949
|
+
async _search(query, topK, signal, filter) {
|
|
1950
|
+
await this._resolve(signal);
|
|
1951
|
+
const documents = this._api === "documents";
|
|
1952
|
+
const [path, body] = documents ? [
|
|
1953
|
+
this._docs("search"),
|
|
1954
|
+
{
|
|
1955
|
+
score_by: [
|
|
1956
|
+
this._integrated ? { type: "embed", fields: [this._textField], query } : {
|
|
1957
|
+
type: "dense_vector",
|
|
1958
|
+
fields: [this._vectorField],
|
|
1959
|
+
values: query
|
|
1960
|
+
}
|
|
1961
|
+
],
|
|
1962
|
+
top_k: topK,
|
|
1963
|
+
include_fields: FIELDS,
|
|
1964
|
+
...filter && { filter }
|
|
1965
|
+
}
|
|
1966
|
+
] : [
|
|
1967
|
+
"/query",
|
|
1968
|
+
{
|
|
1969
|
+
vector: query,
|
|
1970
|
+
topK,
|
|
1971
|
+
namespace: this.namespace,
|
|
1972
|
+
includeMetadata: true,
|
|
1973
|
+
...filter && { filter }
|
|
1974
|
+
}
|
|
1975
|
+
];
|
|
1976
|
+
const deadline = Date.now() + this.freshnessTimeoutMs;
|
|
1977
|
+
for (let delay = 100; ; delay = Math.min(delay * 2, 2e3)) {
|
|
1978
|
+
let json;
|
|
1979
|
+
let headers;
|
|
1980
|
+
try {
|
|
1981
|
+
({ json, headers } = await this._data("POST", path, body, { signal }));
|
|
1982
|
+
} catch (err) {
|
|
1983
|
+
if (err.status === 404) return [];
|
|
1984
|
+
throw err;
|
|
1985
|
+
}
|
|
1986
|
+
const header = headers.get("x-pinecone-max-indexed-lsn");
|
|
1987
|
+
const fresh = !this._writeLsn || header === null || !(Number(header) < this._writeLsn) || Date.now() + delay > deadline;
|
|
1988
|
+
if (fresh) {
|
|
1989
|
+
return (json.matches || []).map(
|
|
1990
|
+
(m) => documents ? { id: m._id, score: m._score, fields: m } : { id: m.id, score: m.score, fields: m.metadata }
|
|
1991
|
+
);
|
|
1992
|
+
}
|
|
1993
|
+
await sleep(delay);
|
|
1994
|
+
throwIfAborted(signal);
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
/** Waits until this retriever's writes are indexed (for list and size). @private */
|
|
1998
|
+
async _waitForWrites(signal) {
|
|
1999
|
+
if (!this._writeLsn) return;
|
|
2000
|
+
const probe = this._probe();
|
|
2001
|
+
if (probe) await this._search(probe, 1, signal);
|
|
2002
|
+
}
|
|
2003
|
+
/**
|
|
2004
|
+
* What list() and size() search with: a unit vector of the index's
|
|
2005
|
+
* dimension, or a fixed text on an index that embeds text itself.
|
|
2006
|
+
* @private
|
|
2007
|
+
*/
|
|
2008
|
+
_probe() {
|
|
2009
|
+
if (this._integrated) return PROBE_QUERY;
|
|
2010
|
+
const dimension = this._indexDimension || this._dimension;
|
|
2011
|
+
if (!dimension) return null;
|
|
2012
|
+
return new Array(dimension).fill(0).map((_, i) => i === 0 ? 1 : 0);
|
|
2013
|
+
}
|
|
2014
|
+
/**
|
|
2015
|
+
* Inserts or replaces documents. On an index that embeds text itself,
|
|
2016
|
+
* Pinecone embeds `content` (required there) and `vector` is ignored.
|
|
2017
|
+
* @param {Array<{ id: string, vector?: number[], content?: string, metadata?: Object }>} docs
|
|
2018
|
+
* `vector` is required unless the index embeds text itself
|
|
2019
|
+
* @returns {Promise<void>}
|
|
2020
|
+
*/
|
|
2021
|
+
async store(docs) {
|
|
2022
|
+
if (!Array.isArray(docs)) {
|
|
2023
|
+
throw new Error("PineconeRetriever.store() takes an array");
|
|
2024
|
+
}
|
|
2025
|
+
if (docs.length === 0) return;
|
|
2026
|
+
await this._resolve();
|
|
2027
|
+
if (this._integrated) return this._storeText(docs);
|
|
2028
|
+
const expected = this._indexDimension || this._dimension;
|
|
2029
|
+
let dimensions = null;
|
|
2030
|
+
const records = docs.map((doc) => {
|
|
2031
|
+
const id = doc && doc.id;
|
|
2032
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
2033
|
+
throw new Error(
|
|
2034
|
+
"PineconeRetriever.store(): every document needs a string id"
|
|
2035
|
+
);
|
|
2036
|
+
}
|
|
2037
|
+
checkVector(doc.vector, id);
|
|
2038
|
+
if (dimensions === null) dimensions = doc.vector.length;
|
|
2039
|
+
else if (doc.vector.length !== dimensions) {
|
|
2040
|
+
throw new Error(
|
|
2041
|
+
`PineconeRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the batch has ${dimensions}`
|
|
2042
|
+
);
|
|
2043
|
+
}
|
|
2044
|
+
if (expected && doc.vector.length !== expected) {
|
|
2045
|
+
throw new Error(
|
|
2046
|
+
`PineconeRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the index has ${expected}`
|
|
2047
|
+
);
|
|
2048
|
+
}
|
|
2049
|
+
const metadata = doc.metadata || {};
|
|
2050
|
+
return {
|
|
2051
|
+
id: recordId(id),
|
|
2052
|
+
values: doc.vector,
|
|
2053
|
+
fields: {
|
|
2054
|
+
rag_id: id,
|
|
2055
|
+
rag_content: doc.content || "",
|
|
2056
|
+
rag_metadata: JSON.stringify(metadata),
|
|
2057
|
+
...metadata.source !== void 0 && metadata.source !== null && {
|
|
2058
|
+
rag_source: String(metadata.source)
|
|
2059
|
+
}
|
|
2060
|
+
}
|
|
2061
|
+
};
|
|
2062
|
+
});
|
|
2063
|
+
for (let i = 0; i < records.length; i += UPSERT_BATCH) {
|
|
2064
|
+
const batch = records.slice(i, i + UPSERT_BATCH);
|
|
2065
|
+
if (this._api === "documents") {
|
|
2066
|
+
await this._data(
|
|
2067
|
+
"POST",
|
|
2068
|
+
this._docs("upsert"),
|
|
2069
|
+
{
|
|
2070
|
+
documents: batch.map((r) => ({
|
|
2071
|
+
_id: r.id,
|
|
2072
|
+
[this._vectorField]: r.values,
|
|
2073
|
+
...r.fields
|
|
2074
|
+
}))
|
|
2075
|
+
},
|
|
2076
|
+
{ write: true }
|
|
2077
|
+
);
|
|
2078
|
+
} else {
|
|
2079
|
+
await this._data(
|
|
2080
|
+
"POST",
|
|
2081
|
+
"/vectors/upsert",
|
|
2082
|
+
{
|
|
2083
|
+
vectors: batch.map((r) => ({
|
|
2084
|
+
id: r.id,
|
|
2085
|
+
values: r.values,
|
|
2086
|
+
metadata: r.fields
|
|
2087
|
+
})),
|
|
2088
|
+
namespace: this.namespace
|
|
2089
|
+
},
|
|
2090
|
+
{ write: true }
|
|
2091
|
+
);
|
|
2092
|
+
}
|
|
2093
|
+
}
|
|
2094
|
+
this._dimension = dimensions;
|
|
2095
|
+
}
|
|
2096
|
+
/** store() on an index that embeds text itself. @private */
|
|
2097
|
+
async _storeText(docs) {
|
|
2098
|
+
const records = docs.map((doc) => {
|
|
2099
|
+
const id = doc && doc.id;
|
|
2100
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
2101
|
+
throw new Error(
|
|
2102
|
+
"PineconeRetriever.store(): every document needs a string id"
|
|
2103
|
+
);
|
|
2104
|
+
}
|
|
2105
|
+
if (typeof doc.content !== "string" || doc.content.trim() === "") {
|
|
2106
|
+
throw new Error(
|
|
2107
|
+
`PineconeRetriever.store(): document "${id}" has no content for Pinecone to embed (the index embeds ${this._textField})`
|
|
2108
|
+
);
|
|
2109
|
+
}
|
|
2110
|
+
const metadata = doc.metadata || {};
|
|
2111
|
+
return {
|
|
2112
|
+
_id: recordId(id),
|
|
2113
|
+
[this._textField]: doc.content,
|
|
2114
|
+
rag_id: id,
|
|
2115
|
+
rag_content: doc.content,
|
|
2116
|
+
rag_metadata: JSON.stringify(metadata),
|
|
2117
|
+
...metadata.source !== void 0 && metadata.source !== null && {
|
|
2118
|
+
rag_source: String(metadata.source)
|
|
2119
|
+
}
|
|
2120
|
+
};
|
|
2121
|
+
});
|
|
2122
|
+
const encoder = new TextEncoder();
|
|
2123
|
+
let batch = [];
|
|
2124
|
+
let bytes = 0;
|
|
2125
|
+
const send = async () => {
|
|
2126
|
+
if (batch.length === 0) return;
|
|
2127
|
+
await this._data(
|
|
2128
|
+
"POST",
|
|
2129
|
+
this._docs("upsert"),
|
|
2130
|
+
{ documents: batch },
|
|
2131
|
+
{ write: true }
|
|
2132
|
+
);
|
|
2133
|
+
batch = [];
|
|
2134
|
+
bytes = 0;
|
|
2135
|
+
};
|
|
2136
|
+
for (const record of records) {
|
|
2137
|
+
const size = encoder.encode(JSON.stringify(record)).length + 1;
|
|
2138
|
+
if (batch.length === EMBED_UPSERT_BATCH || batch.length > 0 && bytes + size > REQUEST_BYTES) {
|
|
2139
|
+
await send();
|
|
2140
|
+
}
|
|
2141
|
+
batch.push(record);
|
|
2142
|
+
bytes += size;
|
|
2143
|
+
}
|
|
2144
|
+
await send();
|
|
2145
|
+
}
|
|
2146
|
+
/**
|
|
2147
|
+
* The documents most similar to a query vector, or to the query text on
|
|
2148
|
+
* an index that embeds text itself (Pinecone embeds the query; a
|
|
2149
|
+
* `queryVector` is then ignored).
|
|
2150
|
+
* @param {number[]|string|{ query?: string, queryVector?: number[], topK?: number, signal?: AbortSignal }} request
|
|
2151
|
+
* A vector, the query text, or an object with them: `queryVector`, or
|
|
2152
|
+
* `query` on an index that embeds text itself (required; optional in
|
|
2153
|
+
* the type to match the retriever contract)
|
|
2154
|
+
* @param {number} [k=10] - Results when `request` is a vector or text
|
|
2155
|
+
* @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
|
|
2156
|
+
*/
|
|
2157
|
+
async retrieve(request, k = 10) {
|
|
2158
|
+
let input = request || {};
|
|
2159
|
+
if (Array.isArray(request)) input = { queryVector: request };
|
|
2160
|
+
else if (typeof request === "string") input = { query: request };
|
|
2161
|
+
const { query, queryVector, topK = k, signal } = input;
|
|
2162
|
+
if (!Number.isInteger(topK) || topK < 1) {
|
|
2163
|
+
throw new Error(
|
|
2164
|
+
"PineconeRetriever.retrieve(): topK must be a positive integer"
|
|
2165
|
+
);
|
|
2166
|
+
}
|
|
2167
|
+
throwIfAborted(signal);
|
|
2168
|
+
await this._resolve(signal);
|
|
2169
|
+
if (!this._integrated) {
|
|
2170
|
+
if (queryVector === void 0 || queryVector === null) {
|
|
2171
|
+
throw new Error(
|
|
2172
|
+
"PineconeRetriever.retrieve(): needs a queryVector; this index does not embed text itself, so use an embedder"
|
|
2173
|
+
);
|
|
2174
|
+
}
|
|
2175
|
+
checkVector(queryVector, "query");
|
|
2176
|
+
} else if (typeof query !== "string" || query.trim() === "") {
|
|
2177
|
+
throw new Error(
|
|
2178
|
+
`PineconeRetriever.retrieve(): the index embeds text itself (integrated embedding, field ${this._textField}), so it needs the query text: pass { query }`
|
|
2179
|
+
);
|
|
2180
|
+
}
|
|
2181
|
+
const matches = await this._search(
|
|
2182
|
+
this._integrated ? query : queryVector,
|
|
2183
|
+
Math.min(topK, 1e4),
|
|
2184
|
+
signal
|
|
2185
|
+
);
|
|
2186
|
+
throwIfAborted(signal);
|
|
2187
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2188
|
+
const out = [];
|
|
2189
|
+
for (const m of matches) {
|
|
2190
|
+
if (seen.has(m.id)) continue;
|
|
2191
|
+
seen.add(m.id);
|
|
2192
|
+
out.push({ ...fromFields(m.fields, m.id), score: m.score });
|
|
2193
|
+
}
|
|
2194
|
+
return out;
|
|
2195
|
+
}
|
|
2196
|
+
/** @private */
|
|
2197
|
+
async _delete(body) {
|
|
2198
|
+
await this._resolve();
|
|
2199
|
+
try {
|
|
2200
|
+
if (this._api === "documents") {
|
|
2201
|
+
await this._data("POST", this._docs("delete"), body.documents, {
|
|
2202
|
+
write: true
|
|
2203
|
+
});
|
|
2204
|
+
} else {
|
|
2205
|
+
await this._data(
|
|
2206
|
+
"POST",
|
|
2207
|
+
"/vectors/delete",
|
|
2208
|
+
{ namespace: this.namespace, ...body.vectors },
|
|
2209
|
+
{ write: true }
|
|
2210
|
+
);
|
|
2211
|
+
}
|
|
2212
|
+
} catch (err) {
|
|
2213
|
+
if (err.status !== 404) throw err;
|
|
2214
|
+
}
|
|
2215
|
+
}
|
|
2216
|
+
/**
|
|
2217
|
+
* Deletes the chunks of one source (`metadata.source`), for re-ingesting.
|
|
2218
|
+
* @param {string} source
|
|
2219
|
+
*/
|
|
2220
|
+
async removeBySource(source) {
|
|
2221
|
+
const filter = { rag_source: { $eq: String(source) } };
|
|
2222
|
+
await this._delete({ documents: { filter }, vectors: { filter } });
|
|
2223
|
+
}
|
|
2224
|
+
/**
|
|
2225
|
+
* Deletes documents by id.
|
|
2226
|
+
* @param {string[]} ids
|
|
2227
|
+
*/
|
|
2228
|
+
async delete(ids) {
|
|
2229
|
+
const recordIds = ids.map(recordId);
|
|
2230
|
+
for (let i = 0; i < recordIds.length; i += DELETE_BATCH) {
|
|
2231
|
+
const batch = recordIds.slice(i, i + DELETE_BATCH);
|
|
2232
|
+
await this._delete({
|
|
2233
|
+
documents: { ids: batch },
|
|
2234
|
+
vectors: { ids: batch }
|
|
2235
|
+
});
|
|
2236
|
+
}
|
|
2237
|
+
}
|
|
2238
|
+
/** Deletes every document this retriever stored in its namespace. */
|
|
2239
|
+
async clear() {
|
|
2240
|
+
await this._delete({
|
|
2241
|
+
documents: { filter: OURS },
|
|
2242
|
+
vectors: { deleteAll: true }
|
|
2243
|
+
});
|
|
2244
|
+
}
|
|
2245
|
+
/**
|
|
2246
|
+
* Every document, without vectors.
|
|
2247
|
+
* @param {{ limit?: number }} [options]
|
|
2248
|
+
* @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
|
|
2249
|
+
*/
|
|
2250
|
+
async list({ limit } = {}) {
|
|
2251
|
+
await this._resolve();
|
|
2252
|
+
await this._waitForWrites();
|
|
2253
|
+
return this._api === "documents" ? this._listDocuments(limit) : this._listVectors(limit);
|
|
2254
|
+
}
|
|
2255
|
+
/** @returns {Promise<number>} The documents this retriever stored */
|
|
2256
|
+
async size() {
|
|
2257
|
+
return (await this.list()).length;
|
|
2258
|
+
}
|
|
2259
|
+
/** Documents API: fetch by filter, page by page. @private */
|
|
2260
|
+
async _listDocuments(limit) {
|
|
2261
|
+
const probe = this._probe();
|
|
2262
|
+
if (probe) {
|
|
2263
|
+
const max = Math.min(limit || SEARCH_MAX, SEARCH_MAX);
|
|
2264
|
+
const found = await this._search(probe, max, void 0, OURS);
|
|
2265
|
+
if (found.length < SEARCH_MAX || limit) {
|
|
2266
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2267
|
+
return found.filter((m) => !seen.has(m.id) && seen.add(m.id)).map((m) => fromFields(m.fields, m.id));
|
|
2268
|
+
}
|
|
2269
|
+
}
|
|
2270
|
+
const out = [];
|
|
2271
|
+
let token;
|
|
2272
|
+
do {
|
|
2273
|
+
let json;
|
|
2274
|
+
try {
|
|
2275
|
+
({ json } = await this._data("POST", this._docs("fetch"), {
|
|
2276
|
+
filter: OURS,
|
|
2277
|
+
include_fields: FIELDS,
|
|
2278
|
+
limit: limit ? Math.min(PAGE, limit - out.length) : PAGE,
|
|
2279
|
+
...token && { pagination_token: token }
|
|
2280
|
+
}));
|
|
2281
|
+
} catch (err) {
|
|
2282
|
+
if (err.status === 404) break;
|
|
2283
|
+
throw err;
|
|
2284
|
+
}
|
|
2285
|
+
for (const [id, fields] of Object.entries(json.documents || {})) {
|
|
2286
|
+
out.push(fromFields(fields, id));
|
|
2287
|
+
}
|
|
2288
|
+
token = json.pagination && json.pagination.next;
|
|
2289
|
+
} while (token && (!limit || out.length < limit));
|
|
2290
|
+
return limit ? out.slice(0, limit) : out;
|
|
2291
|
+
}
|
|
2292
|
+
/** Vectors API: list ids, then fetch them. @private */
|
|
2293
|
+
async _listVectors(limit) {
|
|
2294
|
+
const ids = [];
|
|
2295
|
+
let token;
|
|
2296
|
+
do {
|
|
2297
|
+
const params = new URLSearchParams({
|
|
2298
|
+
namespace: this.namespace,
|
|
2299
|
+
limit: String(PAGE)
|
|
2300
|
+
});
|
|
2301
|
+
if (token) params.set("paginationToken", token);
|
|
2302
|
+
let json;
|
|
2303
|
+
try {
|
|
2304
|
+
({ json } = await this._data("GET", `/vectors/list?${params}`));
|
|
2305
|
+
} catch (err) {
|
|
2306
|
+
if (err.status === 404) break;
|
|
2307
|
+
throw err;
|
|
2308
|
+
}
|
|
2309
|
+
for (const v of json.vectors || []) ids.push(v.id);
|
|
2310
|
+
token = json.pagination && json.pagination.next;
|
|
2311
|
+
} while (token && (!limit || ids.length < limit));
|
|
2312
|
+
const wanted = limit ? ids.slice(0, limit) : ids;
|
|
2313
|
+
const out = [];
|
|
2314
|
+
for (let i = 0; i < wanted.length; i += PAGE) {
|
|
2315
|
+
const params = new URLSearchParams({ namespace: this.namespace });
|
|
2316
|
+
for (const id of wanted.slice(i, i + PAGE)) params.append("ids", id);
|
|
2317
|
+
const { json } = await this._data("GET", `/vectors/fetch?${params}`);
|
|
2318
|
+
for (const id of wanted.slice(i, i + PAGE)) {
|
|
2319
|
+
const record = json.vectors && json.vectors[id];
|
|
2320
|
+
if (record) out.push(fromFields(record.metadata, id));
|
|
2321
|
+
}
|
|
2322
|
+
}
|
|
2323
|
+
return out;
|
|
2324
|
+
}
|
|
2325
|
+
};
|
|
2326
|
+
module2.exports = { PineconeRetriever, recordId, API_VERSION };
|
|
2327
|
+
}
|
|
2328
|
+
});
|
|
2329
|
+
|
|
2330
|
+
// src/retrieval/pgvector-retriever.js
|
|
2331
|
+
var require_pgvector_retriever = __commonJS({
|
|
2332
|
+
"src/retrieval/pgvector-retriever.js"(exports2, module2) {
|
|
2333
|
+
"use strict";
|
|
2334
|
+
var { throwIfAborted, raceAbort } = require_connector_utils();
|
|
2335
|
+
var IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
|
|
2336
|
+
var DEFAULT_TABLE = "rag_documents";
|
|
2337
|
+
var PG_DEFAULT_EF_SEARCH = 40;
|
|
2338
|
+
var MAX_BATCH = 1e3;
|
|
2339
|
+
function quoteTable(name) {
|
|
2340
|
+
const parts = String(name).split(".");
|
|
2341
|
+
if (parts.length > 2 || !parts.every((p) => IDENTIFIER.test(p))) {
|
|
2342
|
+
throw new Error(
|
|
2343
|
+
`PgVectorRetriever: invalid table name "${name}" (letters, digits and underscores; optionally schema.table)`
|
|
2344
|
+
);
|
|
2345
|
+
}
|
|
2346
|
+
return parts.map((p) => `"${p}"`).join(".");
|
|
2347
|
+
}
|
|
2348
|
+
function vectorLiteral(vector, id) {
|
|
2349
|
+
if (!Array.isArray(vector) || vector.length === 0) {
|
|
2350
|
+
throw new Error(`PgVectorRetriever: document "${id}" has no vector`);
|
|
2351
|
+
}
|
|
2352
|
+
for (const x of vector) {
|
|
2353
|
+
if (typeof x !== "number" || !Number.isFinite(x)) {
|
|
2354
|
+
throw new Error(
|
|
2355
|
+
`PgVectorRetriever: document "${id}" has a vector with a non-finite value`
|
|
2356
|
+
);
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
return `[${vector.join(",")}]`;
|
|
2360
|
+
}
|
|
2361
|
+
function explain(err, table) {
|
|
2362
|
+
const code = err && err.code;
|
|
2363
|
+
if (code === "42P01") {
|
|
2364
|
+
return new Error(
|
|
2365
|
+
`PgVectorRetriever: table ${table} does not exist. Store documents first (rag-pipeline ingest), or create it.`
|
|
2366
|
+
);
|
|
2367
|
+
}
|
|
2368
|
+
if (code === "42501" || /permission denied/i.test(String(err && err.message))) {
|
|
2369
|
+
return new Error(
|
|
2370
|
+
`PgVectorRetriever: permission denied. Run \`CREATE EXTENSION IF NOT EXISTS vector;\` as a database owner, and grant this user access to ${table}. (${err.message})`
|
|
2371
|
+
);
|
|
2372
|
+
}
|
|
2373
|
+
if (/different vector dimensions|expected \d+ dimensions/i.test(
|
|
2374
|
+
String(err && err.message)
|
|
2375
|
+
)) {
|
|
2376
|
+
return new Error(
|
|
2377
|
+
`PgVectorRetriever: ${err.message}. The table holds vectors from another embedder; use the same embedder or a new table.`
|
|
2378
|
+
);
|
|
2379
|
+
}
|
|
2380
|
+
return err;
|
|
2381
|
+
}
|
|
2382
|
+
var PgVectorRetriever = class {
|
|
2383
|
+
/**
|
|
2384
|
+
* @param {Object} options
|
|
2385
|
+
* @param {{ query: Function, connect?: Function }} [options.pool] - A pg Pool or Client you manage
|
|
2386
|
+
* @param {string} [options.connectionString] - Used to create a Pool when no `pool` is given
|
|
2387
|
+
* @param {string} [options.table='rag_documents'] - Table name, optionally `schema.table`
|
|
2388
|
+
* @param {number} [options.dimensions] - Vector size for a new table; defaults to the first stored vector's
|
|
2389
|
+
* @param {boolean} [options.createTable=true] - Create the extension, table and index on first store
|
|
2390
|
+
* @param {'hnsw'|'none'} [options.index='hnsw'] - Index created with the table
|
|
2391
|
+
* @param {number} [options.efSearch] - HNSW candidate list size per query (pgvector's default is 40). Higher finds more of the true nearest neighbours, more slowly; a query always uses at least its topK
|
|
2392
|
+
* @param {string} [options.cacheKey] - Cache identity; change it when the table changes (see getCacheKey)
|
|
2393
|
+
*/
|
|
2394
|
+
constructor(options = {}) {
|
|
2395
|
+
const {
|
|
2396
|
+
pool,
|
|
2397
|
+
connectionString,
|
|
2398
|
+
table = DEFAULT_TABLE,
|
|
2399
|
+
dimensions,
|
|
2400
|
+
createTable = true,
|
|
2401
|
+
index = "hnsw",
|
|
2402
|
+
efSearch,
|
|
2403
|
+
cacheKey
|
|
2404
|
+
} = options;
|
|
2405
|
+
if (!pool && !connectionString) {
|
|
2406
|
+
throw new Error(
|
|
2407
|
+
"PgVectorRetriever needs a pg `pool` (or Client) or a `connectionString`"
|
|
2408
|
+
);
|
|
2409
|
+
}
|
|
2410
|
+
if (pool && typeof pool.query !== "function") {
|
|
2411
|
+
throw new Error("PgVectorRetriever: `pool` must have a query() method");
|
|
2412
|
+
}
|
|
2413
|
+
if (!["hnsw", "none"].includes(index)) {
|
|
2414
|
+
throw new Error(
|
|
2415
|
+
`PgVectorRetriever: unknown index "${index}" (use "hnsw" or "none")`
|
|
2416
|
+
);
|
|
2417
|
+
}
|
|
2418
|
+
if (dimensions !== void 0 && !(Number.isInteger(dimensions) && dimensions > 0)) {
|
|
2419
|
+
throw new Error(
|
|
2420
|
+
"PgVectorRetriever: `dimensions` must be a positive integer"
|
|
2421
|
+
);
|
|
2422
|
+
}
|
|
2423
|
+
if (efSearch !== void 0 && !(Number.isInteger(efSearch) && efSearch > 0 && efSearch <= 1e3)) {
|
|
2424
|
+
throw new Error(
|
|
2425
|
+
"PgVectorRetriever: `efSearch` must be an integer from 1 to 1000"
|
|
2426
|
+
);
|
|
2427
|
+
}
|
|
2428
|
+
this.efSearch = efSearch;
|
|
2429
|
+
this.table = quoteTable(table);
|
|
2430
|
+
this.tableName = table;
|
|
2431
|
+
this.dimensions = dimensions;
|
|
2432
|
+
this.createTable = createTable;
|
|
2433
|
+
this.index = index;
|
|
2434
|
+
this.cacheKey = cacheKey;
|
|
2435
|
+
this._pool = pool || null;
|
|
2436
|
+
this._connectionString = connectionString;
|
|
2437
|
+
this._ownsPool = !pool;
|
|
2438
|
+
this._ready = null;
|
|
2439
|
+
}
|
|
2440
|
+
/** The pg Pool or Client in use; creates the Pool on first use. */
|
|
2441
|
+
_db() {
|
|
2442
|
+
if (!this._pool) {
|
|
2443
|
+
let pg;
|
|
2444
|
+
try {
|
|
2445
|
+
pg = require("pg");
|
|
2446
|
+
} catch {
|
|
2447
|
+
throw new Error(
|
|
2448
|
+
'PgVectorRetriever with a connectionString needs the "pg" package: npm install pg'
|
|
2449
|
+
);
|
|
2450
|
+
}
|
|
2451
|
+
this._pool = new pg.Pool({
|
|
2452
|
+
connectionString: this._connectionString,
|
|
2453
|
+
allowExitOnIdle: true
|
|
2454
|
+
});
|
|
2455
|
+
}
|
|
2456
|
+
return this._pool;
|
|
2457
|
+
}
|
|
2458
|
+
async _query(text, params, signal) {
|
|
2459
|
+
throwIfAborted(signal);
|
|
2460
|
+
try {
|
|
2461
|
+
return await raceAbort(this._db().query(text, params), signal);
|
|
2462
|
+
} catch (err) {
|
|
2463
|
+
throw explain(err, this.table);
|
|
2464
|
+
}
|
|
2465
|
+
}
|
|
2466
|
+
/** Runs one query with `SET LOCAL hnsw.ef_search`, in its own transaction. */
|
|
2467
|
+
async _queryWithEfSearch(text, params, ef, signal) {
|
|
2468
|
+
throwIfAborted(signal);
|
|
2469
|
+
const db = this._db();
|
|
2470
|
+
const pooled = typeof db.connect === "function" && typeof db.end === "function" && db.totalCount !== void 0;
|
|
2471
|
+
const client = pooled ? await db.connect() : db;
|
|
2472
|
+
try {
|
|
2473
|
+
await client.query("BEGIN");
|
|
2474
|
+
await client.query(`SET LOCAL hnsw.ef_search = ${Math.min(ef, 1e3)}`);
|
|
2475
|
+
const result = await client.query(text, params);
|
|
2476
|
+
await client.query("COMMIT");
|
|
2477
|
+
return result;
|
|
2478
|
+
} catch (err) {
|
|
2479
|
+
await client.query("ROLLBACK").catch(() => {
|
|
2480
|
+
});
|
|
2481
|
+
throw explain(err, this.table);
|
|
2482
|
+
} finally {
|
|
2483
|
+
if (pooled) client.release();
|
|
2484
|
+
}
|
|
2485
|
+
}
|
|
2486
|
+
/** Creates the extension, table and index once, sized to `dimensions`. */
|
|
2487
|
+
_ensureTable(dimensions) {
|
|
2488
|
+
if (!this.createTable) return Promise.resolve();
|
|
2489
|
+
if (!this._ready) {
|
|
2490
|
+
const size = this.dimensions || dimensions;
|
|
2491
|
+
const indexName = `"${this.tableName.split(".").pop()}_embedding_hnsw"`;
|
|
2492
|
+
this._ready = (async () => {
|
|
2493
|
+
await this._query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
2494
|
+
await this._query(
|
|
2495
|
+
`CREATE TABLE IF NOT EXISTS ${this.table} (id text PRIMARY KEY, content text NOT NULL DEFAULT '', metadata jsonb NOT NULL DEFAULT '{}'::jsonb, embedding vector(${size}) NOT NULL)`
|
|
2496
|
+
);
|
|
2497
|
+
if (this.index === "hnsw") {
|
|
2498
|
+
await this._query(
|
|
2499
|
+
`CREATE INDEX IF NOT EXISTS ${indexName} ON ${this.table} USING hnsw (embedding vector_cosine_ops)`
|
|
2500
|
+
);
|
|
2501
|
+
}
|
|
2502
|
+
})().catch((err) => {
|
|
2503
|
+
this._ready = null;
|
|
2504
|
+
throw err;
|
|
2505
|
+
});
|
|
2506
|
+
}
|
|
2507
|
+
return this._ready;
|
|
2508
|
+
}
|
|
2509
|
+
/**
|
|
2510
|
+
* Cache identity for createRagPipeline({ cache }). Other processes can
|
|
2511
|
+
* write to the table without this one knowing, so there is none unless
|
|
2512
|
+
* you set `cacheKey`: then cached retrieval results are reused until you
|
|
2513
|
+
* change the key or their time to live ends.
|
|
2514
|
+
* @param {'retriever'} role
|
|
2515
|
+
* @returns {string|null}
|
|
2516
|
+
*/
|
|
2517
|
+
getCacheKey(role) {
|
|
2518
|
+
if (role !== "retriever" || !this.cacheKey) return null;
|
|
2519
|
+
return `pgvector|${this.tableName}|${this.cacheKey}`;
|
|
2520
|
+
}
|
|
2521
|
+
/**
|
|
2522
|
+
* Inserts or replaces documents, in one transaction: a failed batch
|
|
2523
|
+
* changes nothing.
|
|
2524
|
+
* @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
|
|
2525
|
+
* @returns {Promise<void>}
|
|
2526
|
+
*/
|
|
2527
|
+
async store(docs) {
|
|
2528
|
+
if (!Array.isArray(docs))
|
|
2529
|
+
throw new Error("PgVectorRetriever.store() takes an array");
|
|
2530
|
+
if (docs.length === 0) return;
|
|
2531
|
+
let dimensions = null;
|
|
2532
|
+
const rows = docs.map((doc) => {
|
|
2533
|
+
const id = doc && doc.id;
|
|
2534
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
2535
|
+
throw new Error(
|
|
2536
|
+
"PgVectorRetriever.store(): every document needs a string id"
|
|
2537
|
+
);
|
|
2538
|
+
}
|
|
2539
|
+
const literal = vectorLiteral(doc.vector, id);
|
|
2540
|
+
if (dimensions === null) dimensions = doc.vector.length;
|
|
2541
|
+
else if (doc.vector.length !== dimensions) {
|
|
2542
|
+
throw new Error(
|
|
2543
|
+
`PgVectorRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the batch has ${dimensions}`
|
|
1825
2544
|
);
|
|
1826
2545
|
}
|
|
1827
2546
|
return [
|
|
@@ -1961,6 +2680,453 @@ var require_pgvector_retriever = __commonJS({
|
|
|
1961
2680
|
}
|
|
1962
2681
|
});
|
|
1963
2682
|
|
|
2683
|
+
// src/retrieval/weaviate-retriever.js
|
|
2684
|
+
var require_weaviate_retriever = __commonJS({
|
|
2685
|
+
"src/retrieval/weaviate-retriever.js"(exports2, module2) {
|
|
2686
|
+
"use strict";
|
|
2687
|
+
var { throwIfAborted } = require_connector_utils();
|
|
2688
|
+
var { pointId } = require_qdrant_retriever();
|
|
2689
|
+
var COLLECTION = /^[A-Za-z][A-Za-z0-9_]{0,254}$/;
|
|
2690
|
+
var DEFAULT_COLLECTION = "RagDocuments";
|
|
2691
|
+
var PAGE = 256;
|
|
2692
|
+
var UPSERT_BATCH = 256;
|
|
2693
|
+
var DELETE_BATCH = 256;
|
|
2694
|
+
var PROPERTIES = [
|
|
2695
|
+
{
|
|
2696
|
+
name: "rag_id",
|
|
2697
|
+
dataType: ["text"],
|
|
2698
|
+
tokenization: "field",
|
|
2699
|
+
indexFilterable: true,
|
|
2700
|
+
indexSearchable: false
|
|
2701
|
+
},
|
|
2702
|
+
{ name: "content", dataType: ["text"], indexFilterable: false },
|
|
2703
|
+
{
|
|
2704
|
+
name: "rag_source",
|
|
2705
|
+
dataType: ["text"],
|
|
2706
|
+
tokenization: "field",
|
|
2707
|
+
indexFilterable: true,
|
|
2708
|
+
indexSearchable: false
|
|
2709
|
+
},
|
|
2710
|
+
{
|
|
2711
|
+
name: "rag_metadata",
|
|
2712
|
+
dataType: ["text"],
|
|
2713
|
+
indexFilterable: false,
|
|
2714
|
+
indexSearchable: false
|
|
2715
|
+
}
|
|
2716
|
+
];
|
|
2717
|
+
var OURS = { path: ["rag_id"], operator: "Like", valueText: "*" };
|
|
2718
|
+
var OURS_GRAPHQL = '{ path: ["rag_id"], operator: Like, valueText: "*" }';
|
|
2719
|
+
function checkVector(vector, id) {
|
|
2720
|
+
if (!Array.isArray(vector) || vector.length === 0) {
|
|
2721
|
+
throw new Error(`WeaviateRetriever: document "${id}" has no vector`);
|
|
2722
|
+
}
|
|
2723
|
+
if (!vector.every((x) => typeof x === "number" && Number.isFinite(x))) {
|
|
2724
|
+
throw new Error(
|
|
2725
|
+
`WeaviateRetriever: document "${id}" has a vector with a non-finite value`
|
|
2726
|
+
);
|
|
2727
|
+
}
|
|
2728
|
+
}
|
|
2729
|
+
function trimSlashes(url) {
|
|
2730
|
+
let end = url.length;
|
|
2731
|
+
while (end > 0 && url[end - 1] === "/") end--;
|
|
2732
|
+
return url.slice(0, end);
|
|
2733
|
+
}
|
|
2734
|
+
function parseMetadata(text) {
|
|
2735
|
+
if (!text) return {};
|
|
2736
|
+
try {
|
|
2737
|
+
const value = JSON.parse(text);
|
|
2738
|
+
return value && typeof value === "object" ? value : {};
|
|
2739
|
+
} catch {
|
|
2740
|
+
return {};
|
|
2741
|
+
}
|
|
2742
|
+
}
|
|
2743
|
+
function errorDetail(json) {
|
|
2744
|
+
const list = json && (json.error || json.errors) || [];
|
|
2745
|
+
if (Array.isArray(list) && list.length) {
|
|
2746
|
+
return list.map((e) => e && e.message || String(e)).join("; ");
|
|
2747
|
+
}
|
|
2748
|
+
return json && json.message || "";
|
|
2749
|
+
}
|
|
2750
|
+
var WeaviateRetriever = class {
|
|
2751
|
+
/**
|
|
2752
|
+
* @param {Object} [options]
|
|
2753
|
+
* @param {string} [options.url='http://localhost:8080'] - Weaviate's REST endpoint (without /v1)
|
|
2754
|
+
* @param {string} [options.apiKey] - Sent as `Authorization: Bearer <key>`
|
|
2755
|
+
* @param {string} [options.collection='RagDocuments'] - Class name; the first letter is upper-cased, as Weaviate does
|
|
2756
|
+
* @param {number} [options.dimensions] - Expected vector size; defaults to the collection's or the first stored vector's
|
|
2757
|
+
* @param {boolean} [options.createCollection=true] - Create the collection (and any missing properties) on first store
|
|
2758
|
+
* @param {string} [options.cacheKey] - Cache identity; change it when the collection changes
|
|
2759
|
+
* @param {Function} [options.fetch] - fetch implementation (defaults to the global)
|
|
2760
|
+
*/
|
|
2761
|
+
constructor(options = {}) {
|
|
2762
|
+
const {
|
|
2763
|
+
url = "http://localhost:8080",
|
|
2764
|
+
apiKey,
|
|
2765
|
+
collection = DEFAULT_COLLECTION,
|
|
2766
|
+
dimensions,
|
|
2767
|
+
createCollection = true,
|
|
2768
|
+
cacheKey,
|
|
2769
|
+
fetch: fetchImpl = globalThis.fetch
|
|
2770
|
+
} = options;
|
|
2771
|
+
if (typeof collection !== "string" || !COLLECTION.test(collection)) {
|
|
2772
|
+
throw new Error(
|
|
2773
|
+
`WeaviateRetriever: invalid collection name "${collection}" (a letter, then letters, digits and "_")`
|
|
2774
|
+
);
|
|
2775
|
+
}
|
|
2776
|
+
if (dimensions !== void 0 && !(Number.isInteger(dimensions) && dimensions > 0)) {
|
|
2777
|
+
throw new Error(
|
|
2778
|
+
"WeaviateRetriever: `dimensions` must be a positive integer"
|
|
2779
|
+
);
|
|
2780
|
+
}
|
|
2781
|
+
if (typeof fetchImpl !== "function") {
|
|
2782
|
+
throw new Error("WeaviateRetriever needs fetch (Node.js 18 or later)");
|
|
2783
|
+
}
|
|
2784
|
+
this.url = trimSlashes(String(url));
|
|
2785
|
+
this.collection = collection[0].toUpperCase() + collection.slice(1);
|
|
2786
|
+
this.dimensions = dimensions;
|
|
2787
|
+
this.createCollection = createCollection;
|
|
2788
|
+
this.cacheKey = cacheKey;
|
|
2789
|
+
this._apiKey = apiKey;
|
|
2790
|
+
this._fetch = fetchImpl === globalThis.fetch ? fetchImpl.bind(globalThis) : fetchImpl;
|
|
2791
|
+
this._ready = null;
|
|
2792
|
+
this._exists = false;
|
|
2793
|
+
}
|
|
2794
|
+
/**
|
|
2795
|
+
* Cache identity for createRagPipeline({ cache }): none unless you set
|
|
2796
|
+
* `cacheKey`, since other clients can change the collection.
|
|
2797
|
+
* @param {'retriever'} role
|
|
2798
|
+
* @returns {string|null}
|
|
2799
|
+
*/
|
|
2800
|
+
getCacheKey(role) {
|
|
2801
|
+
if (role !== "retriever" || !this.cacheKey) return null;
|
|
2802
|
+
return `weaviate|${this.url}|${this.collection}|${this.cacheKey}`;
|
|
2803
|
+
}
|
|
2804
|
+
/** The message without the API key, should a server ever echo it. */
|
|
2805
|
+
_redact(message) {
|
|
2806
|
+
const text = String(message);
|
|
2807
|
+
return this._apiKey ? text.split(this._apiKey).join("***") : text;
|
|
2808
|
+
}
|
|
2809
|
+
/**
|
|
2810
|
+
* One REST call; resolves the parsed body.
|
|
2811
|
+
* @private
|
|
2812
|
+
* @param {string} method
|
|
2813
|
+
* @param {string} path - After /v1
|
|
2814
|
+
* @param {Object} [body]
|
|
2815
|
+
* @param {{ signal?: AbortSignal, allow404?: boolean }} [options] - allow404 resolves null on 404
|
|
2816
|
+
* @returns {Promise<any>}
|
|
2817
|
+
*/
|
|
2818
|
+
async _request(method, path, body, { signal, allow404 = false } = {}) {
|
|
2819
|
+
throwIfAborted(signal);
|
|
2820
|
+
const headers = { "content-type": "application/json" };
|
|
2821
|
+
if (this._apiKey) headers.authorization = `Bearer ${this._apiKey}`;
|
|
2822
|
+
let res;
|
|
2823
|
+
try {
|
|
2824
|
+
res = await this._fetch(`${this.url}/v1${path}`, {
|
|
2825
|
+
method,
|
|
2826
|
+
headers,
|
|
2827
|
+
...body !== void 0 && { body: JSON.stringify(body) },
|
|
2828
|
+
...signal && { signal }
|
|
2829
|
+
});
|
|
2830
|
+
} catch (err) {
|
|
2831
|
+
throwIfAborted(signal);
|
|
2832
|
+
throw new Error(
|
|
2833
|
+
this._redact(
|
|
2834
|
+
`WeaviateRetriever: cannot reach Weaviate at ${this.url} (${err.message})`
|
|
2835
|
+
)
|
|
2836
|
+
);
|
|
2837
|
+
}
|
|
2838
|
+
if (res.status === 404 && allow404) return null;
|
|
2839
|
+
const text = await res.text();
|
|
2840
|
+
let json;
|
|
2841
|
+
try {
|
|
2842
|
+
json = text ? JSON.parse(text) : {};
|
|
2843
|
+
} catch {
|
|
2844
|
+
json = {};
|
|
2845
|
+
}
|
|
2846
|
+
if (!res.ok) {
|
|
2847
|
+
if (res.status === 401 || res.status === 403) {
|
|
2848
|
+
throw new Error(
|
|
2849
|
+
`WeaviateRetriever: Weaviate refused the request (${res.status}); check WEAVIATE_API_KEY`
|
|
2850
|
+
);
|
|
2851
|
+
}
|
|
2852
|
+
const detail = errorDetail(json) || text || res.statusText;
|
|
2853
|
+
const where = path.split("?")[0];
|
|
2854
|
+
throw new Error(
|
|
2855
|
+
this._redact(
|
|
2856
|
+
`WeaviateRetriever: ${method} ${where} failed (${res.status}): ${detail}`
|
|
2857
|
+
)
|
|
2858
|
+
);
|
|
2859
|
+
}
|
|
2860
|
+
return json;
|
|
2861
|
+
}
|
|
2862
|
+
/** A GraphQL query; resolves `data`, throws on GraphQL errors. */
|
|
2863
|
+
async _graphql(query, signal) {
|
|
2864
|
+
const json = await this._request("POST", "/graphql", { query }, { signal });
|
|
2865
|
+
if (json.errors && json.errors.length) {
|
|
2866
|
+
throw new Error(
|
|
2867
|
+
this._redact(
|
|
2868
|
+
`WeaviateRetriever: GraphQL query failed: ${errorDetail(json)}`
|
|
2869
|
+
)
|
|
2870
|
+
);
|
|
2871
|
+
}
|
|
2872
|
+
return json.data || {};
|
|
2873
|
+
}
|
|
2874
|
+
/** The collection's schema, or null when it doesn't exist. */
|
|
2875
|
+
_schema(signal) {
|
|
2876
|
+
return this._request("GET", `/schema/${this.collection}`, void 0, {
|
|
2877
|
+
signal,
|
|
2878
|
+
allow404: true
|
|
2879
|
+
});
|
|
2880
|
+
}
|
|
2881
|
+
/** Whether the collection exists; only a yes is remembered. */
|
|
2882
|
+
async _collectionExists(signal) {
|
|
2883
|
+
if (!this._exists) this._exists = Boolean(await this._schema(signal));
|
|
2884
|
+
return this._exists;
|
|
2885
|
+
}
|
|
2886
|
+
_ensureCollection() {
|
|
2887
|
+
if (!this._ready) {
|
|
2888
|
+
this._ready = (async () => {
|
|
2889
|
+
if (this.createCollection) {
|
|
2890
|
+
const schema = await this._schema();
|
|
2891
|
+
if (!schema) {
|
|
2892
|
+
try {
|
|
2893
|
+
await this._request("POST", "/schema", {
|
|
2894
|
+
class: this.collection,
|
|
2895
|
+
vectorizer: "none",
|
|
2896
|
+
vectorIndexType: "hnsw",
|
|
2897
|
+
vectorIndexConfig: { distance: "cosine" },
|
|
2898
|
+
properties: PROPERTIES
|
|
2899
|
+
});
|
|
2900
|
+
} catch (err) {
|
|
2901
|
+
if (!/already exists/i.test(err.message)) throw err;
|
|
2902
|
+
}
|
|
2903
|
+
} else {
|
|
2904
|
+
const have = new Set((schema.properties || []).map((p) => p.name));
|
|
2905
|
+
for (const property of PROPERTIES) {
|
|
2906
|
+
if (!have.has(property.name)) {
|
|
2907
|
+
await this._request(
|
|
2908
|
+
"POST",
|
|
2909
|
+
`/schema/${this.collection}/properties`,
|
|
2910
|
+
property
|
|
2911
|
+
);
|
|
2912
|
+
}
|
|
2913
|
+
}
|
|
2914
|
+
}
|
|
2915
|
+
this._exists = true;
|
|
2916
|
+
}
|
|
2917
|
+
if (!this.dimensions) {
|
|
2918
|
+
const page = await this._request(
|
|
2919
|
+
"GET",
|
|
2920
|
+
`/objects?class=${this.collection}&limit=1&include=vector`,
|
|
2921
|
+
void 0,
|
|
2922
|
+
{ allow404: true }
|
|
2923
|
+
);
|
|
2924
|
+
const vector = page && page.objects && page.objects[0] && page.objects[0].vector;
|
|
2925
|
+
if (Array.isArray(vector) && vector.length) {
|
|
2926
|
+
this.dimensions = vector.length;
|
|
2927
|
+
}
|
|
2928
|
+
}
|
|
2929
|
+
})().catch((err) => {
|
|
2930
|
+
this._ready = null;
|
|
2931
|
+
throw err;
|
|
2932
|
+
});
|
|
2933
|
+
}
|
|
2934
|
+
return this._ready;
|
|
2935
|
+
}
|
|
2936
|
+
/**
|
|
2937
|
+
* Inserts or replaces documents.
|
|
2938
|
+
* @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
|
|
2939
|
+
* @returns {Promise<void>}
|
|
2940
|
+
*/
|
|
2941
|
+
async store(docs) {
|
|
2942
|
+
if (!Array.isArray(docs))
|
|
2943
|
+
throw new Error("WeaviateRetriever.store() takes an array");
|
|
2944
|
+
if (docs.length === 0) return;
|
|
2945
|
+
let dimensions = null;
|
|
2946
|
+
const objects = docs.map((doc) => {
|
|
2947
|
+
const id = doc && doc.id;
|
|
2948
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
2949
|
+
throw new Error(
|
|
2950
|
+
"WeaviateRetriever.store(): every document needs a string id"
|
|
2951
|
+
);
|
|
2952
|
+
}
|
|
2953
|
+
checkVector(doc.vector, id);
|
|
2954
|
+
if (dimensions === null) dimensions = doc.vector.length;
|
|
2955
|
+
else if (doc.vector.length !== dimensions) {
|
|
2956
|
+
throw new Error(
|
|
2957
|
+
`WeaviateRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the batch has ${dimensions}`
|
|
2958
|
+
);
|
|
2959
|
+
}
|
|
2960
|
+
const metadata = doc.metadata || {};
|
|
2961
|
+
const properties = {
|
|
2962
|
+
rag_id: id,
|
|
2963
|
+
content: doc.content || "",
|
|
2964
|
+
rag_metadata: JSON.stringify(metadata)
|
|
2965
|
+
};
|
|
2966
|
+
if (metadata.source !== void 0 && metadata.source !== null) {
|
|
2967
|
+
properties.rag_source = String(metadata.source);
|
|
2968
|
+
}
|
|
2969
|
+
return {
|
|
2970
|
+
class: this.collection,
|
|
2971
|
+
id: pointId(id),
|
|
2972
|
+
vector: doc.vector,
|
|
2973
|
+
properties
|
|
2974
|
+
};
|
|
2975
|
+
});
|
|
2976
|
+
await this._ensureCollection();
|
|
2977
|
+
if (this.dimensions && dimensions !== this.dimensions) {
|
|
2978
|
+
throw new Error(
|
|
2979
|
+
`WeaviateRetriever.store(): the documents have ${dimensions} dimensions, the collection has ${this.dimensions}`
|
|
2980
|
+
);
|
|
2981
|
+
}
|
|
2982
|
+
for (let i = 0; i < objects.length; i += UPSERT_BATCH) {
|
|
2983
|
+
const batch = objects.slice(i, i + UPSERT_BATCH);
|
|
2984
|
+
const results = await this._request("POST", "/batch/objects", {
|
|
2985
|
+
objects: batch
|
|
2986
|
+
});
|
|
2987
|
+
for (const [j, item] of (Array.isArray(results) ? results : []).entries()) {
|
|
2988
|
+
const detail = item && item.result && errorDetail(item.result.errors);
|
|
2989
|
+
if (!detail) continue;
|
|
2990
|
+
const id = batch[j] ? batch[j].properties.rag_id : "?";
|
|
2991
|
+
const kind = /vectors? with length/i.test(detail) ? "has the wrong vector dimensions" : "was not stored";
|
|
2992
|
+
throw new Error(
|
|
2993
|
+
this._redact(
|
|
2994
|
+
`WeaviateRetriever.store(): document "${id}" ${kind}: ${detail}`
|
|
2995
|
+
)
|
|
2996
|
+
);
|
|
2997
|
+
}
|
|
2998
|
+
}
|
|
2999
|
+
this._exists = true;
|
|
3000
|
+
if (!this.dimensions) this.dimensions = dimensions;
|
|
3001
|
+
}
|
|
3002
|
+
/**
|
|
3003
|
+
* The documents most similar to a query vector.
|
|
3004
|
+
* @param {number[]|{ queryVector?: number[], topK?: number, signal?: AbortSignal }} request
|
|
3005
|
+
* A vector, or an object with one (required; optional in the type to
|
|
3006
|
+
* match the retriever contract)
|
|
3007
|
+
* @param {number} [k=10] - Results when `request` is a vector
|
|
3008
|
+
* @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
|
|
3009
|
+
*/
|
|
3010
|
+
async retrieve(request, k = 10) {
|
|
3011
|
+
const {
|
|
3012
|
+
queryVector,
|
|
3013
|
+
topK = k,
|
|
3014
|
+
signal
|
|
3015
|
+
} = Array.isArray(request) ? { queryVector: request } : request || {};
|
|
3016
|
+
if (!Number.isInteger(topK) || topK < 1) {
|
|
3017
|
+
throw new Error(
|
|
3018
|
+
"WeaviateRetriever.retrieve(): topK must be a positive integer"
|
|
3019
|
+
);
|
|
3020
|
+
}
|
|
3021
|
+
checkVector(queryVector, "query");
|
|
3022
|
+
throwIfAborted(signal);
|
|
3023
|
+
if (!await this._collectionExists(signal)) return [];
|
|
3024
|
+
const data = await this._graphql(
|
|
3025
|
+
`{ Get { ${this.collection}(nearVector: { vector: ${JSON.stringify(queryVector)} }, limit: ${topK}, where: ${OURS_GRAPHQL}) { rag_id content rag_metadata _additional { distance } } } }`,
|
|
3026
|
+
signal
|
|
3027
|
+
);
|
|
3028
|
+
throwIfAborted(signal);
|
|
3029
|
+
const hits = data.Get && data.Get[this.collection] || [];
|
|
3030
|
+
return hits.map((hit) => ({
|
|
3031
|
+
id: hit.rag_id,
|
|
3032
|
+
score: 1 - hit._additional.distance,
|
|
3033
|
+
content: hit.content || "",
|
|
3034
|
+
metadata: parseMetadata(hit.rag_metadata)
|
|
3035
|
+
}));
|
|
3036
|
+
}
|
|
3037
|
+
/**
|
|
3038
|
+
* Deletes the matching objects, a server-limited round at a time.
|
|
3039
|
+
* @private
|
|
3040
|
+
*/
|
|
3041
|
+
async _deleteWhere(where) {
|
|
3042
|
+
if (!await this._collectionExists()) return;
|
|
3043
|
+
for (; ; ) {
|
|
3044
|
+
const json = await this._request("DELETE", "/batch/objects", {
|
|
3045
|
+
match: { class: this.collection, where },
|
|
3046
|
+
output: "minimal"
|
|
3047
|
+
});
|
|
3048
|
+
const r = json && json.results || {};
|
|
3049
|
+
if (r.failed > 0) {
|
|
3050
|
+
throw new Error(
|
|
3051
|
+
`WeaviateRetriever: ${r.failed} objects could not be deleted`
|
|
3052
|
+
);
|
|
3053
|
+
}
|
|
3054
|
+
if (!(r.limit > 0 && r.matches > r.limit && r.successful > 0)) return;
|
|
3055
|
+
}
|
|
3056
|
+
}
|
|
3057
|
+
/**
|
|
3058
|
+
* Deletes the chunks of one source (`metadata.source`), for re-ingesting.
|
|
3059
|
+
* @param {string} source
|
|
3060
|
+
*/
|
|
3061
|
+
async removeBySource(source) {
|
|
3062
|
+
await this._deleteWhere({
|
|
3063
|
+
path: ["rag_source"],
|
|
3064
|
+
operator: "Equal",
|
|
3065
|
+
valueText: String(source)
|
|
3066
|
+
});
|
|
3067
|
+
}
|
|
3068
|
+
/**
|
|
3069
|
+
* Deletes documents by id.
|
|
3070
|
+
* @param {string[]} ids
|
|
3071
|
+
*/
|
|
3072
|
+
async delete(ids) {
|
|
3073
|
+
for (let i = 0; i < ids.length; i += DELETE_BATCH) {
|
|
3074
|
+
await this._deleteWhere({
|
|
3075
|
+
path: ["rag_id"],
|
|
3076
|
+
operator: "ContainsAny",
|
|
3077
|
+
valueTextArray: ids.slice(i, i + DELETE_BATCH).map(String)
|
|
3078
|
+
});
|
|
3079
|
+
}
|
|
3080
|
+
}
|
|
3081
|
+
/** Deletes every document this class stored; the collection stays. */
|
|
3082
|
+
async clear() {
|
|
3083
|
+
await this._deleteWhere(OURS);
|
|
3084
|
+
}
|
|
3085
|
+
/** @returns {Promise<number>} */
|
|
3086
|
+
async size() {
|
|
3087
|
+
if (!await this._collectionExists()) return 0;
|
|
3088
|
+
const data = await this._graphql(
|
|
3089
|
+
`{ Aggregate { ${this.collection}(where: ${OURS_GRAPHQL}) { meta { count } } } }`
|
|
3090
|
+
);
|
|
3091
|
+
const rows = data.Aggregate && data.Aggregate[this.collection] || [];
|
|
3092
|
+
return rows[0] && rows[0].meta ? rows[0].meta.count : 0;
|
|
3093
|
+
}
|
|
3094
|
+
/**
|
|
3095
|
+
* Every document, without vectors.
|
|
3096
|
+
* @param {{ limit?: number }} [options]
|
|
3097
|
+
* @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
|
|
3098
|
+
*/
|
|
3099
|
+
async list({ limit } = {}) {
|
|
3100
|
+
const out = [];
|
|
3101
|
+
if (!await this._collectionExists()) return out;
|
|
3102
|
+
let after = "";
|
|
3103
|
+
for (; ; ) {
|
|
3104
|
+
const page = await this._request(
|
|
3105
|
+
"GET",
|
|
3106
|
+
`/objects?class=${this.collection}&limit=${PAGE}&after=${encodeURIComponent(after)}`,
|
|
3107
|
+
void 0,
|
|
3108
|
+
{ allow404: true }
|
|
3109
|
+
);
|
|
3110
|
+
const objects = page && page.objects || [];
|
|
3111
|
+
for (const o of objects) {
|
|
3112
|
+
const p = o.properties || {};
|
|
3113
|
+
if (typeof p.rag_id !== "string") continue;
|
|
3114
|
+
out.push({
|
|
3115
|
+
id: p.rag_id,
|
|
3116
|
+
content: p.content || "",
|
|
3117
|
+
metadata: parseMetadata(p.rag_metadata)
|
|
3118
|
+
});
|
|
3119
|
+
if (limit && out.length >= limit) return out;
|
|
3120
|
+
}
|
|
3121
|
+
if (objects.length < PAGE) return out;
|
|
3122
|
+
after = objects[objects.length - 1].id;
|
|
3123
|
+
}
|
|
3124
|
+
}
|
|
3125
|
+
};
|
|
3126
|
+
module2.exports = { WeaviateRetriever };
|
|
3127
|
+
}
|
|
3128
|
+
});
|
|
3129
|
+
|
|
1964
3130
|
// src/adapters/ai-sdk.js
|
|
1965
3131
|
var require_ai_sdk = __commonJS({
|
|
1966
3132
|
"src/adapters/ai-sdk.js"(exports2, module2) {
|
|
@@ -2119,7 +3285,9 @@ var require_connectors = __commonJS({
|
|
|
2119
3285
|
var { OllamaConnector } = require_ollama_connector();
|
|
2120
3286
|
var { LocalEmbedder } = require_local_embedder();
|
|
2121
3287
|
var { QdrantRetriever } = require_qdrant_retriever();
|
|
3288
|
+
var { PineconeRetriever } = require_pinecone_retriever();
|
|
2122
3289
|
var { PgVectorRetriever } = require_pgvector_retriever();
|
|
3290
|
+
var { WeaviateRetriever } = require_weaviate_retriever();
|
|
2123
3291
|
var { createAiSdkAdapter } = require_ai_sdk();
|
|
2124
3292
|
module2.exports = {
|
|
2125
3293
|
OpenAIConnector,
|
|
@@ -2128,7 +3296,9 @@ var require_connectors = __commonJS({
|
|
|
2128
3296
|
OllamaConnector,
|
|
2129
3297
|
LocalEmbedder,
|
|
2130
3298
|
QdrantRetriever,
|
|
3299
|
+
PineconeRetriever,
|
|
2131
3300
|
PgVectorRetriever,
|
|
3301
|
+
WeaviateRetriever,
|
|
2132
3302
|
createAiSdkAdapter
|
|
2133
3303
|
};
|
|
2134
3304
|
}
|
|
@@ -2147,7 +3317,9 @@ var modules = {
|
|
|
2147
3317
|
"connectors/rag-prompt.js": () => require_rag_prompt(),
|
|
2148
3318
|
"packages/connectors.js": () => require_connectors(),
|
|
2149
3319
|
"retrieval/pgvector-retriever.js": () => require_pgvector_retriever(),
|
|
2150
|
-
"retrieval/
|
|
3320
|
+
"retrieval/pinecone-retriever.js": () => require_pinecone_retriever(),
|
|
3321
|
+
"retrieval/qdrant-retriever.js": () => require_qdrant_retriever(),
|
|
3322
|
+
"retrieval/weaviate-retriever.js": () => require_weaviate_retriever()
|
|
2151
3323
|
};
|
|
2152
3324
|
Object.defineProperty(mod, Symbol.for("@devilsdev/rag-connectors/internal"), {
|
|
2153
3325
|
value: (file) => {
|