@devilsdev/rag-connectors 3.5.0 → 3.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -99,7 +99,7 @@ var require_openai_connector = __commonJS({
99
99
  "src/connectors/openai-connector.js"(exports2, module2) {
100
100
  "use strict";
101
101
  var { BaseConnector } = require_base_connector();
102
- var { loadOptionalPeer } = require_optional_peer();
102
+ var { missingPeer } = require_optional_peer();
103
103
  var {
104
104
  missingApiKeyError,
105
105
  raceAbort,
@@ -121,12 +121,15 @@ var require_openai_connector = __commonJS({
121
121
  }
122
122
  };
123
123
  function loadOpenAI() {
124
- return loadOptionalPeer(
125
- "openai",
126
- () => require("openai"),
127
- // eslint-disable-line global-require
128
- 'OpenAIConnector needs the "openai" package, an optional peer dependency since 3.0.0: npm install openai'
129
- );
124
+ try {
125
+ return require("openai");
126
+ } catch (error) {
127
+ throw missingPeer(
128
+ error,
129
+ "openai",
130
+ 'OpenAIConnector needs the "openai" package, an optional peer dependency since 3.0.0: npm install openai'
131
+ );
132
+ }
130
133
  }
131
134
  var OpenAIConnector = class extends BaseConnector {
132
135
  /**
@@ -230,8 +233,25 @@ var require_openai_connector = __commonJS({
230
233
  _request(signal, fn) {
231
234
  return this.withRetry(fn, this.config.maxRetries, this.config.retryDelay, {
232
235
  signal
236
+ }).catch((err) => {
237
+ throw this._tagError(err);
233
238
  });
234
239
  }
240
+ /**
241
+ * Name the provider on an HTTP error from the SDK, as the axios-based
242
+ * connectors do, so a caller such as the CLI can say which key was refused.
243
+ * A custom baseURL never reads OPENAI_API_KEY, so it gets no envVar.
244
+ * @private
245
+ */
246
+ _tagError(err) {
247
+ if (err && typeof err === "object" && typeof err.status === "number") {
248
+ if (!err.provider) {
249
+ err.provider = this.config.baseURL ? "The OpenAI-compatible API" : "OpenAI";
250
+ }
251
+ if (!this.config.baseURL && !err.envVar) err.envVar = API_KEY_ENV;
252
+ }
253
+ return err;
254
+ }
235
255
  /**
236
256
  * Generate a chat completion using the provided query and optional RAG context.
237
257
  *
@@ -275,19 +295,24 @@ var require_openai_connector = __commonJS({
275
295
  this._requireApiKey();
276
296
  const client = this._getClient();
277
297
  const messages = this._buildMessages(query, context);
278
- const stream = await raceAbort(
279
- client.chat.completions.create(
280
- {
281
- model: options.model || this.config.model,
282
- messages,
283
- temperature: options.temperature ?? this.config.temperature,
284
- max_tokens: options.maxTokens ?? this.config.maxTokens,
285
- stream: true
286
- },
287
- signal ? { signal } : void 0
288
- ),
289
- signal
290
- );
298
+ let stream;
299
+ try {
300
+ stream = await raceAbort(
301
+ client.chat.completions.create(
302
+ {
303
+ model: options.model || this.config.model,
304
+ messages,
305
+ temperature: options.temperature ?? this.config.temperature,
306
+ max_tokens: options.maxTokens ?? this.config.maxTokens,
307
+ stream: true
308
+ },
309
+ signal ? { signal } : void 0
310
+ ),
311
+ signal
312
+ );
313
+ } catch (err) {
314
+ throw this._tagError(err);
315
+ }
291
316
  for await (const chunk of stream) {
292
317
  throwIfAborted(signal);
293
318
  const token = chunk.choices[0]?.delta?.content || "";
@@ -1608,220 +1633,914 @@ var require_qdrant_retriever = __commonJS({
1608
1633
  }
1609
1634
  });
1610
1635
 
1611
- // src/retrieval/pgvector-retriever.js
1612
- var require_pgvector_retriever = __commonJS({
1613
- "src/retrieval/pgvector-retriever.js"(exports2, module2) {
1636
+ // src/retrieval/pinecone-retriever.js
1637
+ var require_pinecone_retriever = __commonJS({
1638
+ "src/retrieval/pinecone-retriever.js"(exports2, module2) {
1614
1639
  "use strict";
1615
- var { throwIfAborted, raceAbort } = require_connector_utils();
1616
- var IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
1617
- var DEFAULT_TABLE = "rag_documents";
1618
- var PG_DEFAULT_EF_SEARCH = 40;
1619
- var MAX_BATCH = 1e3;
1620
- function quoteTable(name) {
1621
- const parts = String(name).split(".");
1622
- if (parts.length > 2 || !parts.every((p) => IDENTIFIER.test(p))) {
1623
- throw new Error(
1624
- `PgVectorRetriever: invalid table name "${name}" (letters, digits and underscores; optionally schema.table)`
1625
- );
1626
- }
1627
- return parts.map((p) => `"${p}"`).join(".");
1640
+ var crypto = require("crypto");
1641
+ var { throwIfAborted } = require_connector_utils();
1642
+ var API_VERSION = "2026-07";
1643
+ var CONTROL_PLANE = "https://api.pinecone.io";
1644
+ var DEFAULT_NAMESPACE = "rag_documents";
1645
+ var UPSERT_BATCH = 100;
1646
+ var EMBED_UPSERT_BATCH = 96;
1647
+ var REQUEST_BYTES = 2e6 - 1024;
1648
+ var DELETE_BATCH = 1e3;
1649
+ var PAGE = 100;
1650
+ var SEARCH_MAX = 1e4;
1651
+ var PLAIN_ID = /^[\x20-\x7e]{1,512}$/;
1652
+ var FIELDS = ["rag_id", "rag_content", "rag_metadata"];
1653
+ var OURS = { rag_id: { $exists: true } };
1654
+ var PROBE_QUERY = "document";
1655
+ function recordId(id) {
1656
+ const s = String(id);
1657
+ if (PLAIN_ID.test(s)) return s;
1658
+ return `rag-${crypto.createHash("sha256").update(s).digest("hex")}`;
1628
1659
  }
1629
- function vectorLiteral(vector, id) {
1660
+ function checkVector(vector, id) {
1630
1661
  if (!Array.isArray(vector) || vector.length === 0) {
1631
- throw new Error(`PgVectorRetriever: document "${id}" has no vector`);
1632
- }
1633
- for (const x of vector) {
1634
- if (typeof x !== "number" || !Number.isFinite(x)) {
1635
- throw new Error(
1636
- `PgVectorRetriever: document "${id}" has a vector with a non-finite value`
1637
- );
1638
- }
1639
- }
1640
- return `[${vector.join(",")}]`;
1641
- }
1642
- function explain(err, table) {
1643
- const code = err && err.code;
1644
- if (code === "42P01") {
1645
- return new Error(
1646
- `PgVectorRetriever: table ${table} does not exist. Store documents first (rag-pipeline ingest), or create it.`
1647
- );
1662
+ throw new Error(`PineconeRetriever: document "${id}" has no vector`);
1648
1663
  }
1649
- if (code === "42501" || /permission denied/i.test(String(err && err.message))) {
1650
- return new Error(
1651
- `PgVectorRetriever: permission denied. Run \`CREATE EXTENSION IF NOT EXISTS vector;\` as a database owner, and grant this user access to ${table}. (${err.message})`
1664
+ if (!vector.every((x) => typeof x === "number" && Number.isFinite(x))) {
1665
+ throw new Error(
1666
+ `PineconeRetriever: document "${id}" has a vector with a non-finite value`
1652
1667
  );
1653
1668
  }
1654
- if (/different vector dimensions|expected \d+ dimensions/i.test(
1655
- String(err && err.message)
1656
- )) {
1657
- return new Error(
1658
- `PgVectorRetriever: ${err.message}. The table holds vectors from another embedder; use the same embedder or a new table.`
1659
- );
1669
+ }
1670
+ function fromFields(fields, fallbackId) {
1671
+ const m = fields || {};
1672
+ let metadata = {};
1673
+ try {
1674
+ metadata = m.rag_metadata ? JSON.parse(m.rag_metadata) : {};
1675
+ } catch {
1676
+ metadata = {};
1660
1677
  }
1661
- return err;
1678
+ return {
1679
+ id: m.rag_id !== void 0 ? String(m.rag_id) : fallbackId,
1680
+ content: m.rag_content || "",
1681
+ metadata
1682
+ };
1662
1683
  }
1663
- var PgVectorRetriever = class {
1684
+ function normalizeHost(host) {
1685
+ let h = String(host);
1686
+ while (h.endsWith("/")) h = h.slice(0, -1);
1687
+ return /^https?:\/\//.test(h) ? h : `https://${h}`;
1688
+ }
1689
+ var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
1690
+ var PineconeRetriever = class {
1664
1691
  /**
1665
1692
  * @param {Object} options
1666
- * @param {{ query: Function, connect?: Function }} [options.pool] - A pg Pool or Client you manage
1667
- * @param {string} [options.connectionString] - Used to create a Pool when no `pool` is given
1668
- * @param {string} [options.table='rag_documents'] - Table name, optionally `schema.table`
1669
- * @param {number} [options.dimensions] - Vector size for a new table; defaults to the first stored vector's
1670
- * @param {boolean} [options.createTable=true] - Create the extension, table and index on first store
1671
- * @param {'hnsw'|'none'} [options.index='hnsw'] - Index created with the table
1672
- * @param {number} [options.efSearch] - HNSW candidate list size per query (pgvector's default is 40). Higher finds more of the true nearest neighbours, more slowly; a query always uses at least its topK
1673
- * @param {string} [options.cacheKey] - Cache identity; change it when the table changes (see getCacheKey)
1693
+ * @param {string} [options.apiKey] - Defaults to PINECONE_API_KEY
1694
+ * @param {string} [options.index] - The index name, described on first use
1695
+ * @param {string} [options.host] - The index host (from the console), instead of or with `index`
1696
+ * @param {string} [options.namespace='rag_documents'] - A namespace this retriever owns
1697
+ * @param {'auto'|'documents'|'vectors'} [options.api='auto'] - Document index (Documents API) or vector index; auto reads it from the index description
1698
+ * @param {string} [options.vectorField] - The document index's dense_vector field (default: the schema's)
1699
+ * @param {string} [options.textField] - The document index's text field that Pinecone embeds (integrated embedding; default: the schema's, when the index has no dense_vector field)
1700
+ * @param {number} [options.freshnessTimeoutMs=30000] - How long a read waits for this retriever's writes to be indexed; 0 to not wait
1701
+ * @param {string} [options.cacheKey] - Cache identity; change it when the index changes
1702
+ * @param {Function} [options.fetch] - fetch implementation (defaults to the global)
1674
1703
  */
1675
1704
  constructor(options = {}) {
1676
1705
  const {
1677
- pool,
1678
- connectionString,
1679
- table = DEFAULT_TABLE,
1680
- dimensions,
1681
- createTable = true,
1682
- index = "hnsw",
1683
- efSearch,
1684
- cacheKey
1706
+ apiKey = typeof process !== "undefined" && process.env ? process.env.PINECONE_API_KEY : void 0,
1707
+ host,
1708
+ index,
1709
+ namespace = DEFAULT_NAMESPACE,
1710
+ api = "auto",
1711
+ vectorField,
1712
+ textField,
1713
+ freshnessTimeoutMs = 3e4,
1714
+ cacheKey,
1715
+ fetch: fetchImpl = globalThis.fetch
1685
1716
  } = options;
1686
- if (!pool && !connectionString) {
1717
+ if (!apiKey) {
1687
1718
  throw new Error(
1688
- "PgVectorRetriever needs a pg `pool` (or Client) or a `connectionString`"
1719
+ "PineconeRetriever needs an API key: pass apiKey or set PINECONE_API_KEY"
1689
1720
  );
1690
1721
  }
1691
- if (pool && typeof pool.query !== "function") {
1692
- throw new Error("PgVectorRetriever: `pool` must have a query() method");
1722
+ if (!host && !index) {
1723
+ throw new Error(
1724
+ "PineconeRetriever needs the index: pass index (its name) or host (the index host)"
1725
+ );
1693
1726
  }
1694
- if (!["hnsw", "none"].includes(index)) {
1727
+ if (typeof namespace !== "string") {
1728
+ throw new Error("PineconeRetriever: namespace must be a string");
1729
+ }
1730
+ if (!["auto", "documents", "vectors"].includes(api)) {
1695
1731
  throw new Error(
1696
- `PgVectorRetriever: unknown index "${index}" (use "hnsw" or "none")`
1732
+ 'PineconeRetriever: api must be "auto", "documents" or "vectors"'
1697
1733
  );
1698
1734
  }
1699
- if (dimensions !== void 0 && !(Number.isInteger(dimensions) && dimensions > 0)) {
1735
+ if (vectorField && textField) {
1700
1736
  throw new Error(
1701
- "PgVectorRetriever: `dimensions` must be a positive integer"
1737
+ "PineconeRetriever: pass vectorField (your vectors) or textField (Pinecone embeds the text), not both"
1702
1738
  );
1703
1739
  }
1704
- if (efSearch !== void 0 && !(Number.isInteger(efSearch) && efSearch > 0 && efSearch <= 1e3)) {
1740
+ if (textField && api === "vectors") {
1705
1741
  throw new Error(
1706
- "PgVectorRetriever: `efSearch` must be an integer from 1 to 1000"
1742
+ 'PineconeRetriever: textField needs a document index (api "documents" or "auto")'
1707
1743
  );
1708
1744
  }
1709
- this.efSearch = efSearch;
1710
- this.table = quoteTable(table);
1711
- this.tableName = table;
1712
- this.dimensions = dimensions;
1713
- this.createTable = createTable;
1745
+ if (typeof fetchImpl !== "function") {
1746
+ throw new Error("PineconeRetriever needs fetch (Node.js 18 or later)");
1747
+ }
1714
1748
  this.index = index;
1749
+ this.namespace = namespace;
1750
+ this.freshnessTimeoutMs = freshnessTimeoutMs;
1715
1751
  this.cacheKey = cacheKey;
1716
- this._pool = pool || null;
1717
- this._connectionString = connectionString;
1718
- this._ownsPool = !pool;
1719
- this._ready = null;
1752
+ this._apiKey = apiKey;
1753
+ this._host = host ? normalizeHost(host) : null;
1754
+ this._api = api;
1755
+ this._vectorField = vectorField;
1756
+ this._textField = textField;
1757
+ this._fetch = fetchImpl === globalThis.fetch ? fetchImpl.bind(globalThis) : fetchImpl;
1758
+ this._resolving = null;
1759
+ this._writeLsn = 0;
1760
+ this._dimension = null;
1720
1761
  }
1721
- /** The pg Pool or Client in use; creates the Pool on first use. */
1722
- _db() {
1723
- if (!this._pool) {
1724
- let pg;
1725
- try {
1726
- pg = require("pg");
1727
- } catch {
1728
- throw new Error(
1729
- 'PgVectorRetriever with a connectionString needs the "pg" package: npm install pg'
1730
- );
1731
- }
1732
- this._pool = new pg.Pool({
1733
- connectionString: this._connectionString,
1734
- allowExitOnIdle: true
1735
- });
1736
- }
1737
- return this._pool;
1762
+ /**
1763
+ * Cache identity for createRagPipeline({ cache }): none unless you set
1764
+ * `cacheKey`, since other clients can change the index.
1765
+ * @param {'retriever'} role
1766
+ * @returns {string|null}
1767
+ */
1768
+ getCacheKey(role) {
1769
+ if (role !== "retriever" || !this.cacheKey) return null;
1770
+ return `pinecone|${this._host || this.index}|${this.namespace}|${this.cacheKey}`;
1738
1771
  }
1739
- async _query(text, params, signal) {
1772
+ /**
1773
+ * Whether the index embeds text itself (integrated embedding): then
1774
+ * store() needs no vectors and retrieve() takes the query text, so no
1775
+ * embedder is needed. The CLI checks this before requiring an embedder.
1776
+ * Reads the index description on first use.
1777
+ * @param {{ signal?: AbortSignal }} [options]
1778
+ * @returns {Promise<boolean>}
1779
+ */
1780
+ async embedsText({ signal } = {}) {
1781
+ await this._resolve(signal);
1782
+ return this._integrated;
1783
+ }
1784
+ /**
1785
+ * One REST call; resolves the parsed body and the response headers.
1786
+ * @private
1787
+ */
1788
+ async _call(url, method, body, signal) {
1740
1789
  throwIfAborted(signal);
1790
+ const headers = {
1791
+ "Api-Key": this._apiKey,
1792
+ "X-Pinecone-Api-Version": API_VERSION,
1793
+ accept: "application/json"
1794
+ };
1795
+ if (body !== void 0) headers["content-type"] = "application/json";
1796
+ let res;
1741
1797
  try {
1742
- return await raceAbort(this._db().query(text, params), signal);
1798
+ res = await this._fetch(url, {
1799
+ method,
1800
+ headers,
1801
+ ...body !== void 0 && { body: JSON.stringify(body) },
1802
+ ...signal && { signal }
1803
+ });
1743
1804
  } catch (err) {
1744
- throw explain(err, this.table);
1805
+ throwIfAborted(signal);
1806
+ throw new Error(
1807
+ `PineconeRetriever: cannot reach Pinecone (${err.message})`
1808
+ );
1745
1809
  }
1746
- }
1747
- /** Runs one query with `SET LOCAL hnsw.ef_search`, in its own transaction. */
1748
- async _queryWithEfSearch(text, params, ef, signal) {
1749
- throwIfAborted(signal);
1750
- const db = this._db();
1751
- const pooled = typeof db.connect === "function" && typeof db.end === "function" && db.totalCount !== void 0;
1752
- const client = pooled ? await db.connect() : db;
1810
+ const text = await res.text();
1811
+ let json;
1753
1812
  try {
1754
- await client.query("BEGIN");
1755
- await client.query(`SET LOCAL hnsw.ef_search = ${Math.min(ef, 1e3)}`);
1756
- const result = await client.query(text, params);
1757
- await client.query("COMMIT");
1758
- return result;
1759
- } catch (err) {
1760
- await client.query("ROLLBACK").catch(() => {
1761
- });
1762
- throw explain(err, this.table);
1763
- } finally {
1764
- if (pooled) client.release();
1813
+ json = text ? JSON.parse(text) : {};
1814
+ } catch {
1815
+ json = {};
1765
1816
  }
1766
- }
1767
- /** Creates the extension, table and index once, sized to `dimensions`. */
1768
- _ensureTable(dimensions) {
1769
- if (!this.createTable) return Promise.resolve();
1770
- if (!this._ready) {
1771
- const size = this.dimensions || dimensions;
1772
- const indexName = `"${this.tableName.split(".").pop()}_embedding_hnsw"`;
1773
- this._ready = (async () => {
1774
- await this._query("CREATE EXTENSION IF NOT EXISTS vector");
1775
- await this._query(
1776
- `CREATE TABLE IF NOT EXISTS ${this.table} (id text PRIMARY KEY, content text NOT NULL DEFAULT '', metadata jsonb NOT NULL DEFAULT '{}'::jsonb, embedding vector(${size}) NOT NULL)`
1817
+ if (!res.ok) {
1818
+ if (res.status === 401 || res.status === 403) {
1819
+ throw new Error(
1820
+ `PineconeRetriever: Pinecone refused the request (${res.status}); check PINECONE_API_KEY`
1777
1821
  );
1778
- if (this.index === "hnsw") {
1779
- await this._query(
1780
- `CREATE INDEX IF NOT EXISTS ${indexName} ON ${this.table} USING hnsw (embedding vector_cosine_ops)`
1781
- );
1782
- }
1783
- })().catch((err) => {
1784
- this._ready = null;
1785
- throw err;
1786
- });
1822
+ }
1823
+ const detail = json.message || json.error && json.error.message || text || res.statusText;
1824
+ const err = new Error(
1825
+ `PineconeRetriever: ${method} ${new URL(url).pathname} failed (${res.status}): ${detail}`
1826
+ );
1827
+ err.status = res.status;
1828
+ throw err;
1787
1829
  }
1788
- return this._ready;
1830
+ return { json, headers: res.headers };
1789
1831
  }
1790
- /**
1791
- * Cache identity for createRagPipeline({ cache }). Other processes can
1792
- * write to the table without this one knowing, so there is none unless
1793
- * you set `cacheKey`: then cached retrieval results are reused until you
1794
- * change the key or their time to live ends.
1795
- * @param {'retriever'} role
1796
- * @returns {string|null}
1797
- */
1798
- getCacheKey(role) {
1799
- if (role !== "retriever" || !this.cacheKey) return null;
1800
- return `pgvector|${this.tableName}|${this.cacheKey}`;
1832
+ /** True on a document index that embeds text itself. @private */
1833
+ get _integrated() {
1834
+ return this._api === "documents" && Boolean(this._textField);
1801
1835
  }
1802
1836
  /**
1803
- * Inserts or replaces documents, in one transaction: a failed batch
1804
- * changes nothing.
1805
- * @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
1806
- * @returns {Promise<void>}
1837
+ * The index's host and kind, from its description (once): a document
1838
+ * index's dense_vector field and dimension or the text field it embeds,
1839
+ * or a vector index's dimension.
1840
+ * @private
1807
1841
  */
1808
- async store(docs) {
1809
- if (!Array.isArray(docs))
1810
- throw new Error("PgVectorRetriever.store() takes an array");
1811
- if (docs.length === 0) return;
1812
- let dimensions = null;
1813
- const rows = docs.map((doc) => {
1814
- const id = doc && doc.id;
1815
- if (typeof id !== "string" || id.length === 0) {
1842
+ _resolve(signal) {
1843
+ if (this._host && this._textField) {
1844
+ this._api = "documents";
1845
+ return Promise.resolve();
1846
+ }
1847
+ if (this._host && this._api !== "auto" && (this._api === "vectors" || this._vectorField)) {
1848
+ return Promise.resolve();
1849
+ }
1850
+ if (!this._resolving) {
1851
+ this._resolving = this._describe(signal).catch((err) => {
1852
+ this._resolving = null;
1853
+ throw err;
1854
+ });
1855
+ }
1856
+ return this._resolving;
1857
+ }
1858
+ /** @private */
1859
+ async _describe(signal) {
1860
+ let info;
1861
+ if (this.index) {
1862
+ ({ json: info } = await this._call(
1863
+ `${CONTROL_PLANE}/indexes/${encodeURIComponent(this.index)}`,
1864
+ "GET",
1865
+ void 0,
1866
+ signal
1867
+ ));
1868
+ } else {
1869
+ const { json } = await this._call(
1870
+ `${CONTROL_PLANE}/indexes`,
1871
+ "GET",
1872
+ void 0,
1873
+ signal
1874
+ );
1875
+ const host = this._host.replace(/^https?:\/\//, "");
1876
+ info = (json.indexes || []).find((i) => i.host === host);
1877
+ if (!info) {
1816
1878
  throw new Error(
1817
- "PgVectorRetriever.store(): every document needs a string id"
1879
+ `PineconeRetriever: no index with host ${host} in this project; check the host, or pass index (its name)`
1818
1880
  );
1819
1881
  }
1820
- const literal = vectorLiteral(doc.vector, id);
1821
- if (dimensions === null) dimensions = doc.vector.length;
1822
- else if (doc.vector.length !== dimensions) {
1882
+ }
1883
+ if (!info.host) {
1884
+ throw new Error(
1885
+ `PineconeRetriever: index "${info.name || this.index}" has no host yet (is it ready?)`
1886
+ );
1887
+ }
1888
+ this._host = this._host || normalizeHost(info.host);
1889
+ const fields = info.schema && info.schema.fields || null;
1890
+ const named = fields ? Object.entries(fields).filter(([name]) => !name.startsWith("_")) : [];
1891
+ const denseFields = named.filter(([, f]) => f && f.type === "dense_vector");
1892
+ const textFields = named.filter(
1893
+ ([, f]) => f && f.type === "string" && f.embed
1894
+ );
1895
+ if (this._api === "auto") {
1896
+ this._api = named.length > 0 ? "documents" : "vectors";
1897
+ }
1898
+ if (this._textField) {
1899
+ if (this._api !== "documents" || !textFields.some(([name]) => name === this._textField)) {
1823
1900
  throw new Error(
1824
- `PgVectorRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the batch has ${dimensions}`
1901
+ `PineconeRetriever: index "${info.name}" has no text field named "${this._textField}" that it embeds`
1902
+ );
1903
+ }
1904
+ } else if (this._api === "documents") {
1905
+ const field = this._vectorField ? denseFields.find(([name]) => name === this._vectorField) : denseFields[0];
1906
+ if (!field && !this._vectorField && textFields.length > 0) {
1907
+ this._textField = textFields[0][0];
1908
+ return;
1909
+ }
1910
+ if (!field) {
1911
+ throw new Error(
1912
+ `PineconeRetriever: index "${info.name}" has no dense_vector field${this._vectorField ? ` named "${this._vectorField}"` : ""}`
1913
+ );
1914
+ }
1915
+ this._vectorField = field[0];
1916
+ this._indexDimension = field[1].dimension;
1917
+ } else {
1918
+ const values = fields && fields._values;
1919
+ this._indexDimension = info.dimension || values && values.dimension;
1920
+ }
1921
+ }
1922
+ /** A data-plane call; records the log sequence number of writes. @private */
1923
+ async _data(method, path, body, { signal, write = false } = {}) {
1924
+ await this._resolve(signal);
1925
+ const result = await this._call(
1926
+ `${this._host}${path}`,
1927
+ method,
1928
+ body,
1929
+ signal
1930
+ );
1931
+ if (write) {
1932
+ const lsn = Number(result.headers.get("x-pinecone-request-lsn"));
1933
+ if (Number.isFinite(lsn) && lsn > this._writeLsn) this._writeLsn = lsn;
1934
+ }
1935
+ return result;
1936
+ }
1937
+ /** The Documents API path for this namespace. @private */
1938
+ _docs(action) {
1939
+ const ns = encodeURIComponent(this.namespace || "__default__");
1940
+ return `/namespaces/${ns}/documents/${action}`;
1941
+ }
1942
+ /**
1943
+ * A search, repeated until it reflects this retriever's writes (when the
1944
+ * index reports log sequence numbers) or `freshnessTimeoutMs` passes.
1945
+ * `query` is a vector, or the query text on an index that embeds text
1946
+ * itself. Resolves `[{ id, score, fields }]`.
1947
+ * @private
1948
+ */
1949
+ async _search(query, topK, signal, filter) {
1950
+ await this._resolve(signal);
1951
+ const documents = this._api === "documents";
1952
+ const [path, body] = documents ? [
1953
+ this._docs("search"),
1954
+ {
1955
+ score_by: [
1956
+ this._integrated ? { type: "embed", fields: [this._textField], query } : {
1957
+ type: "dense_vector",
1958
+ fields: [this._vectorField],
1959
+ values: query
1960
+ }
1961
+ ],
1962
+ top_k: topK,
1963
+ include_fields: FIELDS,
1964
+ ...filter && { filter }
1965
+ }
1966
+ ] : [
1967
+ "/query",
1968
+ {
1969
+ vector: query,
1970
+ topK,
1971
+ namespace: this.namespace,
1972
+ includeMetadata: true,
1973
+ ...filter && { filter }
1974
+ }
1975
+ ];
1976
+ const deadline = Date.now() + this.freshnessTimeoutMs;
1977
+ for (let delay = 100; ; delay = Math.min(delay * 2, 2e3)) {
1978
+ let json;
1979
+ let headers;
1980
+ try {
1981
+ ({ json, headers } = await this._data("POST", path, body, { signal }));
1982
+ } catch (err) {
1983
+ if (err.status === 404) return [];
1984
+ throw err;
1985
+ }
1986
+ const header = headers.get("x-pinecone-max-indexed-lsn");
1987
+ const fresh = !this._writeLsn || header === null || !(Number(header) < this._writeLsn) || Date.now() + delay > deadline;
1988
+ if (fresh) {
1989
+ return (json.matches || []).map(
1990
+ (m) => documents ? { id: m._id, score: m._score, fields: m } : { id: m.id, score: m.score, fields: m.metadata }
1991
+ );
1992
+ }
1993
+ await sleep(delay);
1994
+ throwIfAborted(signal);
1995
+ }
1996
+ }
1997
+ /** Waits until this retriever's writes are indexed (for list and size). @private */
1998
+ async _waitForWrites(signal) {
1999
+ if (!this._writeLsn) return;
2000
+ const probe = this._probe();
2001
+ if (probe) await this._search(probe, 1, signal);
2002
+ }
2003
+ /**
2004
+ * What list() and size() search with: a unit vector of the index's
2005
+ * dimension, or a fixed text on an index that embeds text itself.
2006
+ * @private
2007
+ */
2008
+ _probe() {
2009
+ if (this._integrated) return PROBE_QUERY;
2010
+ const dimension = this._indexDimension || this._dimension;
2011
+ if (!dimension) return null;
2012
+ return new Array(dimension).fill(0).map((_, i) => i === 0 ? 1 : 0);
2013
+ }
2014
+ /**
2015
+ * Inserts or replaces documents. On an index that embeds text itself,
2016
+ * Pinecone embeds `content` (required there) and `vector` is ignored.
2017
+ * @param {Array<{ id: string, vector?: number[], content?: string, metadata?: Object }>} docs
2018
+ * `vector` is required unless the index embeds text itself
2019
+ * @returns {Promise<void>}
2020
+ */
2021
+ async store(docs) {
2022
+ if (!Array.isArray(docs)) {
2023
+ throw new Error("PineconeRetriever.store() takes an array");
2024
+ }
2025
+ if (docs.length === 0) return;
2026
+ await this._resolve();
2027
+ if (this._integrated) return this._storeText(docs);
2028
+ const expected = this._indexDimension || this._dimension;
2029
+ let dimensions = null;
2030
+ const records = docs.map((doc) => {
2031
+ const id = doc && doc.id;
2032
+ if (typeof id !== "string" || id.length === 0) {
2033
+ throw new Error(
2034
+ "PineconeRetriever.store(): every document needs a string id"
2035
+ );
2036
+ }
2037
+ checkVector(doc.vector, id);
2038
+ if (dimensions === null) dimensions = doc.vector.length;
2039
+ else if (doc.vector.length !== dimensions) {
2040
+ throw new Error(
2041
+ `PineconeRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the batch has ${dimensions}`
2042
+ );
2043
+ }
2044
+ if (expected && doc.vector.length !== expected) {
2045
+ throw new Error(
2046
+ `PineconeRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the index has ${expected}`
2047
+ );
2048
+ }
2049
+ const metadata = doc.metadata || {};
2050
+ return {
2051
+ id: recordId(id),
2052
+ values: doc.vector,
2053
+ fields: {
2054
+ rag_id: id,
2055
+ rag_content: doc.content || "",
2056
+ rag_metadata: JSON.stringify(metadata),
2057
+ ...metadata.source !== void 0 && metadata.source !== null && {
2058
+ rag_source: String(metadata.source)
2059
+ }
2060
+ }
2061
+ };
2062
+ });
2063
+ for (let i = 0; i < records.length; i += UPSERT_BATCH) {
2064
+ const batch = records.slice(i, i + UPSERT_BATCH);
2065
+ if (this._api === "documents") {
2066
+ await this._data(
2067
+ "POST",
2068
+ this._docs("upsert"),
2069
+ {
2070
+ documents: batch.map((r) => ({
2071
+ _id: r.id,
2072
+ [this._vectorField]: r.values,
2073
+ ...r.fields
2074
+ }))
2075
+ },
2076
+ { write: true }
2077
+ );
2078
+ } else {
2079
+ await this._data(
2080
+ "POST",
2081
+ "/vectors/upsert",
2082
+ {
2083
+ vectors: batch.map((r) => ({
2084
+ id: r.id,
2085
+ values: r.values,
2086
+ metadata: r.fields
2087
+ })),
2088
+ namespace: this.namespace
2089
+ },
2090
+ { write: true }
2091
+ );
2092
+ }
2093
+ }
2094
+ this._dimension = dimensions;
2095
+ }
2096
+ /** store() on an index that embeds text itself. @private */
2097
+ async _storeText(docs) {
2098
+ const records = docs.map((doc) => {
2099
+ const id = doc && doc.id;
2100
+ if (typeof id !== "string" || id.length === 0) {
2101
+ throw new Error(
2102
+ "PineconeRetriever.store(): every document needs a string id"
2103
+ );
2104
+ }
2105
+ if (typeof doc.content !== "string" || doc.content.trim() === "") {
2106
+ throw new Error(
2107
+ `PineconeRetriever.store(): document "${id}" has no content for Pinecone to embed (the index embeds ${this._textField})`
2108
+ );
2109
+ }
2110
+ const metadata = doc.metadata || {};
2111
+ return {
2112
+ _id: recordId(id),
2113
+ [this._textField]: doc.content,
2114
+ rag_id: id,
2115
+ rag_content: doc.content,
2116
+ rag_metadata: JSON.stringify(metadata),
2117
+ ...metadata.source !== void 0 && metadata.source !== null && {
2118
+ rag_source: String(metadata.source)
2119
+ }
2120
+ };
2121
+ });
2122
+ const encoder = new TextEncoder();
2123
+ let batch = [];
2124
+ let bytes = 0;
2125
+ const send = async () => {
2126
+ if (batch.length === 0) return;
2127
+ await this._data(
2128
+ "POST",
2129
+ this._docs("upsert"),
2130
+ { documents: batch },
2131
+ { write: true }
2132
+ );
2133
+ batch = [];
2134
+ bytes = 0;
2135
+ };
2136
+ for (const record of records) {
2137
+ const size = encoder.encode(JSON.stringify(record)).length + 1;
2138
+ if (batch.length === EMBED_UPSERT_BATCH || batch.length > 0 && bytes + size > REQUEST_BYTES) {
2139
+ await send();
2140
+ }
2141
+ batch.push(record);
2142
+ bytes += size;
2143
+ }
2144
+ await send();
2145
+ }
2146
+ /**
2147
+ * The documents most similar to a query vector, or to the query text on
2148
+ * an index that embeds text itself (Pinecone embeds the query; a
2149
+ * `queryVector` is then ignored).
2150
+ * @param {number[]|string|{ query?: string, queryVector?: number[], topK?: number, signal?: AbortSignal }} request
2151
+ * A vector, the query text, or an object with them: `queryVector`, or
2152
+ * `query` on an index that embeds text itself (required; optional in
2153
+ * the type to match the retriever contract)
2154
+ * @param {number} [k=10] - Results when `request` is a vector or text
2155
+ * @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
2156
+ */
2157
+ async retrieve(request, k = 10) {
2158
+ let input = request || {};
2159
+ if (Array.isArray(request)) input = { queryVector: request };
2160
+ else if (typeof request === "string") input = { query: request };
2161
+ const { query, queryVector, topK = k, signal } = input;
2162
+ if (!Number.isInteger(topK) || topK < 1) {
2163
+ throw new Error(
2164
+ "PineconeRetriever.retrieve(): topK must be a positive integer"
2165
+ );
2166
+ }
2167
+ throwIfAborted(signal);
2168
+ await this._resolve(signal);
2169
+ if (!this._integrated) {
2170
+ if (queryVector === void 0 || queryVector === null) {
2171
+ throw new Error(
2172
+ "PineconeRetriever.retrieve(): needs a queryVector; this index does not embed text itself, so use an embedder"
2173
+ );
2174
+ }
2175
+ checkVector(queryVector, "query");
2176
+ } else if (typeof query !== "string" || query.trim() === "") {
2177
+ throw new Error(
2178
+ `PineconeRetriever.retrieve(): the index embeds text itself (integrated embedding, field ${this._textField}), so it needs the query text: pass { query }`
2179
+ );
2180
+ }
2181
+ const matches = await this._search(
2182
+ this._integrated ? query : queryVector,
2183
+ Math.min(topK, 1e4),
2184
+ signal
2185
+ );
2186
+ throwIfAborted(signal);
2187
+ const seen = /* @__PURE__ */ new Set();
2188
+ const out = [];
2189
+ for (const m of matches) {
2190
+ if (seen.has(m.id)) continue;
2191
+ seen.add(m.id);
2192
+ out.push({ ...fromFields(m.fields, m.id), score: m.score });
2193
+ }
2194
+ return out;
2195
+ }
2196
+ /** @private */
2197
+ async _delete(body) {
2198
+ await this._resolve();
2199
+ try {
2200
+ if (this._api === "documents") {
2201
+ await this._data("POST", this._docs("delete"), body.documents, {
2202
+ write: true
2203
+ });
2204
+ } else {
2205
+ await this._data(
2206
+ "POST",
2207
+ "/vectors/delete",
2208
+ { namespace: this.namespace, ...body.vectors },
2209
+ { write: true }
2210
+ );
2211
+ }
2212
+ } catch (err) {
2213
+ if (err.status !== 404) throw err;
2214
+ }
2215
+ }
2216
+ /**
2217
+ * Deletes the chunks of one source (`metadata.source`), for re-ingesting.
2218
+ * @param {string} source
2219
+ */
2220
+ async removeBySource(source) {
2221
+ const filter = { rag_source: { $eq: String(source) } };
2222
+ await this._delete({ documents: { filter }, vectors: { filter } });
2223
+ }
2224
+ /**
2225
+ * Deletes documents by id.
2226
+ * @param {string[]} ids
2227
+ */
2228
+ async delete(ids) {
2229
+ const recordIds = ids.map(recordId);
2230
+ for (let i = 0; i < recordIds.length; i += DELETE_BATCH) {
2231
+ const batch = recordIds.slice(i, i + DELETE_BATCH);
2232
+ await this._delete({
2233
+ documents: { ids: batch },
2234
+ vectors: { ids: batch }
2235
+ });
2236
+ }
2237
+ }
2238
+ /** Deletes every document this retriever stored in its namespace. */
2239
+ async clear() {
2240
+ await this._delete({
2241
+ documents: { filter: OURS },
2242
+ vectors: { deleteAll: true }
2243
+ });
2244
+ }
2245
+ /**
2246
+ * Every document, without vectors.
2247
+ * @param {{ limit?: number }} [options]
2248
+ * @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
2249
+ */
2250
+ async list({ limit } = {}) {
2251
+ await this._resolve();
2252
+ await this._waitForWrites();
2253
+ return this._api === "documents" ? this._listDocuments(limit) : this._listVectors(limit);
2254
+ }
2255
+ /** @returns {Promise<number>} The documents this retriever stored */
2256
+ async size() {
2257
+ return (await this.list()).length;
2258
+ }
2259
+ /** Documents API: fetch by filter, page by page. @private */
2260
+ async _listDocuments(limit) {
2261
+ const probe = this._probe();
2262
+ if (probe) {
2263
+ const max = Math.min(limit || SEARCH_MAX, SEARCH_MAX);
2264
+ const found = await this._search(probe, max, void 0, OURS);
2265
+ if (found.length < SEARCH_MAX || limit) {
2266
+ const seen = /* @__PURE__ */ new Set();
2267
+ return found.filter((m) => !seen.has(m.id) && seen.add(m.id)).map((m) => fromFields(m.fields, m.id));
2268
+ }
2269
+ }
2270
+ const out = [];
2271
+ let token;
2272
+ do {
2273
+ let json;
2274
+ try {
2275
+ ({ json } = await this._data("POST", this._docs("fetch"), {
2276
+ filter: OURS,
2277
+ include_fields: FIELDS,
2278
+ limit: limit ? Math.min(PAGE, limit - out.length) : PAGE,
2279
+ ...token && { pagination_token: token }
2280
+ }));
2281
+ } catch (err) {
2282
+ if (err.status === 404) break;
2283
+ throw err;
2284
+ }
2285
+ for (const [id, fields] of Object.entries(json.documents || {})) {
2286
+ out.push(fromFields(fields, id));
2287
+ }
2288
+ token = json.pagination && json.pagination.next;
2289
+ } while (token && (!limit || out.length < limit));
2290
+ return limit ? out.slice(0, limit) : out;
2291
+ }
2292
+ /** Vectors API: list ids, then fetch them. @private */
2293
+ async _listVectors(limit) {
2294
+ const ids = [];
2295
+ let token;
2296
+ do {
2297
+ const params = new URLSearchParams({
2298
+ namespace: this.namespace,
2299
+ limit: String(PAGE)
2300
+ });
2301
+ if (token) params.set("paginationToken", token);
2302
+ let json;
2303
+ try {
2304
+ ({ json } = await this._data("GET", `/vectors/list?${params}`));
2305
+ } catch (err) {
2306
+ if (err.status === 404) break;
2307
+ throw err;
2308
+ }
2309
+ for (const v of json.vectors || []) ids.push(v.id);
2310
+ token = json.pagination && json.pagination.next;
2311
+ } while (token && (!limit || ids.length < limit));
2312
+ const wanted = limit ? ids.slice(0, limit) : ids;
2313
+ const out = [];
2314
+ for (let i = 0; i < wanted.length; i += PAGE) {
2315
+ const params = new URLSearchParams({ namespace: this.namespace });
2316
+ for (const id of wanted.slice(i, i + PAGE)) params.append("ids", id);
2317
+ const { json } = await this._data("GET", `/vectors/fetch?${params}`);
2318
+ for (const id of wanted.slice(i, i + PAGE)) {
2319
+ const record = json.vectors && json.vectors[id];
2320
+ if (record) out.push(fromFields(record.metadata, id));
2321
+ }
2322
+ }
2323
+ return out;
2324
+ }
2325
+ };
2326
+ module2.exports = { PineconeRetriever, recordId, API_VERSION };
2327
+ }
2328
+ });
2329
+
2330
+ // src/retrieval/pgvector-retriever.js
2331
+ var require_pgvector_retriever = __commonJS({
2332
+ "src/retrieval/pgvector-retriever.js"(exports2, module2) {
2333
+ "use strict";
2334
+ var { throwIfAborted, raceAbort } = require_connector_utils();
2335
+ var IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
2336
+ var DEFAULT_TABLE = "rag_documents";
2337
+ var PG_DEFAULT_EF_SEARCH = 40;
2338
+ var MAX_BATCH = 1e3;
2339
+ function quoteTable(name) {
2340
+ const parts = String(name).split(".");
2341
+ if (parts.length > 2 || !parts.every((p) => IDENTIFIER.test(p))) {
2342
+ throw new Error(
2343
+ `PgVectorRetriever: invalid table name "${name}" (letters, digits and underscores; optionally schema.table)`
2344
+ );
2345
+ }
2346
+ return parts.map((p) => `"${p}"`).join(".");
2347
+ }
2348
+ function vectorLiteral(vector, id) {
2349
+ if (!Array.isArray(vector) || vector.length === 0) {
2350
+ throw new Error(`PgVectorRetriever: document "${id}" has no vector`);
2351
+ }
2352
+ for (const x of vector) {
2353
+ if (typeof x !== "number" || !Number.isFinite(x)) {
2354
+ throw new Error(
2355
+ `PgVectorRetriever: document "${id}" has a vector with a non-finite value`
2356
+ );
2357
+ }
2358
+ }
2359
+ return `[${vector.join(",")}]`;
2360
+ }
2361
+ function explain(err, table) {
2362
+ const code = err && err.code;
2363
+ if (code === "42P01") {
2364
+ return new Error(
2365
+ `PgVectorRetriever: table ${table} does not exist. Store documents first (rag-pipeline ingest), or create it.`
2366
+ );
2367
+ }
2368
+ if (code === "42501" || /permission denied/i.test(String(err && err.message))) {
2369
+ return new Error(
2370
+ `PgVectorRetriever: permission denied. Run \`CREATE EXTENSION IF NOT EXISTS vector;\` as a database owner, and grant this user access to ${table}. (${err.message})`
2371
+ );
2372
+ }
2373
+ if (/different vector dimensions|expected \d+ dimensions/i.test(
2374
+ String(err && err.message)
2375
+ )) {
2376
+ return new Error(
2377
+ `PgVectorRetriever: ${err.message}. The table holds vectors from another embedder; use the same embedder or a new table.`
2378
+ );
2379
+ }
2380
+ return err;
2381
+ }
2382
+ var PgVectorRetriever = class {
2383
+ /**
2384
+ * @param {Object} options
2385
+ * @param {{ query: Function, connect?: Function }} [options.pool] - A pg Pool or Client you manage
2386
+ * @param {string} [options.connectionString] - Used to create a Pool when no `pool` is given
2387
+ * @param {string} [options.table='rag_documents'] - Table name, optionally `schema.table`
2388
+ * @param {number} [options.dimensions] - Vector size for a new table; defaults to the first stored vector's
2389
+ * @param {boolean} [options.createTable=true] - Create the extension, table and index on first store
2390
+ * @param {'hnsw'|'none'} [options.index='hnsw'] - Index created with the table
2391
+ * @param {number} [options.efSearch] - HNSW candidate list size per query (pgvector's default is 40). Higher finds more of the true nearest neighbours, more slowly; a query always uses at least its topK
2392
+ * @param {string} [options.cacheKey] - Cache identity; change it when the table changes (see getCacheKey)
2393
+ */
2394
+ constructor(options = {}) {
2395
+ const {
2396
+ pool,
2397
+ connectionString,
2398
+ table = DEFAULT_TABLE,
2399
+ dimensions,
2400
+ createTable = true,
2401
+ index = "hnsw",
2402
+ efSearch,
2403
+ cacheKey
2404
+ } = options;
2405
+ if (!pool && !connectionString) {
2406
+ throw new Error(
2407
+ "PgVectorRetriever needs a pg `pool` (or Client) or a `connectionString`"
2408
+ );
2409
+ }
2410
+ if (pool && typeof pool.query !== "function") {
2411
+ throw new Error("PgVectorRetriever: `pool` must have a query() method");
2412
+ }
2413
+ if (!["hnsw", "none"].includes(index)) {
2414
+ throw new Error(
2415
+ `PgVectorRetriever: unknown index "${index}" (use "hnsw" or "none")`
2416
+ );
2417
+ }
2418
+ if (dimensions !== void 0 && !(Number.isInteger(dimensions) && dimensions > 0)) {
2419
+ throw new Error(
2420
+ "PgVectorRetriever: `dimensions` must be a positive integer"
2421
+ );
2422
+ }
2423
+ if (efSearch !== void 0 && !(Number.isInteger(efSearch) && efSearch > 0 && efSearch <= 1e3)) {
2424
+ throw new Error(
2425
+ "PgVectorRetriever: `efSearch` must be an integer from 1 to 1000"
2426
+ );
2427
+ }
2428
+ this.efSearch = efSearch;
2429
+ this.table = quoteTable(table);
2430
+ this.tableName = table;
2431
+ this.dimensions = dimensions;
2432
+ this.createTable = createTable;
2433
+ this.index = index;
2434
+ this.cacheKey = cacheKey;
2435
+ this._pool = pool || null;
2436
+ this._connectionString = connectionString;
2437
+ this._ownsPool = !pool;
2438
+ this._ready = null;
2439
+ }
2440
+ /** The pg Pool or Client in use; creates the Pool on first use. */
2441
+ _db() {
2442
+ if (!this._pool) {
2443
+ let pg;
2444
+ try {
2445
+ pg = require("pg");
2446
+ } catch {
2447
+ throw new Error(
2448
+ 'PgVectorRetriever with a connectionString needs the "pg" package: npm install pg'
2449
+ );
2450
+ }
2451
+ this._pool = new pg.Pool({
2452
+ connectionString: this._connectionString,
2453
+ allowExitOnIdle: true
2454
+ });
2455
+ }
2456
+ return this._pool;
2457
+ }
2458
+ async _query(text, params, signal) {
2459
+ throwIfAborted(signal);
2460
+ try {
2461
+ return await raceAbort(this._db().query(text, params), signal);
2462
+ } catch (err) {
2463
+ throw explain(err, this.table);
2464
+ }
2465
+ }
2466
+ /** Runs one query with `SET LOCAL hnsw.ef_search`, in its own transaction. */
2467
+ async _queryWithEfSearch(text, params, ef, signal) {
2468
+ throwIfAborted(signal);
2469
+ const db = this._db();
2470
+ const pooled = typeof db.connect === "function" && typeof db.end === "function" && db.totalCount !== void 0;
2471
+ const client = pooled ? await db.connect() : db;
2472
+ try {
2473
+ await client.query("BEGIN");
2474
+ await client.query(`SET LOCAL hnsw.ef_search = ${Math.min(ef, 1e3)}`);
2475
+ const result = await client.query(text, params);
2476
+ await client.query("COMMIT");
2477
+ return result;
2478
+ } catch (err) {
2479
+ await client.query("ROLLBACK").catch(() => {
2480
+ });
2481
+ throw explain(err, this.table);
2482
+ } finally {
2483
+ if (pooled) client.release();
2484
+ }
2485
+ }
2486
+ /** Creates the extension, table and index once, sized to `dimensions`. */
2487
+ _ensureTable(dimensions) {
2488
+ if (!this.createTable) return Promise.resolve();
2489
+ if (!this._ready) {
2490
+ const size = this.dimensions || dimensions;
2491
+ const indexName = `"${this.tableName.split(".").pop()}_embedding_hnsw"`;
2492
+ this._ready = (async () => {
2493
+ await this._query("CREATE EXTENSION IF NOT EXISTS vector");
2494
+ await this._query(
2495
+ `CREATE TABLE IF NOT EXISTS ${this.table} (id text PRIMARY KEY, content text NOT NULL DEFAULT '', metadata jsonb NOT NULL DEFAULT '{}'::jsonb, embedding vector(${size}) NOT NULL)`
2496
+ );
2497
+ if (this.index === "hnsw") {
2498
+ await this._query(
2499
+ `CREATE INDEX IF NOT EXISTS ${indexName} ON ${this.table} USING hnsw (embedding vector_cosine_ops)`
2500
+ );
2501
+ }
2502
+ })().catch((err) => {
2503
+ this._ready = null;
2504
+ throw err;
2505
+ });
2506
+ }
2507
+ return this._ready;
2508
+ }
2509
+ /**
2510
+ * Cache identity for createRagPipeline({ cache }). Other processes can
2511
+ * write to the table without this one knowing, so there is none unless
2512
+ * you set `cacheKey`: then cached retrieval results are reused until you
2513
+ * change the key or their time to live ends.
2514
+ * @param {'retriever'} role
2515
+ * @returns {string|null}
2516
+ */
2517
+ getCacheKey(role) {
2518
+ if (role !== "retriever" || !this.cacheKey) return null;
2519
+ return `pgvector|${this.tableName}|${this.cacheKey}`;
2520
+ }
2521
+ /**
2522
+ * Inserts or replaces documents, in one transaction: a failed batch
2523
+ * changes nothing.
2524
+ * @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
2525
+ * @returns {Promise<void>}
2526
+ */
2527
+ async store(docs) {
2528
+ if (!Array.isArray(docs))
2529
+ throw new Error("PgVectorRetriever.store() takes an array");
2530
+ if (docs.length === 0) return;
2531
+ let dimensions = null;
2532
+ const rows = docs.map((doc) => {
2533
+ const id = doc && doc.id;
2534
+ if (typeof id !== "string" || id.length === 0) {
2535
+ throw new Error(
2536
+ "PgVectorRetriever.store(): every document needs a string id"
2537
+ );
2538
+ }
2539
+ const literal = vectorLiteral(doc.vector, id);
2540
+ if (dimensions === null) dimensions = doc.vector.length;
2541
+ else if (doc.vector.length !== dimensions) {
2542
+ throw new Error(
2543
+ `PgVectorRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the batch has ${dimensions}`
1825
2544
  );
1826
2545
  }
1827
2546
  return [
@@ -1961,6 +2680,453 @@ var require_pgvector_retriever = __commonJS({
1961
2680
  }
1962
2681
  });
1963
2682
 
2683
+ // src/retrieval/weaviate-retriever.js
2684
+ var require_weaviate_retriever = __commonJS({
2685
+ "src/retrieval/weaviate-retriever.js"(exports2, module2) {
2686
+ "use strict";
2687
+ var { throwIfAborted } = require_connector_utils();
2688
+ var { pointId } = require_qdrant_retriever();
2689
+ var COLLECTION = /^[A-Za-z][A-Za-z0-9_]{0,254}$/;
2690
+ var DEFAULT_COLLECTION = "RagDocuments";
2691
+ var PAGE = 256;
2692
+ var UPSERT_BATCH = 256;
2693
+ var DELETE_BATCH = 256;
2694
+ var PROPERTIES = [
2695
+ {
2696
+ name: "rag_id",
2697
+ dataType: ["text"],
2698
+ tokenization: "field",
2699
+ indexFilterable: true,
2700
+ indexSearchable: false
2701
+ },
2702
+ { name: "content", dataType: ["text"], indexFilterable: false },
2703
+ {
2704
+ name: "rag_source",
2705
+ dataType: ["text"],
2706
+ tokenization: "field",
2707
+ indexFilterable: true,
2708
+ indexSearchable: false
2709
+ },
2710
+ {
2711
+ name: "rag_metadata",
2712
+ dataType: ["text"],
2713
+ indexFilterable: false,
2714
+ indexSearchable: false
2715
+ }
2716
+ ];
2717
+ var OURS = { path: ["rag_id"], operator: "Like", valueText: "*" };
2718
+ var OURS_GRAPHQL = '{ path: ["rag_id"], operator: Like, valueText: "*" }';
2719
+ function checkVector(vector, id) {
2720
+ if (!Array.isArray(vector) || vector.length === 0) {
2721
+ throw new Error(`WeaviateRetriever: document "${id}" has no vector`);
2722
+ }
2723
+ if (!vector.every((x) => typeof x === "number" && Number.isFinite(x))) {
2724
+ throw new Error(
2725
+ `WeaviateRetriever: document "${id}" has a vector with a non-finite value`
2726
+ );
2727
+ }
2728
+ }
2729
+ function trimSlashes(url) {
2730
+ let end = url.length;
2731
+ while (end > 0 && url[end - 1] === "/") end--;
2732
+ return url.slice(0, end);
2733
+ }
2734
+ function parseMetadata(text) {
2735
+ if (!text) return {};
2736
+ try {
2737
+ const value = JSON.parse(text);
2738
+ return value && typeof value === "object" ? value : {};
2739
+ } catch {
2740
+ return {};
2741
+ }
2742
+ }
2743
+ function errorDetail(json) {
2744
+ const list = json && (json.error || json.errors) || [];
2745
+ if (Array.isArray(list) && list.length) {
2746
+ return list.map((e) => e && e.message || String(e)).join("; ");
2747
+ }
2748
+ return json && json.message || "";
2749
+ }
2750
+ var WeaviateRetriever = class {
2751
+ /**
2752
+ * @param {Object} [options]
2753
+ * @param {string} [options.url='http://localhost:8080'] - Weaviate's REST endpoint (without /v1)
2754
+ * @param {string} [options.apiKey] - Sent as `Authorization: Bearer <key>`
2755
+ * @param {string} [options.collection='RagDocuments'] - Class name; the first letter is upper-cased, as Weaviate does
2756
+ * @param {number} [options.dimensions] - Expected vector size; defaults to the collection's or the first stored vector's
2757
+ * @param {boolean} [options.createCollection=true] - Create the collection (and any missing properties) on first store
2758
+ * @param {string} [options.cacheKey] - Cache identity; change it when the collection changes
2759
+ * @param {Function} [options.fetch] - fetch implementation (defaults to the global)
2760
+ */
2761
+ constructor(options = {}) {
2762
+ const {
2763
+ url = "http://localhost:8080",
2764
+ apiKey,
2765
+ collection = DEFAULT_COLLECTION,
2766
+ dimensions,
2767
+ createCollection = true,
2768
+ cacheKey,
2769
+ fetch: fetchImpl = globalThis.fetch
2770
+ } = options;
2771
+ if (typeof collection !== "string" || !COLLECTION.test(collection)) {
2772
+ throw new Error(
2773
+ `WeaviateRetriever: invalid collection name "${collection}" (a letter, then letters, digits and "_")`
2774
+ );
2775
+ }
2776
+ if (dimensions !== void 0 && !(Number.isInteger(dimensions) && dimensions > 0)) {
2777
+ throw new Error(
2778
+ "WeaviateRetriever: `dimensions` must be a positive integer"
2779
+ );
2780
+ }
2781
+ if (typeof fetchImpl !== "function") {
2782
+ throw new Error("WeaviateRetriever needs fetch (Node.js 18 or later)");
2783
+ }
2784
+ this.url = trimSlashes(String(url));
2785
+ this.collection = collection[0].toUpperCase() + collection.slice(1);
2786
+ this.dimensions = dimensions;
2787
+ this.createCollection = createCollection;
2788
+ this.cacheKey = cacheKey;
2789
+ this._apiKey = apiKey;
2790
+ this._fetch = fetchImpl === globalThis.fetch ? fetchImpl.bind(globalThis) : fetchImpl;
2791
+ this._ready = null;
2792
+ this._exists = false;
2793
+ }
2794
+ /**
2795
+ * Cache identity for createRagPipeline({ cache }): none unless you set
2796
+ * `cacheKey`, since other clients can change the collection.
2797
+ * @param {'retriever'} role
2798
+ * @returns {string|null}
2799
+ */
2800
+ getCacheKey(role) {
2801
+ if (role !== "retriever" || !this.cacheKey) return null;
2802
+ return `weaviate|${this.url}|${this.collection}|${this.cacheKey}`;
2803
+ }
2804
+ /** The message without the API key, should a server ever echo it. */
2805
+ _redact(message) {
2806
+ const text = String(message);
2807
+ return this._apiKey ? text.split(this._apiKey).join("***") : text;
2808
+ }
2809
+ /**
2810
+ * One REST call; resolves the parsed body.
2811
+ * @private
2812
+ * @param {string} method
2813
+ * @param {string} path - After /v1
2814
+ * @param {Object} [body]
2815
+ * @param {{ signal?: AbortSignal, allow404?: boolean }} [options] - allow404 resolves null on 404
2816
+ * @returns {Promise<any>}
2817
+ */
2818
+ async _request(method, path, body, { signal, allow404 = false } = {}) {
2819
+ throwIfAborted(signal);
2820
+ const headers = { "content-type": "application/json" };
2821
+ if (this._apiKey) headers.authorization = `Bearer ${this._apiKey}`;
2822
+ let res;
2823
+ try {
2824
+ res = await this._fetch(`${this.url}/v1${path}`, {
2825
+ method,
2826
+ headers,
2827
+ ...body !== void 0 && { body: JSON.stringify(body) },
2828
+ ...signal && { signal }
2829
+ });
2830
+ } catch (err) {
2831
+ throwIfAborted(signal);
2832
+ throw new Error(
2833
+ this._redact(
2834
+ `WeaviateRetriever: cannot reach Weaviate at ${this.url} (${err.message})`
2835
+ )
2836
+ );
2837
+ }
2838
+ if (res.status === 404 && allow404) return null;
2839
+ const text = await res.text();
2840
+ let json;
2841
+ try {
2842
+ json = text ? JSON.parse(text) : {};
2843
+ } catch {
2844
+ json = {};
2845
+ }
2846
+ if (!res.ok) {
2847
+ if (res.status === 401 || res.status === 403) {
2848
+ throw new Error(
2849
+ `WeaviateRetriever: Weaviate refused the request (${res.status}); check WEAVIATE_API_KEY`
2850
+ );
2851
+ }
2852
+ const detail = errorDetail(json) || text || res.statusText;
2853
+ const where = path.split("?")[0];
2854
+ throw new Error(
2855
+ this._redact(
2856
+ `WeaviateRetriever: ${method} ${where} failed (${res.status}): ${detail}`
2857
+ )
2858
+ );
2859
+ }
2860
+ return json;
2861
+ }
2862
+ /** A GraphQL query; resolves `data`, throws on GraphQL errors. */
2863
+ async _graphql(query, signal) {
2864
+ const json = await this._request("POST", "/graphql", { query }, { signal });
2865
+ if (json.errors && json.errors.length) {
2866
+ throw new Error(
2867
+ this._redact(
2868
+ `WeaviateRetriever: GraphQL query failed: ${errorDetail(json)}`
2869
+ )
2870
+ );
2871
+ }
2872
+ return json.data || {};
2873
+ }
2874
+ /** The collection's schema, or null when it doesn't exist. */
2875
+ _schema(signal) {
2876
+ return this._request("GET", `/schema/${this.collection}`, void 0, {
2877
+ signal,
2878
+ allow404: true
2879
+ });
2880
+ }
2881
+ /** Whether the collection exists; only a yes is remembered. */
2882
+ async _collectionExists(signal) {
2883
+ if (!this._exists) this._exists = Boolean(await this._schema(signal));
2884
+ return this._exists;
2885
+ }
2886
+ _ensureCollection() {
2887
+ if (!this._ready) {
2888
+ this._ready = (async () => {
2889
+ if (this.createCollection) {
2890
+ const schema = await this._schema();
2891
+ if (!schema) {
2892
+ try {
2893
+ await this._request("POST", "/schema", {
2894
+ class: this.collection,
2895
+ vectorizer: "none",
2896
+ vectorIndexType: "hnsw",
2897
+ vectorIndexConfig: { distance: "cosine" },
2898
+ properties: PROPERTIES
2899
+ });
2900
+ } catch (err) {
2901
+ if (!/already exists/i.test(err.message)) throw err;
2902
+ }
2903
+ } else {
2904
+ const have = new Set((schema.properties || []).map((p) => p.name));
2905
+ for (const property of PROPERTIES) {
2906
+ if (!have.has(property.name)) {
2907
+ await this._request(
2908
+ "POST",
2909
+ `/schema/${this.collection}/properties`,
2910
+ property
2911
+ );
2912
+ }
2913
+ }
2914
+ }
2915
+ this._exists = true;
2916
+ }
2917
+ if (!this.dimensions) {
2918
+ const page = await this._request(
2919
+ "GET",
2920
+ `/objects?class=${this.collection}&limit=1&include=vector`,
2921
+ void 0,
2922
+ { allow404: true }
2923
+ );
2924
+ const vector = page && page.objects && page.objects[0] && page.objects[0].vector;
2925
+ if (Array.isArray(vector) && vector.length) {
2926
+ this.dimensions = vector.length;
2927
+ }
2928
+ }
2929
+ })().catch((err) => {
2930
+ this._ready = null;
2931
+ throw err;
2932
+ });
2933
+ }
2934
+ return this._ready;
2935
+ }
2936
+ /**
2937
+ * Inserts or replaces documents.
2938
+ * @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
2939
+ * @returns {Promise<void>}
2940
+ */
2941
+ async store(docs) {
2942
+ if (!Array.isArray(docs))
2943
+ throw new Error("WeaviateRetriever.store() takes an array");
2944
+ if (docs.length === 0) return;
2945
+ let dimensions = null;
2946
+ const objects = docs.map((doc) => {
2947
+ const id = doc && doc.id;
2948
+ if (typeof id !== "string" || id.length === 0) {
2949
+ throw new Error(
2950
+ "WeaviateRetriever.store(): every document needs a string id"
2951
+ );
2952
+ }
2953
+ checkVector(doc.vector, id);
2954
+ if (dimensions === null) dimensions = doc.vector.length;
2955
+ else if (doc.vector.length !== dimensions) {
2956
+ throw new Error(
2957
+ `WeaviateRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the batch has ${dimensions}`
2958
+ );
2959
+ }
2960
+ const metadata = doc.metadata || {};
2961
+ const properties = {
2962
+ rag_id: id,
2963
+ content: doc.content || "",
2964
+ rag_metadata: JSON.stringify(metadata)
2965
+ };
2966
+ if (metadata.source !== void 0 && metadata.source !== null) {
2967
+ properties.rag_source = String(metadata.source);
2968
+ }
2969
+ return {
2970
+ class: this.collection,
2971
+ id: pointId(id),
2972
+ vector: doc.vector,
2973
+ properties
2974
+ };
2975
+ });
2976
+ await this._ensureCollection();
2977
+ if (this.dimensions && dimensions !== this.dimensions) {
2978
+ throw new Error(
2979
+ `WeaviateRetriever.store(): the documents have ${dimensions} dimensions, the collection has ${this.dimensions}`
2980
+ );
2981
+ }
2982
+ for (let i = 0; i < objects.length; i += UPSERT_BATCH) {
2983
+ const batch = objects.slice(i, i + UPSERT_BATCH);
2984
+ const results = await this._request("POST", "/batch/objects", {
2985
+ objects: batch
2986
+ });
2987
+ for (const [j, item] of (Array.isArray(results) ? results : []).entries()) {
2988
+ const detail = item && item.result && errorDetail(item.result.errors);
2989
+ if (!detail) continue;
2990
+ const id = batch[j] ? batch[j].properties.rag_id : "?";
2991
+ const kind = /vectors? with length/i.test(detail) ? "has the wrong vector dimensions" : "was not stored";
2992
+ throw new Error(
2993
+ this._redact(
2994
+ `WeaviateRetriever.store(): document "${id}" ${kind}: ${detail}`
2995
+ )
2996
+ );
2997
+ }
2998
+ }
2999
+ this._exists = true;
3000
+ if (!this.dimensions) this.dimensions = dimensions;
3001
+ }
3002
+ /**
3003
+ * The documents most similar to a query vector.
3004
+ * @param {number[]|{ queryVector?: number[], topK?: number, signal?: AbortSignal }} request
3005
+ * A vector, or an object with one (required; optional in the type to
3006
+ * match the retriever contract)
3007
+ * @param {number} [k=10] - Results when `request` is a vector
3008
+ * @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
3009
+ */
3010
+ async retrieve(request, k = 10) {
3011
+ const {
3012
+ queryVector,
3013
+ topK = k,
3014
+ signal
3015
+ } = Array.isArray(request) ? { queryVector: request } : request || {};
3016
+ if (!Number.isInteger(topK) || topK < 1) {
3017
+ throw new Error(
3018
+ "WeaviateRetriever.retrieve(): topK must be a positive integer"
3019
+ );
3020
+ }
3021
+ checkVector(queryVector, "query");
3022
+ throwIfAborted(signal);
3023
+ if (!await this._collectionExists(signal)) return [];
3024
+ const data = await this._graphql(
3025
+ `{ Get { ${this.collection}(nearVector: { vector: ${JSON.stringify(queryVector)} }, limit: ${topK}, where: ${OURS_GRAPHQL}) { rag_id content rag_metadata _additional { distance } } } }`,
3026
+ signal
3027
+ );
3028
+ throwIfAborted(signal);
3029
+ const hits = data.Get && data.Get[this.collection] || [];
3030
+ return hits.map((hit) => ({
3031
+ id: hit.rag_id,
3032
+ score: 1 - hit._additional.distance,
3033
+ content: hit.content || "",
3034
+ metadata: parseMetadata(hit.rag_metadata)
3035
+ }));
3036
+ }
3037
+ /**
3038
+ * Deletes the matching objects, a server-limited round at a time.
3039
+ * @private
3040
+ */
3041
+ async _deleteWhere(where) {
3042
+ if (!await this._collectionExists()) return;
3043
+ for (; ; ) {
3044
+ const json = await this._request("DELETE", "/batch/objects", {
3045
+ match: { class: this.collection, where },
3046
+ output: "minimal"
3047
+ });
3048
+ const r = json && json.results || {};
3049
+ if (r.failed > 0) {
3050
+ throw new Error(
3051
+ `WeaviateRetriever: ${r.failed} objects could not be deleted`
3052
+ );
3053
+ }
3054
+ if (!(r.limit > 0 && r.matches > r.limit && r.successful > 0)) return;
3055
+ }
3056
+ }
3057
+ /**
3058
+ * Deletes the chunks of one source (`metadata.source`), for re-ingesting.
3059
+ * @param {string} source
3060
+ */
3061
+ async removeBySource(source) {
3062
+ await this._deleteWhere({
3063
+ path: ["rag_source"],
3064
+ operator: "Equal",
3065
+ valueText: String(source)
3066
+ });
3067
+ }
3068
+ /**
3069
+ * Deletes documents by id.
3070
+ * @param {string[]} ids
3071
+ */
3072
+ async delete(ids) {
3073
+ for (let i = 0; i < ids.length; i += DELETE_BATCH) {
3074
+ await this._deleteWhere({
3075
+ path: ["rag_id"],
3076
+ operator: "ContainsAny",
3077
+ valueTextArray: ids.slice(i, i + DELETE_BATCH).map(String)
3078
+ });
3079
+ }
3080
+ }
3081
+ /** Deletes every document this class stored; the collection stays. */
3082
+ async clear() {
3083
+ await this._deleteWhere(OURS);
3084
+ }
3085
+ /** @returns {Promise<number>} */
3086
+ async size() {
3087
+ if (!await this._collectionExists()) return 0;
3088
+ const data = await this._graphql(
3089
+ `{ Aggregate { ${this.collection}(where: ${OURS_GRAPHQL}) { meta { count } } } }`
3090
+ );
3091
+ const rows = data.Aggregate && data.Aggregate[this.collection] || [];
3092
+ return rows[0] && rows[0].meta ? rows[0].meta.count : 0;
3093
+ }
3094
+ /**
3095
+ * Every document, without vectors.
3096
+ * @param {{ limit?: number }} [options]
3097
+ * @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
3098
+ */
3099
+ async list({ limit } = {}) {
3100
+ const out = [];
3101
+ if (!await this._collectionExists()) return out;
3102
+ let after = "";
3103
+ for (; ; ) {
3104
+ const page = await this._request(
3105
+ "GET",
3106
+ `/objects?class=${this.collection}&limit=${PAGE}&after=${encodeURIComponent(after)}`,
3107
+ void 0,
3108
+ { allow404: true }
3109
+ );
3110
+ const objects = page && page.objects || [];
3111
+ for (const o of objects) {
3112
+ const p = o.properties || {};
3113
+ if (typeof p.rag_id !== "string") continue;
3114
+ out.push({
3115
+ id: p.rag_id,
3116
+ content: p.content || "",
3117
+ metadata: parseMetadata(p.rag_metadata)
3118
+ });
3119
+ if (limit && out.length >= limit) return out;
3120
+ }
3121
+ if (objects.length < PAGE) return out;
3122
+ after = objects[objects.length - 1].id;
3123
+ }
3124
+ }
3125
+ };
3126
+ module2.exports = { WeaviateRetriever };
3127
+ }
3128
+ });
3129
+
1964
3130
  // src/adapters/ai-sdk.js
1965
3131
  var require_ai_sdk = __commonJS({
1966
3132
  "src/adapters/ai-sdk.js"(exports2, module2) {
@@ -2119,7 +3285,9 @@ var require_connectors = __commonJS({
2119
3285
  var { OllamaConnector } = require_ollama_connector();
2120
3286
  var { LocalEmbedder } = require_local_embedder();
2121
3287
  var { QdrantRetriever } = require_qdrant_retriever();
3288
+ var { PineconeRetriever } = require_pinecone_retriever();
2122
3289
  var { PgVectorRetriever } = require_pgvector_retriever();
3290
+ var { WeaviateRetriever } = require_weaviate_retriever();
2123
3291
  var { createAiSdkAdapter } = require_ai_sdk();
2124
3292
  module2.exports = {
2125
3293
  OpenAIConnector,
@@ -2128,7 +3296,9 @@ var require_connectors = __commonJS({
2128
3296
  OllamaConnector,
2129
3297
  LocalEmbedder,
2130
3298
  QdrantRetriever,
3299
+ PineconeRetriever,
2131
3300
  PgVectorRetriever,
3301
+ WeaviateRetriever,
2132
3302
  createAiSdkAdapter
2133
3303
  };
2134
3304
  }
@@ -2147,7 +3317,9 @@ var modules = {
2147
3317
  "connectors/rag-prompt.js": () => require_rag_prompt(),
2148
3318
  "packages/connectors.js": () => require_connectors(),
2149
3319
  "retrieval/pgvector-retriever.js": () => require_pgvector_retriever(),
2150
- "retrieval/qdrant-retriever.js": () => require_qdrant_retriever()
3320
+ "retrieval/pinecone-retriever.js": () => require_pinecone_retriever(),
3321
+ "retrieval/qdrant-retriever.js": () => require_qdrant_retriever(),
3322
+ "retrieval/weaviate-retriever.js": () => require_weaviate_retriever()
2151
3323
  };
2152
3324
  Object.defineProperty(mod, Symbol.for("@devilsdev/rag-connectors/internal"), {
2153
3325
  value: (file) => {