@devilsdev/rag-connectors 3.5.0 → 3.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # @devilsdev/rag-connectors
2
2
 
3
- Embedders, LLMs and vector stores for @devilsdev/rag-core: OpenAI, Anthropic, Cohere, Ollama, the AI SDK, Qdrant and pgvector.
3
+ Embedders, LLMs and vector stores for @devilsdev/rag-core: OpenAI, Anthropic, Cohere, Ollama, the AI SDK, Qdrant, pgvector and Pinecone.
4
4
 
5
5
  Part of [@devilsdev/rag-pipeline-utils](https://www.npmjs.com/package/@devilsdev/rag-pipeline-utils), split into packages so you install only what you use. The objects are the same ones the full package exports; versions move in lockstep.
6
6
 
@@ -8,7 +8,7 @@ Part of [@devilsdev/rag-pipeline-utils](https://www.npmjs.com/package/@devilsdev
8
8
  npm install @devilsdev/rag-connectors
9
9
  ```
10
10
 
11
- Exports: `OpenAIConnector`, `AnthropicConnector`, `CohereConnector`, `OllamaConnector`, `LocalEmbedder`, `QdrantRetriever`, `PgVectorRetriever`, `createAiSdkAdapter`.
11
+ Exports: `OpenAIConnector`, `AnthropicConnector`, `CohereConnector`, `OllamaConnector`, `LocalEmbedder`, `QdrantRetriever`, `PineconeRetriever`, `PgVectorRetriever`, `createAiSdkAdapter`.
12
12
 
13
13
  Documentation: https://devilsdev.github.io/rag-pipeline-utils/
14
14
 
package/dist/index.cjs CHANGED
@@ -99,7 +99,7 @@ var require_openai_connector = __commonJS({
99
99
  "src/connectors/openai-connector.js"(exports2, module2) {
100
100
  "use strict";
101
101
  var { BaseConnector } = require_base_connector();
102
- var { loadOptionalPeer } = require_optional_peer();
102
+ var { missingPeer } = require_optional_peer();
103
103
  var {
104
104
  missingApiKeyError,
105
105
  raceAbort,
@@ -121,12 +121,15 @@ var require_openai_connector = __commonJS({
121
121
  }
122
122
  };
123
123
  function loadOpenAI() {
124
- return loadOptionalPeer(
125
- "openai",
126
- () => require("openai"),
127
- // eslint-disable-line global-require
128
- 'OpenAIConnector needs the "openai" package, an optional peer dependency since 3.0.0: npm install openai'
129
- );
124
+ try {
125
+ return require("openai");
126
+ } catch (error) {
127
+ throw missingPeer(
128
+ error,
129
+ "openai",
130
+ 'OpenAIConnector needs the "openai" package, an optional peer dependency since 3.0.0: npm install openai'
131
+ );
132
+ }
130
133
  }
131
134
  var OpenAIConnector = class extends BaseConnector {
132
135
  /**
@@ -1608,6 +1611,582 @@ var require_qdrant_retriever = __commonJS({
1608
1611
  }
1609
1612
  });
1610
1613
 
1614
+ // src/retrieval/pinecone-retriever.js
1615
+ var require_pinecone_retriever = __commonJS({
1616
+ "src/retrieval/pinecone-retriever.js"(exports2, module2) {
1617
+ "use strict";
1618
+ var crypto = require("crypto");
1619
+ var { throwIfAborted } = require_connector_utils();
1620
+ var API_VERSION = "2026-07";
1621
+ var CONTROL_PLANE = "https://api.pinecone.io";
1622
+ var DEFAULT_NAMESPACE = "rag_documents";
1623
+ var UPSERT_BATCH = 100;
1624
+ var DELETE_BATCH = 1e3;
1625
+ var PAGE = 100;
1626
+ var SEARCH_MAX = 1e4;
1627
+ var PLAIN_ID = /^[\x20-\x7e]{1,512}$/;
1628
+ var FIELDS = ["rag_id", "rag_content", "rag_metadata"];
1629
+ var OURS = { rag_id: { $exists: true } };
1630
+ function recordId(id) {
1631
+ const s = String(id);
1632
+ if (PLAIN_ID.test(s)) return s;
1633
+ return `rag-${crypto.createHash("sha256").update(s).digest("hex")}`;
1634
+ }
1635
+ function checkVector(vector, id) {
1636
+ if (!Array.isArray(vector) || vector.length === 0) {
1637
+ throw new Error(`PineconeRetriever: document "${id}" has no vector`);
1638
+ }
1639
+ if (!vector.every((x) => typeof x === "number" && Number.isFinite(x))) {
1640
+ throw new Error(
1641
+ `PineconeRetriever: document "${id}" has a vector with a non-finite value`
1642
+ );
1643
+ }
1644
+ }
1645
+ function fromFields(fields, fallbackId) {
1646
+ const m = fields || {};
1647
+ let metadata = {};
1648
+ try {
1649
+ metadata = m.rag_metadata ? JSON.parse(m.rag_metadata) : {};
1650
+ } catch {
1651
+ metadata = {};
1652
+ }
1653
+ return {
1654
+ id: m.rag_id !== void 0 ? String(m.rag_id) : fallbackId,
1655
+ content: m.rag_content || "",
1656
+ metadata
1657
+ };
1658
+ }
1659
+ function normalizeHost(host) {
1660
+ let h = String(host);
1661
+ while (h.endsWith("/")) h = h.slice(0, -1);
1662
+ return /^https?:\/\//.test(h) ? h : `https://${h}`;
1663
+ }
1664
+ var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
1665
+ var PineconeRetriever = class {
1666
+ /**
1667
+ * @param {Object} options
1668
+ * @param {string} [options.apiKey] - Defaults to PINECONE_API_KEY
1669
+ * @param {string} [options.index] - The index name, described on first use
1670
+ * @param {string} [options.host] - The index host (from the console), instead of or with `index`
1671
+ * @param {string} [options.namespace='rag_documents'] - A namespace this retriever owns
1672
+ * @param {'auto'|'documents'|'vectors'} [options.api='auto'] - Document index (Documents API) or vector index; auto reads it from the index description
1673
+ * @param {string} [options.vectorField] - The document index's dense_vector field (default: the schema's)
1674
+ * @param {number} [options.freshnessTimeoutMs=30000] - How long a read waits for this retriever's writes to be indexed; 0 to not wait
1675
+ * @param {string} [options.cacheKey] - Cache identity; change it when the index changes
1676
+ * @param {Function} [options.fetch] - fetch implementation (defaults to the global)
1677
+ */
1678
+ constructor(options = {}) {
1679
+ const {
1680
+ apiKey = typeof process !== "undefined" && process.env ? process.env.PINECONE_API_KEY : void 0,
1681
+ host,
1682
+ index,
1683
+ namespace = DEFAULT_NAMESPACE,
1684
+ api = "auto",
1685
+ vectorField,
1686
+ freshnessTimeoutMs = 3e4,
1687
+ cacheKey,
1688
+ fetch: fetchImpl = globalThis.fetch
1689
+ } = options;
1690
+ if (!apiKey) {
1691
+ throw new Error(
1692
+ "PineconeRetriever needs an API key: pass apiKey or set PINECONE_API_KEY"
1693
+ );
1694
+ }
1695
+ if (!host && !index) {
1696
+ throw new Error(
1697
+ "PineconeRetriever needs the index: pass index (its name) or host (the index host)"
1698
+ );
1699
+ }
1700
+ if (typeof namespace !== "string") {
1701
+ throw new Error("PineconeRetriever: namespace must be a string");
1702
+ }
1703
+ if (!["auto", "documents", "vectors"].includes(api)) {
1704
+ throw new Error(
1705
+ 'PineconeRetriever: api must be "auto", "documents" or "vectors"'
1706
+ );
1707
+ }
1708
+ if (typeof fetchImpl !== "function") {
1709
+ throw new Error("PineconeRetriever needs fetch (Node.js 18 or later)");
1710
+ }
1711
+ this.index = index;
1712
+ this.namespace = namespace;
1713
+ this.freshnessTimeoutMs = freshnessTimeoutMs;
1714
+ this.cacheKey = cacheKey;
1715
+ this._apiKey = apiKey;
1716
+ this._host = host ? normalizeHost(host) : null;
1717
+ this._api = api;
1718
+ this._vectorField = vectorField;
1719
+ this._fetch = fetchImpl === globalThis.fetch ? fetchImpl.bind(globalThis) : fetchImpl;
1720
+ this._resolving = null;
1721
+ this._writeLsn = 0;
1722
+ this._dimension = null;
1723
+ }
1724
+ /**
1725
+ * Cache identity for createRagPipeline({ cache }): none unless you set
1726
+ * `cacheKey`, since other clients can change the index.
1727
+ * @param {'retriever'} role
1728
+ * @returns {string|null}
1729
+ */
1730
+ getCacheKey(role) {
1731
+ if (role !== "retriever" || !this.cacheKey) return null;
1732
+ return `pinecone|${this._host || this.index}|${this.namespace}|${this.cacheKey}`;
1733
+ }
1734
+ /**
1735
+ * One REST call; resolves the parsed body and the response headers.
1736
+ * @private
1737
+ */
1738
+ async _call(url, method, body, signal) {
1739
+ throwIfAborted(signal);
1740
+ const headers = {
1741
+ "Api-Key": this._apiKey,
1742
+ "X-Pinecone-Api-Version": API_VERSION,
1743
+ accept: "application/json"
1744
+ };
1745
+ if (body !== void 0) headers["content-type"] = "application/json";
1746
+ let res;
1747
+ try {
1748
+ res = await this._fetch(url, {
1749
+ method,
1750
+ headers,
1751
+ ...body !== void 0 && { body: JSON.stringify(body) },
1752
+ ...signal && { signal }
1753
+ });
1754
+ } catch (err) {
1755
+ throwIfAborted(signal);
1756
+ throw new Error(
1757
+ `PineconeRetriever: cannot reach Pinecone (${err.message})`
1758
+ );
1759
+ }
1760
+ const text = await res.text();
1761
+ let json;
1762
+ try {
1763
+ json = text ? JSON.parse(text) : {};
1764
+ } catch {
1765
+ json = {};
1766
+ }
1767
+ if (!res.ok) {
1768
+ if (res.status === 401 || res.status === 403) {
1769
+ throw new Error(
1770
+ `PineconeRetriever: Pinecone refused the request (${res.status}); check PINECONE_API_KEY`
1771
+ );
1772
+ }
1773
+ const detail = json.message || json.error && json.error.message || text || res.statusText;
1774
+ const err = new Error(
1775
+ `PineconeRetriever: ${method} ${new URL(url).pathname} failed (${res.status}): ${detail}`
1776
+ );
1777
+ err.status = res.status;
1778
+ throw err;
1779
+ }
1780
+ return { json, headers: res.headers };
1781
+ }
1782
+ /**
1783
+ * The index's host and kind, from its description (once): a document
1784
+ * index's dense_vector field and dimension, or a vector index's.
1785
+ * @private
1786
+ */
1787
+ _resolve(signal) {
1788
+ if (this._host && this._api !== "auto" && (this._api === "vectors" || this._vectorField)) {
1789
+ return Promise.resolve();
1790
+ }
1791
+ if (!this._resolving) {
1792
+ this._resolving = this._describe(signal).catch((err) => {
1793
+ this._resolving = null;
1794
+ throw err;
1795
+ });
1796
+ }
1797
+ return this._resolving;
1798
+ }
1799
+ /** @private */
1800
+ async _describe(signal) {
1801
+ let info;
1802
+ if (this.index) {
1803
+ ({ json: info } = await this._call(
1804
+ `${CONTROL_PLANE}/indexes/${encodeURIComponent(this.index)}`,
1805
+ "GET",
1806
+ void 0,
1807
+ signal
1808
+ ));
1809
+ } else {
1810
+ const { json } = await this._call(
1811
+ `${CONTROL_PLANE}/indexes`,
1812
+ "GET",
1813
+ void 0,
1814
+ signal
1815
+ );
1816
+ const host = this._host.replace(/^https?:\/\//, "");
1817
+ info = (json.indexes || []).find((i) => i.host === host);
1818
+ if (!info) {
1819
+ throw new Error(
1820
+ `PineconeRetriever: no index with host ${host} in this project; check the host, or pass index (its name)`
1821
+ );
1822
+ }
1823
+ }
1824
+ if (!info.host) {
1825
+ throw new Error(
1826
+ `PineconeRetriever: index "${info.name || this.index}" has no host yet (is it ready?)`
1827
+ );
1828
+ }
1829
+ this._host = this._host || normalizeHost(info.host);
1830
+ const fields = info.schema && info.schema.fields || null;
1831
+ const named = fields ? Object.entries(fields).filter(([name]) => !name.startsWith("_")) : [];
1832
+ const denseFields = named.filter(([, f]) => f && f.type === "dense_vector");
1833
+ if (this._api === "auto") {
1834
+ this._api = named.length > 0 ? "documents" : "vectors";
1835
+ }
1836
+ if (this._api === "documents") {
1837
+ const field = this._vectorField ? denseFields.find(([name]) => name === this._vectorField) : denseFields[0];
1838
+ if (!field) {
1839
+ const embeds = fields && Object.values(fields).some((f) => f && f.embed);
1840
+ throw new Error(
1841
+ embeds ? `PineconeRetriever: index "${info.name}" embeds text itself (integrated embedding) and has no dense_vector field for your vectors; create an index with a dense_vector field` : `PineconeRetriever: index "${info.name}" has no dense_vector field${this._vectorField ? ` named "${this._vectorField}"` : ""}`
1842
+ );
1843
+ }
1844
+ this._vectorField = field[0];
1845
+ this._indexDimension = field[1].dimension;
1846
+ } else {
1847
+ const values = fields && fields._values;
1848
+ this._indexDimension = info.dimension || values && values.dimension;
1849
+ }
1850
+ }
1851
+ /** A data-plane call; records the log sequence number of writes. @private */
1852
+ async _data(method, path, body, { signal, write = false } = {}) {
1853
+ await this._resolve(signal);
1854
+ const result = await this._call(
1855
+ `${this._host}${path}`,
1856
+ method,
1857
+ body,
1858
+ signal
1859
+ );
1860
+ if (write) {
1861
+ const lsn = Number(result.headers.get("x-pinecone-request-lsn"));
1862
+ if (Number.isFinite(lsn) && lsn > this._writeLsn) this._writeLsn = lsn;
1863
+ }
1864
+ return result;
1865
+ }
1866
+ /** The Documents API path for this namespace. @private */
1867
+ _docs(action) {
1868
+ const ns = encodeURIComponent(this.namespace || "__default__");
1869
+ return `/namespaces/${ns}/documents/${action}`;
1870
+ }
1871
+ /**
1872
+ * A search, repeated until it reflects this retriever's writes (when the
1873
+ * index reports log sequence numbers) or `freshnessTimeoutMs` passes.
1874
+ * Resolves `[{ id, score, fields }]`.
1875
+ * @private
1876
+ */
1877
+ async _search(vector, topK, signal, filter) {
1878
+ await this._resolve(signal);
1879
+ const documents = this._api === "documents";
1880
+ const [path, body] = documents ? [
1881
+ this._docs("search"),
1882
+ {
1883
+ score_by: [
1884
+ {
1885
+ type: "dense_vector",
1886
+ fields: [this._vectorField],
1887
+ values: vector
1888
+ }
1889
+ ],
1890
+ top_k: topK,
1891
+ include_fields: FIELDS,
1892
+ ...filter && { filter }
1893
+ }
1894
+ ] : [
1895
+ "/query",
1896
+ {
1897
+ vector,
1898
+ topK,
1899
+ namespace: this.namespace,
1900
+ includeMetadata: true,
1901
+ ...filter && { filter }
1902
+ }
1903
+ ];
1904
+ const deadline = Date.now() + this.freshnessTimeoutMs;
1905
+ for (let delay = 100; ; delay = Math.min(delay * 2, 2e3)) {
1906
+ let json;
1907
+ let headers;
1908
+ try {
1909
+ ({ json, headers } = await this._data("POST", path, body, { signal }));
1910
+ } catch (err) {
1911
+ if (err.status === 404) return [];
1912
+ throw err;
1913
+ }
1914
+ const header = headers.get("x-pinecone-max-indexed-lsn");
1915
+ const fresh = !this._writeLsn || header === null || !(Number(header) < this._writeLsn) || Date.now() + delay > deadline;
1916
+ if (fresh) {
1917
+ return (json.matches || []).map(
1918
+ (m) => documents ? { id: m._id, score: m._score, fields: m } : { id: m.id, score: m.score, fields: m.metadata }
1919
+ );
1920
+ }
1921
+ await sleep(delay);
1922
+ throwIfAborted(signal);
1923
+ }
1924
+ }
1925
+ /** Waits until this retriever's writes are indexed (for list and size). @private */
1926
+ async _waitForWrites(signal) {
1927
+ if (!this._writeLsn) return;
1928
+ const dimension = this._indexDimension || this._dimension;
1929
+ if (!dimension) return;
1930
+ await this._search(
1931
+ new Array(dimension).fill(0).map((_, i) => i === 0 ? 1 : 0),
1932
+ 1,
1933
+ signal
1934
+ );
1935
+ }
1936
+ /**
1937
+ * Inserts or replaces documents.
1938
+ * @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
1939
+ * @returns {Promise<void>}
1940
+ */
1941
+ async store(docs) {
1942
+ if (!Array.isArray(docs)) {
1943
+ throw new Error("PineconeRetriever.store() takes an array");
1944
+ }
1945
+ if (docs.length === 0) return;
1946
+ await this._resolve();
1947
+ const expected = this._indexDimension || this._dimension;
1948
+ let dimensions = null;
1949
+ const records = docs.map((doc) => {
1950
+ const id = doc && doc.id;
1951
+ if (typeof id !== "string" || id.length === 0) {
1952
+ throw new Error(
1953
+ "PineconeRetriever.store(): every document needs a string id"
1954
+ );
1955
+ }
1956
+ checkVector(doc.vector, id);
1957
+ if (dimensions === null) dimensions = doc.vector.length;
1958
+ else if (doc.vector.length !== dimensions) {
1959
+ throw new Error(
1960
+ `PineconeRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the batch has ${dimensions}`
1961
+ );
1962
+ }
1963
+ if (expected && doc.vector.length !== expected) {
1964
+ throw new Error(
1965
+ `PineconeRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the index has ${expected}`
1966
+ );
1967
+ }
1968
+ const metadata = doc.metadata || {};
1969
+ return {
1970
+ id: recordId(id),
1971
+ values: doc.vector,
1972
+ fields: {
1973
+ rag_id: id,
1974
+ rag_content: doc.content || "",
1975
+ rag_metadata: JSON.stringify(metadata),
1976
+ ...metadata.source !== void 0 && metadata.source !== null && {
1977
+ rag_source: String(metadata.source)
1978
+ }
1979
+ }
1980
+ };
1981
+ });
1982
+ for (let i = 0; i < records.length; i += UPSERT_BATCH) {
1983
+ const batch = records.slice(i, i + UPSERT_BATCH);
1984
+ if (this._api === "documents") {
1985
+ await this._data(
1986
+ "POST",
1987
+ this._docs("upsert"),
1988
+ {
1989
+ documents: batch.map((r) => ({
1990
+ _id: r.id,
1991
+ [this._vectorField]: r.values,
1992
+ ...r.fields
1993
+ }))
1994
+ },
1995
+ { write: true }
1996
+ );
1997
+ } else {
1998
+ await this._data(
1999
+ "POST",
2000
+ "/vectors/upsert",
2001
+ {
2002
+ vectors: batch.map((r) => ({
2003
+ id: r.id,
2004
+ values: r.values,
2005
+ metadata: r.fields
2006
+ })),
2007
+ namespace: this.namespace
2008
+ },
2009
+ { write: true }
2010
+ );
2011
+ }
2012
+ }
2013
+ this._dimension = dimensions;
2014
+ }
2015
+ /**
2016
+ * The documents most similar to a query vector.
2017
+ * @param {number[]|{ queryVector?: number[], topK?: number, signal?: AbortSignal }} request
2018
+ * A vector, or an object with one (required; optional in the type to
2019
+ * match the retriever contract)
2020
+ * @param {number} [k=10] - Results when `request` is a vector
2021
+ * @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
2022
+ */
2023
+ async retrieve(request, k = 10) {
2024
+ const {
2025
+ queryVector,
2026
+ topK = k,
2027
+ signal
2028
+ } = Array.isArray(request) ? { queryVector: request } : request || {};
2029
+ if (!Number.isInteger(topK) || topK < 1) {
2030
+ throw new Error(
2031
+ "PineconeRetriever.retrieve(): topK must be a positive integer"
2032
+ );
2033
+ }
2034
+ checkVector(queryVector, "query");
2035
+ throwIfAborted(signal);
2036
+ const matches = await this._search(
2037
+ queryVector,
2038
+ Math.min(topK, 1e4),
2039
+ signal
2040
+ );
2041
+ throwIfAborted(signal);
2042
+ const seen = /* @__PURE__ */ new Set();
2043
+ const out = [];
2044
+ for (const m of matches) {
2045
+ if (seen.has(m.id)) continue;
2046
+ seen.add(m.id);
2047
+ out.push({ ...fromFields(m.fields, m.id), score: m.score });
2048
+ }
2049
+ return out;
2050
+ }
2051
+ /** @private */
2052
+ async _delete(body) {
2053
+ await this._resolve();
2054
+ try {
2055
+ if (this._api === "documents") {
2056
+ await this._data("POST", this._docs("delete"), body.documents, {
2057
+ write: true
2058
+ });
2059
+ } else {
2060
+ await this._data(
2061
+ "POST",
2062
+ "/vectors/delete",
2063
+ { namespace: this.namespace, ...body.vectors },
2064
+ { write: true }
2065
+ );
2066
+ }
2067
+ } catch (err) {
2068
+ if (err.status !== 404) throw err;
2069
+ }
2070
+ }
2071
+ /**
2072
+ * Deletes the chunks of one source (`metadata.source`), for re-ingesting.
2073
+ * @param {string} source
2074
+ */
2075
+ async removeBySource(source) {
2076
+ const filter = { rag_source: { $eq: String(source) } };
2077
+ await this._delete({ documents: { filter }, vectors: { filter } });
2078
+ }
2079
+ /**
2080
+ * Deletes documents by id.
2081
+ * @param {string[]} ids
2082
+ */
2083
+ async delete(ids) {
2084
+ const recordIds = ids.map(recordId);
2085
+ for (let i = 0; i < recordIds.length; i += DELETE_BATCH) {
2086
+ const batch = recordIds.slice(i, i + DELETE_BATCH);
2087
+ await this._delete({
2088
+ documents: { ids: batch },
2089
+ vectors: { ids: batch }
2090
+ });
2091
+ }
2092
+ }
2093
+ /** Deletes every document this retriever stored in its namespace. */
2094
+ async clear() {
2095
+ await this._delete({
2096
+ documents: { filter: OURS },
2097
+ vectors: { deleteAll: true }
2098
+ });
2099
+ }
2100
+ /**
2101
+ * Every document, without vectors.
2102
+ * @param {{ limit?: number }} [options]
2103
+ * @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
2104
+ */
2105
+ async list({ limit } = {}) {
2106
+ await this._resolve();
2107
+ await this._waitForWrites();
2108
+ return this._api === "documents" ? this._listDocuments(limit) : this._listVectors(limit);
2109
+ }
2110
+ /** @returns {Promise<number>} The documents this retriever stored */
2111
+ async size() {
2112
+ return (await this.list()).length;
2113
+ }
2114
+ /** Documents API: fetch by filter, page by page. @private */
2115
+ async _listDocuments(limit) {
2116
+ const dimension = this._indexDimension || this._dimension;
2117
+ if (dimension) {
2118
+ const max = Math.min(limit || SEARCH_MAX, SEARCH_MAX);
2119
+ const found = await this._search(
2120
+ new Array(dimension).fill(0).map((_, i) => i === 0 ? 1 : 0),
2121
+ max,
2122
+ void 0,
2123
+ OURS
2124
+ );
2125
+ if (found.length < SEARCH_MAX || limit) {
2126
+ const seen = /* @__PURE__ */ new Set();
2127
+ return found.filter((m) => !seen.has(m.id) && seen.add(m.id)).map((m) => fromFields(m.fields, m.id));
2128
+ }
2129
+ }
2130
+ const out = [];
2131
+ let token;
2132
+ do {
2133
+ let json;
2134
+ try {
2135
+ ({ json } = await this._data("POST", this._docs("fetch"), {
2136
+ filter: OURS,
2137
+ include_fields: FIELDS,
2138
+ limit: limit ? Math.min(PAGE, limit - out.length) : PAGE,
2139
+ ...token && { pagination_token: token }
2140
+ }));
2141
+ } catch (err) {
2142
+ if (err.status === 404) break;
2143
+ throw err;
2144
+ }
2145
+ for (const [id, fields] of Object.entries(json.documents || {})) {
2146
+ out.push(fromFields(fields, id));
2147
+ }
2148
+ token = json.pagination && json.pagination.next;
2149
+ } while (token && (!limit || out.length < limit));
2150
+ return limit ? out.slice(0, limit) : out;
2151
+ }
2152
+ /** Vectors API: list ids, then fetch them. @private */
2153
+ async _listVectors(limit) {
2154
+ const ids = [];
2155
+ let token;
2156
+ do {
2157
+ const params = new URLSearchParams({
2158
+ namespace: this.namespace,
2159
+ limit: String(PAGE)
2160
+ });
2161
+ if (token) params.set("paginationToken", token);
2162
+ let json;
2163
+ try {
2164
+ ({ json } = await this._data("GET", `/vectors/list?${params}`));
2165
+ } catch (err) {
2166
+ if (err.status === 404) break;
2167
+ throw err;
2168
+ }
2169
+ for (const v of json.vectors || []) ids.push(v.id);
2170
+ token = json.pagination && json.pagination.next;
2171
+ } while (token && (!limit || ids.length < limit));
2172
+ const wanted = limit ? ids.slice(0, limit) : ids;
2173
+ const out = [];
2174
+ for (let i = 0; i < wanted.length; i += PAGE) {
2175
+ const params = new URLSearchParams({ namespace: this.namespace });
2176
+ for (const id of wanted.slice(i, i + PAGE)) params.append("ids", id);
2177
+ const { json } = await this._data("GET", `/vectors/fetch?${params}`);
2178
+ for (const id of wanted.slice(i, i + PAGE)) {
2179
+ const record = json.vectors && json.vectors[id];
2180
+ if (record) out.push(fromFields(record.metadata, id));
2181
+ }
2182
+ }
2183
+ return out;
2184
+ }
2185
+ };
2186
+ module2.exports = { PineconeRetriever, recordId, API_VERSION };
2187
+ }
2188
+ });
2189
+
1611
2190
  // src/retrieval/pgvector-retriever.js
1612
2191
  var require_pgvector_retriever = __commonJS({
1613
2192
  "src/retrieval/pgvector-retriever.js"(exports2, module2) {
@@ -2119,6 +2698,7 @@ var require_connectors = __commonJS({
2119
2698
  var { OllamaConnector } = require_ollama_connector();
2120
2699
  var { LocalEmbedder } = require_local_embedder();
2121
2700
  var { QdrantRetriever } = require_qdrant_retriever();
2701
+ var { PineconeRetriever } = require_pinecone_retriever();
2122
2702
  var { PgVectorRetriever } = require_pgvector_retriever();
2123
2703
  var { createAiSdkAdapter } = require_ai_sdk();
2124
2704
  module2.exports = {
@@ -2128,6 +2708,7 @@ var require_connectors = __commonJS({
2128
2708
  OllamaConnector,
2129
2709
  LocalEmbedder,
2130
2710
  QdrantRetriever,
2711
+ PineconeRetriever,
2131
2712
  PgVectorRetriever,
2132
2713
  createAiSdkAdapter
2133
2714
  };
@@ -2147,6 +2728,7 @@ var modules = {
2147
2728
  "connectors/rag-prompt.js": () => require_rag_prompt(),
2148
2729
  "packages/connectors.js": () => require_connectors(),
2149
2730
  "retrieval/pgvector-retriever.js": () => require_pgvector_retriever(),
2731
+ "retrieval/pinecone-retriever.js": () => require_pinecone_retriever(),
2150
2732
  "retrieval/qdrant-retriever.js": () => require_qdrant_retriever()
2151
2733
  };
2152
2734
  Object.defineProperty(mod, Symbol.for("@devilsdev/rag-connectors/internal"), {
package/dist/index.d.ts CHANGED
@@ -1 +1 @@
1
- export { OpenAIConnector, AnthropicConnector, CohereConnector, OllamaConnector, LocalEmbedder, QdrantRetriever, PgVectorRetriever, createAiSdkAdapter } from "@devilsdev/rag-core/all-types";
1
+ export { OpenAIConnector, AnthropicConnector, CohereConnector, OllamaConnector, LocalEmbedder, QdrantRetriever, PineconeRetriever, PgVectorRetriever, createAiSdkAdapter } from "@devilsdev/rag-core/all-types";
package/dist/index.mjs CHANGED
@@ -1,5 +1,5 @@
1
1
  import { createRequire } from 'module';
2
2
  const require = createRequire(import.meta.url);
3
3
  const mod = require('./index.cjs');
4
- export const { OpenAIConnector, AnthropicConnector, CohereConnector, OllamaConnector, LocalEmbedder, QdrantRetriever, PgVectorRetriever, createAiSdkAdapter } = mod;
4
+ export const { OpenAIConnector, AnthropicConnector, CohereConnector, OllamaConnector, LocalEmbedder, QdrantRetriever, PineconeRetriever, PgVectorRetriever, createAiSdkAdapter } = mod;
5
5
  export default mod;
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@devilsdev/rag-connectors",
3
- "version": "3.5.0",
4
- "description": "Embedders, LLMs and vector stores for @devilsdev/rag-core: OpenAI, Anthropic, Cohere, Ollama, the AI SDK, Qdrant and pgvector.",
3
+ "version": "3.6.0",
4
+ "description": "Embedders, LLMs and vector stores for @devilsdev/rag-core: OpenAI, Anthropic, Cohere, Ollama, the AI SDK, Qdrant, pgvector and Pinecone.",
5
5
  "keywords": [
6
6
  "rag",
7
7
  "openai",
@@ -28,6 +28,7 @@
28
28
  "exports": {
29
29
  ".": {
30
30
  "types": "./dist/index.d.ts",
31
+ "workerd": "./dist/index.cjs",
31
32
  "require": "./dist/index.cjs",
32
33
  "import": "./dist/index.mjs"
33
34
  },
@@ -43,7 +44,7 @@
43
44
  "access": "public"
44
45
  },
45
46
  "dependencies": {
46
- "@devilsdev/rag-core": "3.5.0",
47
+ "@devilsdev/rag-core": "3.6.0",
47
48
  "axios": "^1.21.0"
48
49
  },
49
50
  "peerDependencies": {