@devilsdev/rag-connectors 3.5.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.cjs +589 -7
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +4 -3
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @devilsdev/rag-connectors
|
|
2
2
|
|
|
3
|
-
Embedders, LLMs and vector stores for @devilsdev/rag-core: OpenAI, Anthropic, Cohere, Ollama, the AI SDK, Qdrant and
|
|
3
|
+
Embedders, LLMs and vector stores for @devilsdev/rag-core: OpenAI, Anthropic, Cohere, Ollama, the AI SDK, Qdrant, pgvector and Pinecone.
|
|
4
4
|
|
|
5
5
|
Part of [@devilsdev/rag-pipeline-utils](https://www.npmjs.com/package/@devilsdev/rag-pipeline-utils), split into packages so you install only what you use. The objects are the same ones the full package exports; versions move in lockstep.
|
|
6
6
|
|
|
@@ -8,7 +8,7 @@ Part of [@devilsdev/rag-pipeline-utils](https://www.npmjs.com/package/@devilsdev
|
|
|
8
8
|
npm install @devilsdev/rag-connectors
|
|
9
9
|
```
|
|
10
10
|
|
|
11
|
-
Exports: `OpenAIConnector`, `AnthropicConnector`, `CohereConnector`, `OllamaConnector`, `LocalEmbedder`, `QdrantRetriever`, `PgVectorRetriever`, `createAiSdkAdapter`.
|
|
11
|
+
Exports: `OpenAIConnector`, `AnthropicConnector`, `CohereConnector`, `OllamaConnector`, `LocalEmbedder`, `QdrantRetriever`, `PineconeRetriever`, `PgVectorRetriever`, `createAiSdkAdapter`.
|
|
12
12
|
|
|
13
13
|
Documentation: https://devilsdev.github.io/rag-pipeline-utils/
|
|
14
14
|
|
package/dist/index.cjs
CHANGED
|
@@ -99,7 +99,7 @@ var require_openai_connector = __commonJS({
|
|
|
99
99
|
"src/connectors/openai-connector.js"(exports2, module2) {
|
|
100
100
|
"use strict";
|
|
101
101
|
var { BaseConnector } = require_base_connector();
|
|
102
|
-
var {
|
|
102
|
+
var { missingPeer } = require_optional_peer();
|
|
103
103
|
var {
|
|
104
104
|
missingApiKeyError,
|
|
105
105
|
raceAbort,
|
|
@@ -121,12 +121,15 @@ var require_openai_connector = __commonJS({
|
|
|
121
121
|
}
|
|
122
122
|
};
|
|
123
123
|
function loadOpenAI() {
|
|
124
|
-
|
|
125
|
-
"openai"
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
124
|
+
try {
|
|
125
|
+
return require("openai");
|
|
126
|
+
} catch (error) {
|
|
127
|
+
throw missingPeer(
|
|
128
|
+
error,
|
|
129
|
+
"openai",
|
|
130
|
+
'OpenAIConnector needs the "openai" package, an optional peer dependency since 3.0.0: npm install openai'
|
|
131
|
+
);
|
|
132
|
+
}
|
|
130
133
|
}
|
|
131
134
|
var OpenAIConnector = class extends BaseConnector {
|
|
132
135
|
/**
|
|
@@ -1608,6 +1611,582 @@ var require_qdrant_retriever = __commonJS({
|
|
|
1608
1611
|
}
|
|
1609
1612
|
});
|
|
1610
1613
|
|
|
1614
|
+
// src/retrieval/pinecone-retriever.js
|
|
1615
|
+
var require_pinecone_retriever = __commonJS({
|
|
1616
|
+
"src/retrieval/pinecone-retriever.js"(exports2, module2) {
|
|
1617
|
+
"use strict";
|
|
1618
|
+
var crypto = require("crypto");
|
|
1619
|
+
var { throwIfAborted } = require_connector_utils();
|
|
1620
|
+
var API_VERSION = "2026-07";
|
|
1621
|
+
var CONTROL_PLANE = "https://api.pinecone.io";
|
|
1622
|
+
var DEFAULT_NAMESPACE = "rag_documents";
|
|
1623
|
+
var UPSERT_BATCH = 100;
|
|
1624
|
+
var DELETE_BATCH = 1e3;
|
|
1625
|
+
var PAGE = 100;
|
|
1626
|
+
var SEARCH_MAX = 1e4;
|
|
1627
|
+
var PLAIN_ID = /^[\x20-\x7e]{1,512}$/;
|
|
1628
|
+
var FIELDS = ["rag_id", "rag_content", "rag_metadata"];
|
|
1629
|
+
var OURS = { rag_id: { $exists: true } };
|
|
1630
|
+
function recordId(id) {
|
|
1631
|
+
const s = String(id);
|
|
1632
|
+
if (PLAIN_ID.test(s)) return s;
|
|
1633
|
+
return `rag-${crypto.createHash("sha256").update(s).digest("hex")}`;
|
|
1634
|
+
}
|
|
1635
|
+
function checkVector(vector, id) {
|
|
1636
|
+
if (!Array.isArray(vector) || vector.length === 0) {
|
|
1637
|
+
throw new Error(`PineconeRetriever: document "${id}" has no vector`);
|
|
1638
|
+
}
|
|
1639
|
+
if (!vector.every((x) => typeof x === "number" && Number.isFinite(x))) {
|
|
1640
|
+
throw new Error(
|
|
1641
|
+
`PineconeRetriever: document "${id}" has a vector with a non-finite value`
|
|
1642
|
+
);
|
|
1643
|
+
}
|
|
1644
|
+
}
|
|
1645
|
+
function fromFields(fields, fallbackId) {
|
|
1646
|
+
const m = fields || {};
|
|
1647
|
+
let metadata = {};
|
|
1648
|
+
try {
|
|
1649
|
+
metadata = m.rag_metadata ? JSON.parse(m.rag_metadata) : {};
|
|
1650
|
+
} catch {
|
|
1651
|
+
metadata = {};
|
|
1652
|
+
}
|
|
1653
|
+
return {
|
|
1654
|
+
id: m.rag_id !== void 0 ? String(m.rag_id) : fallbackId,
|
|
1655
|
+
content: m.rag_content || "",
|
|
1656
|
+
metadata
|
|
1657
|
+
};
|
|
1658
|
+
}
|
|
1659
|
+
function normalizeHost(host) {
|
|
1660
|
+
let h = String(host);
|
|
1661
|
+
while (h.endsWith("/")) h = h.slice(0, -1);
|
|
1662
|
+
return /^https?:\/\//.test(h) ? h : `https://${h}`;
|
|
1663
|
+
}
|
|
1664
|
+
var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1665
|
+
var PineconeRetriever = class {
|
|
1666
|
+
/**
|
|
1667
|
+
* @param {Object} options
|
|
1668
|
+
* @param {string} [options.apiKey] - Defaults to PINECONE_API_KEY
|
|
1669
|
+
* @param {string} [options.index] - The index name, described on first use
|
|
1670
|
+
* @param {string} [options.host] - The index host (from the console), instead of or with `index`
|
|
1671
|
+
* @param {string} [options.namespace='rag_documents'] - A namespace this retriever owns
|
|
1672
|
+
* @param {'auto'|'documents'|'vectors'} [options.api='auto'] - Document index (Documents API) or vector index; auto reads it from the index description
|
|
1673
|
+
* @param {string} [options.vectorField] - The document index's dense_vector field (default: the schema's)
|
|
1674
|
+
* @param {number} [options.freshnessTimeoutMs=30000] - How long a read waits for this retriever's writes to be indexed; 0 to not wait
|
|
1675
|
+
* @param {string} [options.cacheKey] - Cache identity; change it when the index changes
|
|
1676
|
+
* @param {Function} [options.fetch] - fetch implementation (defaults to the global)
|
|
1677
|
+
*/
|
|
1678
|
+
constructor(options = {}) {
|
|
1679
|
+
const {
|
|
1680
|
+
apiKey = typeof process !== "undefined" && process.env ? process.env.PINECONE_API_KEY : void 0,
|
|
1681
|
+
host,
|
|
1682
|
+
index,
|
|
1683
|
+
namespace = DEFAULT_NAMESPACE,
|
|
1684
|
+
api = "auto",
|
|
1685
|
+
vectorField,
|
|
1686
|
+
freshnessTimeoutMs = 3e4,
|
|
1687
|
+
cacheKey,
|
|
1688
|
+
fetch: fetchImpl = globalThis.fetch
|
|
1689
|
+
} = options;
|
|
1690
|
+
if (!apiKey) {
|
|
1691
|
+
throw new Error(
|
|
1692
|
+
"PineconeRetriever needs an API key: pass apiKey or set PINECONE_API_KEY"
|
|
1693
|
+
);
|
|
1694
|
+
}
|
|
1695
|
+
if (!host && !index) {
|
|
1696
|
+
throw new Error(
|
|
1697
|
+
"PineconeRetriever needs the index: pass index (its name) or host (the index host)"
|
|
1698
|
+
);
|
|
1699
|
+
}
|
|
1700
|
+
if (typeof namespace !== "string") {
|
|
1701
|
+
throw new Error("PineconeRetriever: namespace must be a string");
|
|
1702
|
+
}
|
|
1703
|
+
if (!["auto", "documents", "vectors"].includes(api)) {
|
|
1704
|
+
throw new Error(
|
|
1705
|
+
'PineconeRetriever: api must be "auto", "documents" or "vectors"'
|
|
1706
|
+
);
|
|
1707
|
+
}
|
|
1708
|
+
if (typeof fetchImpl !== "function") {
|
|
1709
|
+
throw new Error("PineconeRetriever needs fetch (Node.js 18 or later)");
|
|
1710
|
+
}
|
|
1711
|
+
this.index = index;
|
|
1712
|
+
this.namespace = namespace;
|
|
1713
|
+
this.freshnessTimeoutMs = freshnessTimeoutMs;
|
|
1714
|
+
this.cacheKey = cacheKey;
|
|
1715
|
+
this._apiKey = apiKey;
|
|
1716
|
+
this._host = host ? normalizeHost(host) : null;
|
|
1717
|
+
this._api = api;
|
|
1718
|
+
this._vectorField = vectorField;
|
|
1719
|
+
this._fetch = fetchImpl === globalThis.fetch ? fetchImpl.bind(globalThis) : fetchImpl;
|
|
1720
|
+
this._resolving = null;
|
|
1721
|
+
this._writeLsn = 0;
|
|
1722
|
+
this._dimension = null;
|
|
1723
|
+
}
|
|
1724
|
+
/**
|
|
1725
|
+
* Cache identity for createRagPipeline({ cache }): none unless you set
|
|
1726
|
+
* `cacheKey`, since other clients can change the index.
|
|
1727
|
+
* @param {'retriever'} role
|
|
1728
|
+
* @returns {string|null}
|
|
1729
|
+
*/
|
|
1730
|
+
getCacheKey(role) {
|
|
1731
|
+
if (role !== "retriever" || !this.cacheKey) return null;
|
|
1732
|
+
return `pinecone|${this._host || this.index}|${this.namespace}|${this.cacheKey}`;
|
|
1733
|
+
}
|
|
1734
|
+
/**
|
|
1735
|
+
* One REST call; resolves the parsed body and the response headers.
|
|
1736
|
+
* @private
|
|
1737
|
+
*/
|
|
1738
|
+
async _call(url, method, body, signal) {
|
|
1739
|
+
throwIfAborted(signal);
|
|
1740
|
+
const headers = {
|
|
1741
|
+
"Api-Key": this._apiKey,
|
|
1742
|
+
"X-Pinecone-Api-Version": API_VERSION,
|
|
1743
|
+
accept: "application/json"
|
|
1744
|
+
};
|
|
1745
|
+
if (body !== void 0) headers["content-type"] = "application/json";
|
|
1746
|
+
let res;
|
|
1747
|
+
try {
|
|
1748
|
+
res = await this._fetch(url, {
|
|
1749
|
+
method,
|
|
1750
|
+
headers,
|
|
1751
|
+
...body !== void 0 && { body: JSON.stringify(body) },
|
|
1752
|
+
...signal && { signal }
|
|
1753
|
+
});
|
|
1754
|
+
} catch (err) {
|
|
1755
|
+
throwIfAborted(signal);
|
|
1756
|
+
throw new Error(
|
|
1757
|
+
`PineconeRetriever: cannot reach Pinecone (${err.message})`
|
|
1758
|
+
);
|
|
1759
|
+
}
|
|
1760
|
+
const text = await res.text();
|
|
1761
|
+
let json;
|
|
1762
|
+
try {
|
|
1763
|
+
json = text ? JSON.parse(text) : {};
|
|
1764
|
+
} catch {
|
|
1765
|
+
json = {};
|
|
1766
|
+
}
|
|
1767
|
+
if (!res.ok) {
|
|
1768
|
+
if (res.status === 401 || res.status === 403) {
|
|
1769
|
+
throw new Error(
|
|
1770
|
+
`PineconeRetriever: Pinecone refused the request (${res.status}); check PINECONE_API_KEY`
|
|
1771
|
+
);
|
|
1772
|
+
}
|
|
1773
|
+
const detail = json.message || json.error && json.error.message || text || res.statusText;
|
|
1774
|
+
const err = new Error(
|
|
1775
|
+
`PineconeRetriever: ${method} ${new URL(url).pathname} failed (${res.status}): ${detail}`
|
|
1776
|
+
);
|
|
1777
|
+
err.status = res.status;
|
|
1778
|
+
throw err;
|
|
1779
|
+
}
|
|
1780
|
+
return { json, headers: res.headers };
|
|
1781
|
+
}
|
|
1782
|
+
/**
|
|
1783
|
+
* The index's host and kind, from its description (once): a document
|
|
1784
|
+
* index's dense_vector field and dimension, or a vector index's.
|
|
1785
|
+
* @private
|
|
1786
|
+
*/
|
|
1787
|
+
_resolve(signal) {
|
|
1788
|
+
if (this._host && this._api !== "auto" && (this._api === "vectors" || this._vectorField)) {
|
|
1789
|
+
return Promise.resolve();
|
|
1790
|
+
}
|
|
1791
|
+
if (!this._resolving) {
|
|
1792
|
+
this._resolving = this._describe(signal).catch((err) => {
|
|
1793
|
+
this._resolving = null;
|
|
1794
|
+
throw err;
|
|
1795
|
+
});
|
|
1796
|
+
}
|
|
1797
|
+
return this._resolving;
|
|
1798
|
+
}
|
|
1799
|
+
/** @private */
|
|
1800
|
+
async _describe(signal) {
|
|
1801
|
+
let info;
|
|
1802
|
+
if (this.index) {
|
|
1803
|
+
({ json: info } = await this._call(
|
|
1804
|
+
`${CONTROL_PLANE}/indexes/${encodeURIComponent(this.index)}`,
|
|
1805
|
+
"GET",
|
|
1806
|
+
void 0,
|
|
1807
|
+
signal
|
|
1808
|
+
));
|
|
1809
|
+
} else {
|
|
1810
|
+
const { json } = await this._call(
|
|
1811
|
+
`${CONTROL_PLANE}/indexes`,
|
|
1812
|
+
"GET",
|
|
1813
|
+
void 0,
|
|
1814
|
+
signal
|
|
1815
|
+
);
|
|
1816
|
+
const host = this._host.replace(/^https?:\/\//, "");
|
|
1817
|
+
info = (json.indexes || []).find((i) => i.host === host);
|
|
1818
|
+
if (!info) {
|
|
1819
|
+
throw new Error(
|
|
1820
|
+
`PineconeRetriever: no index with host ${host} in this project; check the host, or pass index (its name)`
|
|
1821
|
+
);
|
|
1822
|
+
}
|
|
1823
|
+
}
|
|
1824
|
+
if (!info.host) {
|
|
1825
|
+
throw new Error(
|
|
1826
|
+
`PineconeRetriever: index "${info.name || this.index}" has no host yet (is it ready?)`
|
|
1827
|
+
);
|
|
1828
|
+
}
|
|
1829
|
+
this._host = this._host || normalizeHost(info.host);
|
|
1830
|
+
const fields = info.schema && info.schema.fields || null;
|
|
1831
|
+
const named = fields ? Object.entries(fields).filter(([name]) => !name.startsWith("_")) : [];
|
|
1832
|
+
const denseFields = named.filter(([, f]) => f && f.type === "dense_vector");
|
|
1833
|
+
if (this._api === "auto") {
|
|
1834
|
+
this._api = named.length > 0 ? "documents" : "vectors";
|
|
1835
|
+
}
|
|
1836
|
+
if (this._api === "documents") {
|
|
1837
|
+
const field = this._vectorField ? denseFields.find(([name]) => name === this._vectorField) : denseFields[0];
|
|
1838
|
+
if (!field) {
|
|
1839
|
+
const embeds = fields && Object.values(fields).some((f) => f && f.embed);
|
|
1840
|
+
throw new Error(
|
|
1841
|
+
embeds ? `PineconeRetriever: index "${info.name}" embeds text itself (integrated embedding) and has no dense_vector field for your vectors; create an index with a dense_vector field` : `PineconeRetriever: index "${info.name}" has no dense_vector field${this._vectorField ? ` named "${this._vectorField}"` : ""}`
|
|
1842
|
+
);
|
|
1843
|
+
}
|
|
1844
|
+
this._vectorField = field[0];
|
|
1845
|
+
this._indexDimension = field[1].dimension;
|
|
1846
|
+
} else {
|
|
1847
|
+
const values = fields && fields._values;
|
|
1848
|
+
this._indexDimension = info.dimension || values && values.dimension;
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
/** A data-plane call; records the log sequence number of writes. @private */
|
|
1852
|
+
async _data(method, path, body, { signal, write = false } = {}) {
|
|
1853
|
+
await this._resolve(signal);
|
|
1854
|
+
const result = await this._call(
|
|
1855
|
+
`${this._host}${path}`,
|
|
1856
|
+
method,
|
|
1857
|
+
body,
|
|
1858
|
+
signal
|
|
1859
|
+
);
|
|
1860
|
+
if (write) {
|
|
1861
|
+
const lsn = Number(result.headers.get("x-pinecone-request-lsn"));
|
|
1862
|
+
if (Number.isFinite(lsn) && lsn > this._writeLsn) this._writeLsn = lsn;
|
|
1863
|
+
}
|
|
1864
|
+
return result;
|
|
1865
|
+
}
|
|
1866
|
+
/** The Documents API path for this namespace. @private */
|
|
1867
|
+
_docs(action) {
|
|
1868
|
+
const ns = encodeURIComponent(this.namespace || "__default__");
|
|
1869
|
+
return `/namespaces/${ns}/documents/${action}`;
|
|
1870
|
+
}
|
|
1871
|
+
/**
|
|
1872
|
+
* A search, repeated until it reflects this retriever's writes (when the
|
|
1873
|
+
* index reports log sequence numbers) or `freshnessTimeoutMs` passes.
|
|
1874
|
+
* Resolves `[{ id, score, fields }]`.
|
|
1875
|
+
* @private
|
|
1876
|
+
*/
|
|
1877
|
+
async _search(vector, topK, signal, filter) {
|
|
1878
|
+
await this._resolve(signal);
|
|
1879
|
+
const documents = this._api === "documents";
|
|
1880
|
+
const [path, body] = documents ? [
|
|
1881
|
+
this._docs("search"),
|
|
1882
|
+
{
|
|
1883
|
+
score_by: [
|
|
1884
|
+
{
|
|
1885
|
+
type: "dense_vector",
|
|
1886
|
+
fields: [this._vectorField],
|
|
1887
|
+
values: vector
|
|
1888
|
+
}
|
|
1889
|
+
],
|
|
1890
|
+
top_k: topK,
|
|
1891
|
+
include_fields: FIELDS,
|
|
1892
|
+
...filter && { filter }
|
|
1893
|
+
}
|
|
1894
|
+
] : [
|
|
1895
|
+
"/query",
|
|
1896
|
+
{
|
|
1897
|
+
vector,
|
|
1898
|
+
topK,
|
|
1899
|
+
namespace: this.namespace,
|
|
1900
|
+
includeMetadata: true,
|
|
1901
|
+
...filter && { filter }
|
|
1902
|
+
}
|
|
1903
|
+
];
|
|
1904
|
+
const deadline = Date.now() + this.freshnessTimeoutMs;
|
|
1905
|
+
for (let delay = 100; ; delay = Math.min(delay * 2, 2e3)) {
|
|
1906
|
+
let json;
|
|
1907
|
+
let headers;
|
|
1908
|
+
try {
|
|
1909
|
+
({ json, headers } = await this._data("POST", path, body, { signal }));
|
|
1910
|
+
} catch (err) {
|
|
1911
|
+
if (err.status === 404) return [];
|
|
1912
|
+
throw err;
|
|
1913
|
+
}
|
|
1914
|
+
const header = headers.get("x-pinecone-max-indexed-lsn");
|
|
1915
|
+
const fresh = !this._writeLsn || header === null || !(Number(header) < this._writeLsn) || Date.now() + delay > deadline;
|
|
1916
|
+
if (fresh) {
|
|
1917
|
+
return (json.matches || []).map(
|
|
1918
|
+
(m) => documents ? { id: m._id, score: m._score, fields: m } : { id: m.id, score: m.score, fields: m.metadata }
|
|
1919
|
+
);
|
|
1920
|
+
}
|
|
1921
|
+
await sleep(delay);
|
|
1922
|
+
throwIfAborted(signal);
|
|
1923
|
+
}
|
|
1924
|
+
}
|
|
1925
|
+
/** Waits until this retriever's writes are indexed (for list and size). @private */
|
|
1926
|
+
async _waitForWrites(signal) {
|
|
1927
|
+
if (!this._writeLsn) return;
|
|
1928
|
+
const dimension = this._indexDimension || this._dimension;
|
|
1929
|
+
if (!dimension) return;
|
|
1930
|
+
await this._search(
|
|
1931
|
+
new Array(dimension).fill(0).map((_, i) => i === 0 ? 1 : 0),
|
|
1932
|
+
1,
|
|
1933
|
+
signal
|
|
1934
|
+
);
|
|
1935
|
+
}
|
|
1936
|
+
/**
|
|
1937
|
+
* Inserts or replaces documents.
|
|
1938
|
+
* @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
|
|
1939
|
+
* @returns {Promise<void>}
|
|
1940
|
+
*/
|
|
1941
|
+
async store(docs) {
|
|
1942
|
+
if (!Array.isArray(docs)) {
|
|
1943
|
+
throw new Error("PineconeRetriever.store() takes an array");
|
|
1944
|
+
}
|
|
1945
|
+
if (docs.length === 0) return;
|
|
1946
|
+
await this._resolve();
|
|
1947
|
+
const expected = this._indexDimension || this._dimension;
|
|
1948
|
+
let dimensions = null;
|
|
1949
|
+
const records = docs.map((doc) => {
|
|
1950
|
+
const id = doc && doc.id;
|
|
1951
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
1952
|
+
throw new Error(
|
|
1953
|
+
"PineconeRetriever.store(): every document needs a string id"
|
|
1954
|
+
);
|
|
1955
|
+
}
|
|
1956
|
+
checkVector(doc.vector, id);
|
|
1957
|
+
if (dimensions === null) dimensions = doc.vector.length;
|
|
1958
|
+
else if (doc.vector.length !== dimensions) {
|
|
1959
|
+
throw new Error(
|
|
1960
|
+
`PineconeRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the batch has ${dimensions}`
|
|
1961
|
+
);
|
|
1962
|
+
}
|
|
1963
|
+
if (expected && doc.vector.length !== expected) {
|
|
1964
|
+
throw new Error(
|
|
1965
|
+
`PineconeRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the index has ${expected}`
|
|
1966
|
+
);
|
|
1967
|
+
}
|
|
1968
|
+
const metadata = doc.metadata || {};
|
|
1969
|
+
return {
|
|
1970
|
+
id: recordId(id),
|
|
1971
|
+
values: doc.vector,
|
|
1972
|
+
fields: {
|
|
1973
|
+
rag_id: id,
|
|
1974
|
+
rag_content: doc.content || "",
|
|
1975
|
+
rag_metadata: JSON.stringify(metadata),
|
|
1976
|
+
...metadata.source !== void 0 && metadata.source !== null && {
|
|
1977
|
+
rag_source: String(metadata.source)
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
};
|
|
1981
|
+
});
|
|
1982
|
+
for (let i = 0; i < records.length; i += UPSERT_BATCH) {
|
|
1983
|
+
const batch = records.slice(i, i + UPSERT_BATCH);
|
|
1984
|
+
if (this._api === "documents") {
|
|
1985
|
+
await this._data(
|
|
1986
|
+
"POST",
|
|
1987
|
+
this._docs("upsert"),
|
|
1988
|
+
{
|
|
1989
|
+
documents: batch.map((r) => ({
|
|
1990
|
+
_id: r.id,
|
|
1991
|
+
[this._vectorField]: r.values,
|
|
1992
|
+
...r.fields
|
|
1993
|
+
}))
|
|
1994
|
+
},
|
|
1995
|
+
{ write: true }
|
|
1996
|
+
);
|
|
1997
|
+
} else {
|
|
1998
|
+
await this._data(
|
|
1999
|
+
"POST",
|
|
2000
|
+
"/vectors/upsert",
|
|
2001
|
+
{
|
|
2002
|
+
vectors: batch.map((r) => ({
|
|
2003
|
+
id: r.id,
|
|
2004
|
+
values: r.values,
|
|
2005
|
+
metadata: r.fields
|
|
2006
|
+
})),
|
|
2007
|
+
namespace: this.namespace
|
|
2008
|
+
},
|
|
2009
|
+
{ write: true }
|
|
2010
|
+
);
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
this._dimension = dimensions;
|
|
2014
|
+
}
|
|
2015
|
+
/**
|
|
2016
|
+
* The documents most similar to a query vector.
|
|
2017
|
+
* @param {number[]|{ queryVector?: number[], topK?: number, signal?: AbortSignal }} request
|
|
2018
|
+
* A vector, or an object with one (required; optional in the type to
|
|
2019
|
+
* match the retriever contract)
|
|
2020
|
+
* @param {number} [k=10] - Results when `request` is a vector
|
|
2021
|
+
* @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
|
|
2022
|
+
*/
|
|
2023
|
+
async retrieve(request, k = 10) {
|
|
2024
|
+
const {
|
|
2025
|
+
queryVector,
|
|
2026
|
+
topK = k,
|
|
2027
|
+
signal
|
|
2028
|
+
} = Array.isArray(request) ? { queryVector: request } : request || {};
|
|
2029
|
+
if (!Number.isInteger(topK) || topK < 1) {
|
|
2030
|
+
throw new Error(
|
|
2031
|
+
"PineconeRetriever.retrieve(): topK must be a positive integer"
|
|
2032
|
+
);
|
|
2033
|
+
}
|
|
2034
|
+
checkVector(queryVector, "query");
|
|
2035
|
+
throwIfAborted(signal);
|
|
2036
|
+
const matches = await this._search(
|
|
2037
|
+
queryVector,
|
|
2038
|
+
Math.min(topK, 1e4),
|
|
2039
|
+
signal
|
|
2040
|
+
);
|
|
2041
|
+
throwIfAborted(signal);
|
|
2042
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2043
|
+
const out = [];
|
|
2044
|
+
for (const m of matches) {
|
|
2045
|
+
if (seen.has(m.id)) continue;
|
|
2046
|
+
seen.add(m.id);
|
|
2047
|
+
out.push({ ...fromFields(m.fields, m.id), score: m.score });
|
|
2048
|
+
}
|
|
2049
|
+
return out;
|
|
2050
|
+
}
|
|
2051
|
+
/** @private */
|
|
2052
|
+
async _delete(body) {
|
|
2053
|
+
await this._resolve();
|
|
2054
|
+
try {
|
|
2055
|
+
if (this._api === "documents") {
|
|
2056
|
+
await this._data("POST", this._docs("delete"), body.documents, {
|
|
2057
|
+
write: true
|
|
2058
|
+
});
|
|
2059
|
+
} else {
|
|
2060
|
+
await this._data(
|
|
2061
|
+
"POST",
|
|
2062
|
+
"/vectors/delete",
|
|
2063
|
+
{ namespace: this.namespace, ...body.vectors },
|
|
2064
|
+
{ write: true }
|
|
2065
|
+
);
|
|
2066
|
+
}
|
|
2067
|
+
} catch (err) {
|
|
2068
|
+
if (err.status !== 404) throw err;
|
|
2069
|
+
}
|
|
2070
|
+
}
|
|
2071
|
+
/**
|
|
2072
|
+
* Deletes the chunks of one source (`metadata.source`), for re-ingesting.
|
|
2073
|
+
* @param {string} source
|
|
2074
|
+
*/
|
|
2075
|
+
async removeBySource(source) {
|
|
2076
|
+
const filter = { rag_source: { $eq: String(source) } };
|
|
2077
|
+
await this._delete({ documents: { filter }, vectors: { filter } });
|
|
2078
|
+
}
|
|
2079
|
+
/**
|
|
2080
|
+
* Deletes documents by id.
|
|
2081
|
+
* @param {string[]} ids
|
|
2082
|
+
*/
|
|
2083
|
+
async delete(ids) {
|
|
2084
|
+
const recordIds = ids.map(recordId);
|
|
2085
|
+
for (let i = 0; i < recordIds.length; i += DELETE_BATCH) {
|
|
2086
|
+
const batch = recordIds.slice(i, i + DELETE_BATCH);
|
|
2087
|
+
await this._delete({
|
|
2088
|
+
documents: { ids: batch },
|
|
2089
|
+
vectors: { ids: batch }
|
|
2090
|
+
});
|
|
2091
|
+
}
|
|
2092
|
+
}
|
|
2093
|
+
/** Deletes every document this retriever stored in its namespace. */
|
|
2094
|
+
async clear() {
|
|
2095
|
+
await this._delete({
|
|
2096
|
+
documents: { filter: OURS },
|
|
2097
|
+
vectors: { deleteAll: true }
|
|
2098
|
+
});
|
|
2099
|
+
}
|
|
2100
|
+
/**
|
|
2101
|
+
* Every document, without vectors.
|
|
2102
|
+
* @param {{ limit?: number }} [options]
|
|
2103
|
+
* @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
|
|
2104
|
+
*/
|
|
2105
|
+
async list({ limit } = {}) {
|
|
2106
|
+
await this._resolve();
|
|
2107
|
+
await this._waitForWrites();
|
|
2108
|
+
return this._api === "documents" ? this._listDocuments(limit) : this._listVectors(limit);
|
|
2109
|
+
}
|
|
2110
|
+
/** @returns {Promise<number>} The documents this retriever stored */
|
|
2111
|
+
async size() {
|
|
2112
|
+
return (await this.list()).length;
|
|
2113
|
+
}
|
|
2114
|
+
/** Documents API: fetch by filter, page by page. @private */
|
|
2115
|
+
async _listDocuments(limit) {
|
|
2116
|
+
const dimension = this._indexDimension || this._dimension;
|
|
2117
|
+
if (dimension) {
|
|
2118
|
+
const max = Math.min(limit || SEARCH_MAX, SEARCH_MAX);
|
|
2119
|
+
const found = await this._search(
|
|
2120
|
+
new Array(dimension).fill(0).map((_, i) => i === 0 ? 1 : 0),
|
|
2121
|
+
max,
|
|
2122
|
+
void 0,
|
|
2123
|
+
OURS
|
|
2124
|
+
);
|
|
2125
|
+
if (found.length < SEARCH_MAX || limit) {
|
|
2126
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2127
|
+
return found.filter((m) => !seen.has(m.id) && seen.add(m.id)).map((m) => fromFields(m.fields, m.id));
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
2130
|
+
const out = [];
|
|
2131
|
+
let token;
|
|
2132
|
+
do {
|
|
2133
|
+
let json;
|
|
2134
|
+
try {
|
|
2135
|
+
({ json } = await this._data("POST", this._docs("fetch"), {
|
|
2136
|
+
filter: OURS,
|
|
2137
|
+
include_fields: FIELDS,
|
|
2138
|
+
limit: limit ? Math.min(PAGE, limit - out.length) : PAGE,
|
|
2139
|
+
...token && { pagination_token: token }
|
|
2140
|
+
}));
|
|
2141
|
+
} catch (err) {
|
|
2142
|
+
if (err.status === 404) break;
|
|
2143
|
+
throw err;
|
|
2144
|
+
}
|
|
2145
|
+
for (const [id, fields] of Object.entries(json.documents || {})) {
|
|
2146
|
+
out.push(fromFields(fields, id));
|
|
2147
|
+
}
|
|
2148
|
+
token = json.pagination && json.pagination.next;
|
|
2149
|
+
} while (token && (!limit || out.length < limit));
|
|
2150
|
+
return limit ? out.slice(0, limit) : out;
|
|
2151
|
+
}
|
|
2152
|
+
/** Vectors API: list ids, then fetch them. @private */
|
|
2153
|
+
async _listVectors(limit) {
|
|
2154
|
+
const ids = [];
|
|
2155
|
+
let token;
|
|
2156
|
+
do {
|
|
2157
|
+
const params = new URLSearchParams({
|
|
2158
|
+
namespace: this.namespace,
|
|
2159
|
+
limit: String(PAGE)
|
|
2160
|
+
});
|
|
2161
|
+
if (token) params.set("paginationToken", token);
|
|
2162
|
+
let json;
|
|
2163
|
+
try {
|
|
2164
|
+
({ json } = await this._data("GET", `/vectors/list?${params}`));
|
|
2165
|
+
} catch (err) {
|
|
2166
|
+
if (err.status === 404) break;
|
|
2167
|
+
throw err;
|
|
2168
|
+
}
|
|
2169
|
+
for (const v of json.vectors || []) ids.push(v.id);
|
|
2170
|
+
token = json.pagination && json.pagination.next;
|
|
2171
|
+
} while (token && (!limit || ids.length < limit));
|
|
2172
|
+
const wanted = limit ? ids.slice(0, limit) : ids;
|
|
2173
|
+
const out = [];
|
|
2174
|
+
for (let i = 0; i < wanted.length; i += PAGE) {
|
|
2175
|
+
const params = new URLSearchParams({ namespace: this.namespace });
|
|
2176
|
+
for (const id of wanted.slice(i, i + PAGE)) params.append("ids", id);
|
|
2177
|
+
const { json } = await this._data("GET", `/vectors/fetch?${params}`);
|
|
2178
|
+
for (const id of wanted.slice(i, i + PAGE)) {
|
|
2179
|
+
const record = json.vectors && json.vectors[id];
|
|
2180
|
+
if (record) out.push(fromFields(record.metadata, id));
|
|
2181
|
+
}
|
|
2182
|
+
}
|
|
2183
|
+
return out;
|
|
2184
|
+
}
|
|
2185
|
+
};
|
|
2186
|
+
module2.exports = { PineconeRetriever, recordId, API_VERSION };
|
|
2187
|
+
}
|
|
2188
|
+
});
|
|
2189
|
+
|
|
1611
2190
|
// src/retrieval/pgvector-retriever.js
|
|
1612
2191
|
var require_pgvector_retriever = __commonJS({
|
|
1613
2192
|
"src/retrieval/pgvector-retriever.js"(exports2, module2) {
|
|
@@ -2119,6 +2698,7 @@ var require_connectors = __commonJS({
|
|
|
2119
2698
|
var { OllamaConnector } = require_ollama_connector();
|
|
2120
2699
|
var { LocalEmbedder } = require_local_embedder();
|
|
2121
2700
|
var { QdrantRetriever } = require_qdrant_retriever();
|
|
2701
|
+
var { PineconeRetriever } = require_pinecone_retriever();
|
|
2122
2702
|
var { PgVectorRetriever } = require_pgvector_retriever();
|
|
2123
2703
|
var { createAiSdkAdapter } = require_ai_sdk();
|
|
2124
2704
|
module2.exports = {
|
|
@@ -2128,6 +2708,7 @@ var require_connectors = __commonJS({
|
|
|
2128
2708
|
OllamaConnector,
|
|
2129
2709
|
LocalEmbedder,
|
|
2130
2710
|
QdrantRetriever,
|
|
2711
|
+
PineconeRetriever,
|
|
2131
2712
|
PgVectorRetriever,
|
|
2132
2713
|
createAiSdkAdapter
|
|
2133
2714
|
};
|
|
@@ -2147,6 +2728,7 @@ var modules = {
|
|
|
2147
2728
|
"connectors/rag-prompt.js": () => require_rag_prompt(),
|
|
2148
2729
|
"packages/connectors.js": () => require_connectors(),
|
|
2149
2730
|
"retrieval/pgvector-retriever.js": () => require_pgvector_retriever(),
|
|
2731
|
+
"retrieval/pinecone-retriever.js": () => require_pinecone_retriever(),
|
|
2150
2732
|
"retrieval/qdrant-retriever.js": () => require_qdrant_retriever()
|
|
2151
2733
|
};
|
|
2152
2734
|
Object.defineProperty(mod, Symbol.for("@devilsdev/rag-connectors/internal"), {
|
package/dist/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { OpenAIConnector, AnthropicConnector, CohereConnector, OllamaConnector, LocalEmbedder, QdrantRetriever, PgVectorRetriever, createAiSdkAdapter } from "@devilsdev/rag-core/all-types";
|
|
1
|
+
export { OpenAIConnector, AnthropicConnector, CohereConnector, OllamaConnector, LocalEmbedder, QdrantRetriever, PineconeRetriever, PgVectorRetriever, createAiSdkAdapter } from "@devilsdev/rag-core/all-types";
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire } from 'module';
|
|
2
2
|
const require = createRequire(import.meta.url);
|
|
3
3
|
const mod = require('./index.cjs');
|
|
4
|
-
export const { OpenAIConnector, AnthropicConnector, CohereConnector, OllamaConnector, LocalEmbedder, QdrantRetriever, PgVectorRetriever, createAiSdkAdapter } = mod;
|
|
4
|
+
export const { OpenAIConnector, AnthropicConnector, CohereConnector, OllamaConnector, LocalEmbedder, QdrantRetriever, PineconeRetriever, PgVectorRetriever, createAiSdkAdapter } = mod;
|
|
5
5
|
export default mod;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devilsdev/rag-connectors",
|
|
3
|
-
"version": "3.
|
|
4
|
-
"description": "Embedders, LLMs and vector stores for @devilsdev/rag-core: OpenAI, Anthropic, Cohere, Ollama, the AI SDK, Qdrant and
|
|
3
|
+
"version": "3.6.0",
|
|
4
|
+
"description": "Embedders, LLMs and vector stores for @devilsdev/rag-core: OpenAI, Anthropic, Cohere, Ollama, the AI SDK, Qdrant, pgvector and Pinecone.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"rag",
|
|
7
7
|
"openai",
|
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
"exports": {
|
|
29
29
|
".": {
|
|
30
30
|
"types": "./dist/index.d.ts",
|
|
31
|
+
"workerd": "./dist/index.cjs",
|
|
31
32
|
"require": "./dist/index.cjs",
|
|
32
33
|
"import": "./dist/index.mjs"
|
|
33
34
|
},
|
|
@@ -43,7 +44,7 @@
|
|
|
43
44
|
"access": "public"
|
|
44
45
|
},
|
|
45
46
|
"dependencies": {
|
|
46
|
-
"@devilsdev/rag-core": "3.
|
|
47
|
+
"@devilsdev/rag-core": "3.6.0",
|
|
47
48
|
"axios": "^1.21.0"
|
|
48
49
|
},
|
|
49
50
|
"peerDependencies": {
|