@devilsdev/rag-connectors 3.6.0 → 3.7.1

package/README.md

@@ -1,6 +1,6 @@
 # @devilsdev/rag-connectors
 
-Embedders, LLMs and vector stores for @devilsdev/rag-core: OpenAI, Anthropic, Cohere, Ollama, the AI SDK, Qdrant, pgvector and Pinecone.
+Embedders, LLMs and vector stores for @devilsdev/rag-core: OpenAI, Anthropic, Cohere, Ollama, the AI SDK, Qdrant, pgvector, Pinecone and Weaviate.
 
 Part of [@devilsdev/rag-pipeline-utils](https://www.npmjs.com/package/@devilsdev/rag-pipeline-utils), split into packages so you install only what you use. The objects are the same ones the full package exports; versions move in lockstep.
 
@@ -8,7 +8,7 @@ Part of [@devilsdev/rag-pipeline-utils](https://www.npmjs.com/package/@devilsdev
 npm install @devilsdev/rag-connectors
 ```
 
-Exports: `OpenAIConnector`, `AnthropicConnector`, `CohereConnector`, `OllamaConnector`, `LocalEmbedder`, `QdrantRetriever`, `PineconeRetriever`, `PgVectorRetriever`, `createAiSdkAdapter`.
+Exports: `OpenAIConnector`, `AnthropicConnector`, `CohereConnector`, `OllamaConnector`, `LocalEmbedder`, `QdrantRetriever`, `PineconeRetriever`, `PgVectorRetriever`, `WeaviateRetriever`, `createAiSdkAdapter`.
 
 Documentation: https://devilsdev.github.io/rag-pipeline-utils/
 

package/dist/index.cjs

@@ -233,8 +233,25 @@ var require_openai_connector = __commonJS({
       _request(signal, fn) {
         return this.withRetry(fn, this.config.maxRetries, this.config.retryDelay, {
           signal
+        }).catch((err) => {
+          throw this._tagError(err);
         });
       }
+      /**
+       * Name the provider on an HTTP error from the SDK, as the axios-based
+       * connectors do, so a caller such as the CLI can say which key was refused.
+       * A custom baseURL never reads OPENAI_API_KEY, so it gets no envVar.
+       * @private
+       */
+      _tagError(err) {
+        if (err && typeof err === "object" && typeof err.status === "number") {
+          if (!err.provider) {
+            err.provider = this.config.baseURL ? "The OpenAI-compatible API" : "OpenAI";
+          }
+          if (!this.config.baseURL && !err.envVar) err.envVar = API_KEY_ENV;
+        }
+        return err;
+      }
       /**
        * Generate a chat completion using the provided query and optional RAG context.
        *
@@ -278,19 +295,24 @@ var require_openai_connector = __commonJS({
         this._requireApiKey();
         const client = this._getClient();
         const messages = this._buildMessages(query, context);
-        const stream = await raceAbort(
-          client.chat.completions.create(
-            {
-              model: options.model || this.config.model,
-              messages,
-              temperature: options.temperature ?? this.config.temperature,
-              max_tokens: options.maxTokens ?? this.config.maxTokens,
-              stream: true
-            },
-            signal ? { signal } : void 0
-          ),
-          signal
-        );
+        let stream;
+        try {
+          stream = await raceAbort(
+            client.chat.completions.create(
+              {
+                model: options.model || this.config.model,
+                messages,
+                temperature: options.temperature ?? this.config.temperature,
+                max_tokens: options.maxTokens ?? this.config.maxTokens,
+                stream: true
+              },
+              signal ? { signal } : void 0
+            ),
+            signal
+          );
+        } catch (err) {
+          throw this._tagError(err);
+        }
         for await (const chunk of stream) {
           throwIfAborted(signal);
           const token = chunk.choices[0]?.delta?.content || "";
@@ -1350,7 +1372,7 @@ var require_qdrant_retriever = __commonJS({
         if (typeof fetchImpl !== "function") {
           throw new Error("QdrantRetriever needs fetch (Node.js 18 or later)");
         }
-        this.url = String(url).replace(/\/+$/, "");
+        this.url = String(url).replace(/(?<!\/)\/+$/, "");
         this.collection = collection;
         this.dimensions = dimensions;
         this.createCollection = createCollection;
@@ -1621,12 +1643,15 @@ var require_pinecone_retriever = __commonJS({
     var CONTROL_PLANE = "https://api.pinecone.io";
     var DEFAULT_NAMESPACE = "rag_documents";
     var UPSERT_BATCH = 100;
+    var EMBED_UPSERT_BATCH = 96;
+    var REQUEST_BYTES = 2e6 - 1024;
     var DELETE_BATCH = 1e3;
     var PAGE = 100;
     var SEARCH_MAX = 1e4;
     var PLAIN_ID = /^[\x20-\x7e]{1,512}$/;
     var FIELDS = ["rag_id", "rag_content", "rag_metadata"];
     var OURS = { rag_id: { $exists: true } };
+    var PROBE_QUERY = "document";
     function recordId(id) {
       const s = String(id);
       if (PLAIN_ID.test(s)) return s;
@@ -1671,6 +1696,7 @@ var require_pinecone_retriever = __commonJS({
        * @param {string} [options.namespace='rag_documents'] - A namespace this retriever owns
        * @param {'auto'|'documents'|'vectors'} [options.api='auto'] - Document index (Documents API) or vector index; auto reads it from the index description
        * @param {string} [options.vectorField] - The document index's dense_vector field (default: the schema's)
+       * @param {string} [options.textField] - The document index's text field that Pinecone embeds (integrated embedding; default: the schema's, when the index has no dense_vector field)
        * @param {number} [options.freshnessTimeoutMs=30000] - How long a read waits for this retriever's writes to be indexed; 0 to not wait
        * @param {string} [options.cacheKey] - Cache identity; change it when the index changes
        * @param {Function} [options.fetch] - fetch implementation (defaults to the global)
@@ -1683,6 +1709,7 @@ var require_pinecone_retriever = __commonJS({
           namespace = DEFAULT_NAMESPACE,
           api = "auto",
           vectorField,
+          textField,
           freshnessTimeoutMs = 3e4,
           cacheKey,
           fetch: fetchImpl = globalThis.fetch
@@ -1705,6 +1732,16 @@ var require_pinecone_retriever = __commonJS({
             'PineconeRetriever: api must be "auto", "documents" or "vectors"'
           );
         }
+        if (vectorField && textField) {
+          throw new Error(
+            "PineconeRetriever: pass vectorField (your vectors) or textField (Pinecone embeds the text), not both"
+          );
+        }
+        if (textField && api === "vectors") {
+          throw new Error(
+            'PineconeRetriever: textField needs a document index (api "documents" or "auto")'
+          );
+        }
         if (typeof fetchImpl !== "function") {
           throw new Error("PineconeRetriever needs fetch (Node.js 18 or later)");
         }
@@ -1716,6 +1753,7 @@ var require_pinecone_retriever = __commonJS({
         this._host = host ? normalizeHost(host) : null;
         this._api = api;
         this._vectorField = vectorField;
+        this._textField = textField;
         this._fetch = fetchImpl === globalThis.fetch ? fetchImpl.bind(globalThis) : fetchImpl;
         this._resolving = null;
         this._writeLsn = 0;
@@ -1731,6 +1769,18 @@ var require_pinecone_retriever = __commonJS({
         if (role !== "retriever" || !this.cacheKey) return null;
         return `pinecone|${this._host || this.index}|${this.namespace}|${this.cacheKey}`;
       }
+      /**
+       * Whether the index embeds text itself (integrated embedding): then
+       * store() needs no vectors and retrieve() takes the query text, so no
+       * embedder is needed. The CLI checks this before requiring an embedder.
+       * Reads the index description on first use.
+       * @param {{ signal?: AbortSignal }} [options]
+       * @returns {Promise<boolean>}
+       */
+      async embedsText({ signal } = {}) {
+        await this._resolve(signal);
+        return this._integrated;
+      }
       /**
        * One REST call; resolves the parsed body and the response headers.
        * @private
@@ -1779,12 +1829,21 @@ var require_pinecone_retriever = __commonJS({
         }
         return { json, headers: res.headers };
       }
+      /** True on a document index that embeds text itself. @private */
+      get _integrated() {
+        return this._api === "documents" && Boolean(this._textField);
+      }
       /**
        * The index's host and kind, from its description (once): a document
-       * index's dense_vector field and dimension, or a vector index's.
+       * index's dense_vector field and dimension or the text field it embeds,
+       * or a vector index's dimension.
        * @private
        */
       _resolve(signal) {
+        if (this._host && this._textField) {
+          this._api = "documents";
+          return Promise.resolve();
+        }
         if (this._host && this._api !== "auto" && (this._api === "vectors" || this._vectorField)) {
           return Promise.resolve();
         }
@@ -1830,15 +1889,27 @@ var require_pinecone_retriever = __commonJS({
         const fields = info.schema && info.schema.fields || null;
         const named = fields ? Object.entries(fields).filter(([name]) => !name.startsWith("_")) : [];
         const denseFields = named.filter(([, f]) => f && f.type === "dense_vector");
+        const textFields = named.filter(
+          ([, f]) => f && f.type === "string" && f.embed
+        );
         if (this._api === "auto") {
           this._api = named.length > 0 ? "documents" : "vectors";
         }
-        if (this._api === "documents") {
+        if (this._textField) {
+          if (this._api !== "documents" || !textFields.some(([name]) => name === this._textField)) {
+            throw new Error(
+              `PineconeRetriever: index "${info.name}" has no text field named "${this._textField}" that it embeds`
+            );
+          }
+        } else if (this._api === "documents") {
           const field = this._vectorField ? denseFields.find(([name]) => name === this._vectorField) : denseFields[0];
+          if (!field && !this._vectorField && textFields.length > 0) {
+            this._textField = textFields[0][0];
+            return;
+          }
           if (!field) {
-            const embeds = fields && Object.values(fields).some((f) => f && f.embed);
             throw new Error(
-              embeds ? `PineconeRetriever: index "${info.name}" embeds text itself (integrated embedding) and has no dense_vector field for your vectors; create an index with a dense_vector field` : `PineconeRetriever: index "${info.name}" has no dense_vector field${this._vectorField ? ` named "${this._vectorField}"` : ""}`
+              `PineconeRetriever: index "${info.name}" has no dense_vector field${this._vectorField ? ` named "${this._vectorField}"` : ""}`
             );
           }
           this._vectorField = field[0];
@@ -1871,20 +1942,21 @@ var require_pinecone_retriever = __commonJS({
       /**
        * A search, repeated until it reflects this retriever's writes (when the
        * index reports log sequence numbers) or `freshnessTimeoutMs` passes.
-       * Resolves `[{ id, score, fields }]`.
+       * `query` is a vector, or the query text on an index that embeds text
+       * itself. Resolves `[{ id, score, fields }]`.
        * @private
        */
-      async _search(vector, topK, signal, filter) {
+      async _search(query, topK, signal, filter) {
         await this._resolve(signal);
         const documents = this._api === "documents";
         const [path, body] = documents ? [
           this._docs("search"),
           {
             score_by: [
-              {
+              this._integrated ? { type: "embed", fields: [this._textField], query } : {
                 type: "dense_vector",
                 fields: [this._vectorField],
-                values: vector
+                values: query
               }
             ],
             top_k: topK,
@@ -1894,7 +1966,7 @@ var require_pinecone_retriever = __commonJS({
         ] : [
           "/query",
           {
-            vector,
+            vector: query,
             topK,
             namespace: this.namespace,
             includeMetadata: true,
@@ -1925,17 +1997,25 @@ var require_pinecone_retriever = __commonJS({
       /** Waits until this retriever's writes are indexed (for list and size). @private */
       async _waitForWrites(signal) {
         if (!this._writeLsn) return;
+        const probe = this._probe();
+        if (probe) await this._search(probe, 1, signal);
+      }
+      /**
+       * What list() and size() search with: a unit vector of the index's
+       * dimension, or a fixed text on an index that embeds text itself.
+       * @private
+       */
+      _probe() {
+        if (this._integrated) return PROBE_QUERY;
         const dimension = this._indexDimension || this._dimension;
-        if (!dimension) return;
-        await this._search(
-          new Array(dimension).fill(0).map((_, i) => i === 0 ? 1 : 0),
-          1,
-          signal
-        );
+        if (!dimension) return null;
+        return new Array(dimension).fill(0).map((_, i) => i === 0 ? 1 : 0);
       }
       /**
-       * Inserts or replaces documents.
-       * @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
+       * Inserts or replaces documents. On an index that embeds text itself,
+       * Pinecone embeds `content` (required there) and `vector` is ignored.
+       * @param {Array<{ id: string, vector?: number[], content?: string, metadata?: Object }>} docs
+       *   `vector` is required unless the index embeds text itself
        * @returns {Promise<void>}
        */
       async store(docs) {
@@ -1944,6 +2024,7 @@ var require_pinecone_retriever = __commonJS({
         }
         if (docs.length === 0) return;
         await this._resolve();
+        if (this._integrated) return this._storeText(docs);
         const expected = this._indexDimension || this._dimension;
         let dimensions = null;
         const records = docs.map((doc) => {
@@ -2012,29 +2093,94 @@ var require_pinecone_retriever = __commonJS({
         }
         this._dimension = dimensions;
       }
+      /** store() on an index that embeds text itself. @private */
+      async _storeText(docs) {
+        const records = docs.map((doc) => {
+          const id = doc && doc.id;
+          if (typeof id !== "string" || id.length === 0) {
+            throw new Error(
+              "PineconeRetriever.store(): every document needs a string id"
+            );
+          }
+          if (typeof doc.content !== "string" || doc.content.trim() === "") {
+            throw new Error(
+              `PineconeRetriever.store(): document "${id}" has no content for Pinecone to embed (the index embeds ${this._textField})`
+            );
+          }
+          const metadata = doc.metadata || {};
+          return {
+            _id: recordId(id),
+            [this._textField]: doc.content,
+            rag_id: id,
+            rag_content: doc.content,
+            rag_metadata: JSON.stringify(metadata),
+            ...metadata.source !== void 0 && metadata.source !== null && {
+              rag_source: String(metadata.source)
+            }
+          };
+        });
+        const encoder = new TextEncoder();
+        let batch = [];
+        let bytes = 0;
+        const send = async () => {
+          if (batch.length === 0) return;
+          await this._data(
+            "POST",
+            this._docs("upsert"),
+            { documents: batch },
+            { write: true }
+          );
+          batch = [];
+          bytes = 0;
+        };
+        for (const record of records) {
+          const size = encoder.encode(JSON.stringify(record)).length + 1;
+          if (batch.length === EMBED_UPSERT_BATCH || batch.length > 0 && bytes + size > REQUEST_BYTES) {
+            await send();
+          }
+          batch.push(record);
+          bytes += size;
+        }
+        await send();
+      }
       /**
-       * The documents most similar to a query vector.
-       * @param {number[]|{ queryVector?: number[], topK?: number, signal?: AbortSignal }} request
-       *   A vector, or an object with one (required; optional in the type to
-       *   match the retriever contract)
-       * @param {number} [k=10] - Results when `request` is a vector
+       * The documents most similar to a query vector, or to the query text on
+       * an index that embeds text itself (Pinecone embeds the query; a
+       * `queryVector` is then ignored).
+       * @param {number[]|string|{ query?: string, queryVector?: number[], topK?: number, signal?: AbortSignal }} request
+       *   A vector, the query text, or an object with them: `queryVector`, or
+       *   `query` on an index that embeds text itself (required; optional in
+       *   the type to match the retriever contract)
+       * @param {number} [k=10] - Results when `request` is a vector or text
        * @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
        */
       async retrieve(request, k = 10) {
-        const {
-          queryVector,
-          topK = k,
-          signal
-        } = Array.isArray(request) ? { queryVector: request } : request || {};
+        let input;
+        if (Array.isArray(request)) input = { queryVector: request };
+        else if (typeof request === "string") input = { query: request };
+        else input = request || {};
+        const { query, queryVector, topK = k, signal } = input;
         if (!Number.isInteger(topK) || topK < 1) {
           throw new Error(
             "PineconeRetriever.retrieve(): topK must be a positive integer"
           );
         }
-        checkVector(queryVector, "query");
         throwIfAborted(signal);
+        await this._resolve(signal);
+        if (!this._integrated) {
+          if (queryVector === void 0 || queryVector === null) {
+            throw new Error(
+              "PineconeRetriever.retrieve(): needs a queryVector; this index does not embed text itself, so use an embedder"
+            );
+          }
+          checkVector(queryVector, "query");
+        } else if (typeof query !== "string" || query.trim() === "") {
+          throw new Error(
+            `PineconeRetriever.retrieve(): the index embeds text itself (integrated embedding, field ${this._textField}), so it needs the query text: pass { query }`
+          );
+        }
         const matches = await this._search(
-          queryVector,
+          this._integrated ? query : queryVector,
           Math.min(topK, 1e4),
           signal
         );
@@ -2113,15 +2259,10 @@ var require_pinecone_retriever = __commonJS({
       }
       /** Documents API: fetch by filter, page by page. @private */
       async _listDocuments(limit) {
-        const dimension = this._indexDimension || this._dimension;
-        if (dimension) {
+        const probe = this._probe();
+        if (probe) {
           const max = Math.min(limit || SEARCH_MAX, SEARCH_MAX);
-          const found = await this._search(
-            new Array(dimension).fill(0).map((_, i) => i === 0 ? 1 : 0),
-            max,
-            void 0,
-            OURS
-          );
+          const found = await this._search(probe, max, void 0, OURS);
           if (found.length < SEARCH_MAX || limit) {
             const seen = /* @__PURE__ */ new Set();
             return found.filter((m) => !seen.has(m.id) && seen.add(m.id)).map((m) => fromFields(m.fields, m.id));
@@ -2540,6 +2681,453 @@ var require_pgvector_retriever = __commonJS({
   }
 });
 
+// src/retrieval/weaviate-retriever.js
+var require_weaviate_retriever = __commonJS({
+  "src/retrieval/weaviate-retriever.js"(exports2, module2) {
+    "use strict";
+    var { throwIfAborted } = require_connector_utils();
+    var { pointId } = require_qdrant_retriever();
+    var COLLECTION = /^[A-Za-z][A-Za-z0-9_]{0,254}$/;
+    var DEFAULT_COLLECTION = "RagDocuments";
+    var PAGE = 256;
+    var UPSERT_BATCH = 256;
+    var DELETE_BATCH = 256;
+    var PROPERTIES = [
+      {
+        name: "rag_id",
+        dataType: ["text"],
+        tokenization: "field",
+        indexFilterable: true,
+        indexSearchable: false
+      },
+      { name: "content", dataType: ["text"], indexFilterable: false },
+      {
+        name: "rag_source",
+        dataType: ["text"],
+        tokenization: "field",
+        indexFilterable: true,
+        indexSearchable: false
+      },
+      {
+        name: "rag_metadata",
+        dataType: ["text"],
+        indexFilterable: false,
+        indexSearchable: false
+      }
+    ];
+    var OURS = { path: ["rag_id"], operator: "Like", valueText: "*" };
+    var OURS_GRAPHQL = '{ path: ["rag_id"], operator: Like, valueText: "*" }';
+    function checkVector(vector, id) {
+      if (!Array.isArray(vector) || vector.length === 0) {
+        throw new Error(`WeaviateRetriever: document "${id}" has no vector`);
+      }
+      if (!vector.every((x) => typeof x === "number" && Number.isFinite(x))) {
+        throw new Error(
+          `WeaviateRetriever: document "${id}" has a vector with a non-finite value`
+        );
+      }
+    }
+    function trimSlashes(url) {
+      let end = url.length;
+      while (end > 0 && url[end - 1] === "/") end--;
+      return url.slice(0, end);
+    }
+    function parseMetadata(text) {
+      if (!text) return {};
+      try {
+        const value = JSON.parse(text);
+        return value && typeof value === "object" ? value : {};
+      } catch {
+        return {};
+      }
+    }
+    function errorDetail(json) {
+      const list = json && (json.error || json.errors) || [];
+      if (Array.isArray(list) && list.length) {
+        return list.map((e) => e && e.message || String(e)).join("; ");
+      }
+      return json && json.message || "";
+    }
+    var WeaviateRetriever = class {
+      /**
+       * @param {Object} [options]
+       * @param {string} [options.url='http://localhost:8080'] - Weaviate's REST endpoint (without /v1)
+       * @param {string} [options.apiKey] - Sent as `Authorization: Bearer <key>`
+       * @param {string} [options.collection='RagDocuments'] - Class name; the first letter is upper-cased, as Weaviate does
+       * @param {number} [options.dimensions] - Expected vector size; defaults to the collection's or the first stored vector's
+       * @param {boolean} [options.createCollection=true] - Create the collection (and any missing properties) on first store
+       * @param {string} [options.cacheKey] - Cache identity; change it when the collection changes
+       * @param {Function} [options.fetch] - fetch implementation (defaults to the global)
+       */
+      constructor(options = {}) {
+        const {
+          url = "http://localhost:8080",
+          apiKey,
+          collection = DEFAULT_COLLECTION,
+          dimensions,
+          createCollection = true,
+          cacheKey,
+          fetch: fetchImpl = globalThis.fetch
+        } = options;
+        if (typeof collection !== "string" || !COLLECTION.test(collection)) {
+          throw new Error(
+            `WeaviateRetriever: invalid collection name "${collection}" (a letter, then letters, digits and "_")`
+          );
+        }
+        if (dimensions !== void 0 && !(Number.isInteger(dimensions) && dimensions > 0)) {
+          throw new Error(
+            "WeaviateRetriever: `dimensions` must be a positive integer"
+          );
+        }
+        if (typeof fetchImpl !== "function") {
+          throw new Error("WeaviateRetriever needs fetch (Node.js 18 or later)");
+        }
+        this.url = trimSlashes(String(url));
+        this.collection = collection[0].toUpperCase() + collection.slice(1);
+        this.dimensions = dimensions;
+        this.createCollection = createCollection;
+        this.cacheKey = cacheKey;
+        this._apiKey = apiKey;
+        this._fetch = fetchImpl === globalThis.fetch ? fetchImpl.bind(globalThis) : fetchImpl;
+        this._ready = null;
+        this._exists = false;
+      }
+      /**
+       * Cache identity for createRagPipeline({ cache }): none unless you set
+       * `cacheKey`, since other clients can change the collection.
+       * @param {'retriever'} role
+       * @returns {string|null}
+       */
+      getCacheKey(role) {
+        if (role !== "retriever" || !this.cacheKey) return null;
+        return `weaviate|${this.url}|${this.collection}|${this.cacheKey}`;
+      }
+      /** The message without the API key, should a server ever echo it. */
+      _redact(message) {
+        const text = String(message);
+        return this._apiKey ? text.split(this._apiKey).join("***") : text;
+      }
+      /**
+       * One REST call; resolves the parsed body.
+       * @private
+       * @param {string} method
+       * @param {string} path - After /v1
+       * @param {Object} [body]
+       * @param {{ signal?: AbortSignal, allow404?: boolean }} [options] - allow404 resolves null on 404
+       * @returns {Promise<any>}
+       */
+      async _request(method, path, body, { signal, allow404 = false } = {}) {
+        throwIfAborted(signal);
+        const headers = { "content-type": "application/json" };
+        if (this._apiKey) headers.authorization = `Bearer ${this._apiKey}`;
+        let res;
+        try {
+          res = await this._fetch(`${this.url}/v1${path}`, {
+            method,
+            headers,
+            ...body !== void 0 && { body: JSON.stringify(body) },
+            ...signal && { signal }
+          });
+        } catch (err) {
+          throwIfAborted(signal);
+          throw new Error(
+            this._redact(
+              `WeaviateRetriever: cannot reach Weaviate at ${this.url} (${err.message})`
+            )
+          );
+        }
+        if (res.status === 404 && allow404) return null;
+        const text = await res.text();
+        let json;
+        try {
+          json = text ? JSON.parse(text) : {};
+        } catch {
+          json = {};
+        }
+        if (!res.ok) {
+          if (res.status === 401 || res.status === 403) {
+            throw new Error(
+              `WeaviateRetriever: Weaviate refused the request (${res.status}); check WEAVIATE_API_KEY`
+            );
+          }
+          const detail = errorDetail(json) || text || res.statusText;
+          const where = path.split("?")[0];
+          throw new Error(
+            this._redact(
+              `WeaviateRetriever: ${method} ${where} failed (${res.status}): ${detail}`
+            )
+          );
+        }
+        return json;
+      }
+      /** A GraphQL query; resolves `data`, throws on GraphQL errors. */
+      async _graphql(query, signal) {
+        const json = await this._request("POST", "/graphql", { query }, { signal });
+        if (json.errors && json.errors.length) {
+          throw new Error(
+            this._redact(
+              `WeaviateRetriever: GraphQL query failed: ${errorDetail(json)}`
+            )
+          );
+        }
+        return json.data || {};
+      }
+      /** The collection's schema, or null when it doesn't exist. */
+      _schema(signal) {
+        return this._request("GET", `/schema/${this.collection}`, void 0, {
+          signal,
+          allow404: true
+        });
+      }
+      /** Whether the collection exists; only a yes is remembered. */
+      async _collectionExists(signal) {
+        if (!this._exists) this._exists = Boolean(await this._schema(signal));
+        return this._exists;
+      }
+      _ensureCollection() {
+        if (!this._ready) {
+          this._ready = (async () => {
+            if (this.createCollection) {
+              const schema = await this._schema();
+              if (!schema) {
+                try {
+                  await this._request("POST", "/schema", {
+                    class: this.collection,
+                    vectorizer: "none",
+                    vectorIndexType: "hnsw",
+                    vectorIndexConfig: { distance: "cosine" },
+                    properties: PROPERTIES
+                  });
+                } catch (err) {
+                  if (!/already exists/i.test(err.message)) throw err;
+                }
+              } else {
+                const have = new Set((schema.properties || []).map((p) => p.name));
+                for (const property of PROPERTIES) {
+                  if (!have.has(property.name)) {
+                    await this._request(
+                      "POST",
+                      `/schema/${this.collection}/properties`,
+                      property
+                    );
+                  }
+                }
+              }
+              this._exists = true;
+            }
+            if (!this.dimensions) {
+              const page = await this._request(
+                "GET",
+                `/objects?class=${this.collection}&limit=1&include=vector`,
+                void 0,
+                { allow404: true }
+              );
+              const vector = page && page.objects && page.objects[0] && page.objects[0].vector;
+              if (Array.isArray(vector) && vector.length) {
+                this.dimensions = vector.length;
+              }
+            }
+          })().catch((err) => {
+            this._ready = null;
+            throw err;
+          });
+        }
+        return this._ready;
+      }
+      /**
+       * Inserts or replaces documents.
+       * @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
+       * @returns {Promise<void>}
+       */
+      async store(docs) {
+        if (!Array.isArray(docs))
+          throw new Error("WeaviateRetriever.store() takes an array");
+        if (docs.length === 0) return;
+        let dimensions = null;
+        const objects = docs.map((doc) => {
+          const id = doc && doc.id;
+          if (typeof id !== "string" || id.length === 0) {
+            throw new Error(
+              "WeaviateRetriever.store(): every document needs a string id"
+            );
+          }
+          checkVector(doc.vector, id);
+          if (dimensions === null) dimensions = doc.vector.length;
+          else if (doc.vector.length !== dimensions) {
+            throw new Error(
+              `WeaviateRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the batch has ${dimensions}`
+            );
+          }
+          const metadata = doc.metadata || {};
+          const properties = {
+            rag_id: id,
+            content: doc.content || "",
+            rag_metadata: JSON.stringify(metadata)
+          };
+          if (metadata.source !== void 0 && metadata.source !== null) {
+            properties.rag_source = String(metadata.source);
+          }
+          return {
+            class: this.collection,
+            id: pointId(id),
+            vector: doc.vector,
+            properties
+          };
+        });
+        await this._ensureCollection();
+        if (this.dimensions && dimensions !== this.dimensions) {
+          throw new Error(
+            `WeaviateRetriever.store(): the documents have ${dimensions} dimensions, the collection has ${this.dimensions}`
+          );
+        }
+        for (let i = 0; i < objects.length; i += UPSERT_BATCH) {
+          const batch = objects.slice(i, i + UPSERT_BATCH);
+          const results = await this._request("POST", "/batch/objects", {
+            objects: batch
+          });
+          for (const [j, item] of (Array.isArray(results) ? results : []).entries()) {
+            const detail = item && item.result && errorDetail(item.result.errors);
+            if (!detail) continue;
+            const id = batch[j] ? batch[j].properties.rag_id : "?";
+            const kind = /vectors? with length/i.test(detail) ? "has the wrong vector dimensions" : "was not stored";
+            throw new Error(
+              this._redact(
+                `WeaviateRetriever.store(): document "${id}" ${kind}: ${detail}`
+              )
+            );
+          }
+        }
+        this._exists = true;
+        if (!this.dimensions) this.dimensions = dimensions;
+      }
+      /**
+       * The documents most similar to a query vector.
+       * @param {number[]|{ queryVector?: number[], topK?: number, signal?: AbortSignal }} request
+       *   A vector, or an object with one (required; optional in the type to
+       *   match the retriever contract)
+       * @param {number} [k=10] - Results when `request` is a vector
+       * @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
+       */
+      async retrieve(request, k = 10) {
+        const {
+          queryVector,
+          topK = k,
+          signal
+        } = Array.isArray(request) ? { queryVector: request } : request || {};
+        if (!Number.isInteger(topK) || topK < 1) {
+          throw new Error(
+            "WeaviateRetriever.retrieve(): topK must be a positive integer"
+          );
+        }
+        checkVector(queryVector, "query");
+        throwIfAborted(signal);
+        if (!await this._collectionExists(signal)) return [];
+        const data = await this._graphql(
+          `{ Get { ${this.collection}(nearVector: { vector: ${JSON.stringify(queryVector)} }, limit: ${topK}, where: ${OURS_GRAPHQL}) { rag_id content rag_metadata _additional { distance } } } }`,
+          signal
+        );
+        throwIfAborted(signal);
+        const hits = data.Get && data.Get[this.collection] || [];
+        return hits.map((hit) => ({
+          id: hit.rag_id,
+          score: 1 - hit._additional.distance,
+          content: hit.content || "",
+          metadata: parseMetadata(hit.rag_metadata)
+        }));
+      }
+      /**
+       * Deletes the matching objects, a server-limited round at a time.
+       * @private
+       */
+      async _deleteWhere(where) {
+        if (!await this._collectionExists()) return;
+        for (; ; ) {
+          const json = await this._request("DELETE", "/batch/objects", {
+            match: { class: this.collection, where },
+            output: "minimal"
+          });
+          const r = json && json.results || {};
+          if (r.failed > 0) {
+            throw new Error(
+              `WeaviateRetriever: ${r.failed} objects could not be deleted`
+            );
+          }
+          if (!(r.limit > 0 && r.matches > r.limit && r.successful > 0)) return;
+        }
+      }
+      /**
+       * Deletes the chunks of one source (`metadata.source`), for re-ingesting.
+       * @param {string} source
+       */
+      async removeBySource(source) {
+        await this._deleteWhere({
+          path: ["rag_source"],
+          operator: "Equal",
+          valueText: String(source)
+        });
+      }
+      /**
+       * Deletes documents by id.
+       * @param {string[]} ids
+       */
+      async delete(ids) {
+        for (let i = 0; i < ids.length; i += DELETE_BATCH) {
+          await this._deleteWhere({
+            path: ["rag_id"],
+            operator: "ContainsAny",
+            valueTextArray: ids.slice(i, i + DELETE_BATCH).map(String)
+          });
+        }
+      }
+      /** Deletes every document this class stored; the collection stays. */
+      async clear() {
+        await this._deleteWhere(OURS);
+      }
+      /** @returns {Promise<number>} */
+      async size() {
+        if (!await this._collectionExists()) return 0;
+        const data = await this._graphql(
+          `{ Aggregate { ${this.collection}(where: ${OURS_GRAPHQL}) { meta { count } } } }`
+        );
+        const rows = data.Aggregate && data.Aggregate[this.collection] || [];
+        return rows[0] && rows[0].meta ? rows[0].meta.count : 0;
+      }
+      /**
+       * Every document, without vectors.
+       * @param {{ limit?: number }} [options]
+       * @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
+       */
+      async list({ limit } = {}) {
+        const out = [];
+        if (!await this._collectionExists()) return out;
+        let after = "";
+        for (; ; ) {
+          const page = await this._request(
+            "GET",
+            `/objects?class=${this.collection}&limit=${PAGE}&after=${encodeURIComponent(after)}`,
+            void 0,
+            { allow404: true }
+          );
+          const objects = page && page.objects || [];
+          for (const o of objects) {
+            const p = o.properties || {};
+            if (typeof p.rag_id !== "string") continue;
+            out.push({
+              id: p.rag_id,
+              content: p.content || "",
+              metadata: parseMetadata(p.rag_metadata)
+            });
+            if (limit && out.length >= limit) return out;
+          }
+          if (objects.length < PAGE) return out;
+          after = objects[objects.length - 1].id;
+        }
+      }
+    };
+    module2.exports = { WeaviateRetriever };
+  }
+});
+
 // src/adapters/ai-sdk.js
 var require_ai_sdk = __commonJS({
   "src/adapters/ai-sdk.js"(exports2, module2) {
@@ -2700,6 +3288,7 @@ var require_connectors = __commonJS({
     var { QdrantRetriever } = require_qdrant_retriever();
     var { PineconeRetriever } = require_pinecone_retriever();
     var { PgVectorRetriever } = require_pgvector_retriever();
+    var { WeaviateRetriever } = require_weaviate_retriever();
     var { createAiSdkAdapter } = require_ai_sdk();
     module2.exports = {
       OpenAIConnector,
@@ -2710,6 +3299,7 @@ var require_connectors = __commonJS({
       QdrantRetriever,
       PineconeRetriever,
       PgVectorRetriever,
+      WeaviateRetriever,
       createAiSdkAdapter
     };
   }
@@ -2729,7 +3319,8 @@ var modules = {
   "packages/connectors.js": () => require_connectors(),
   "retrieval/pgvector-retriever.js": () => require_pgvector_retriever(),
   "retrieval/pinecone-retriever.js": () => require_pinecone_retriever(),
-  "retrieval/qdrant-retriever.js": () => require_qdrant_retriever()
+  "retrieval/qdrant-retriever.js": () => require_qdrant_retriever(),
+  "retrieval/weaviate-retriever.js": () => require_weaviate_retriever()
 };
 Object.defineProperty(mod, Symbol.for("@devilsdev/rag-connectors/internal"), {
   value: (file) => {

package/dist/index.d.ts

@@ -1 +1 @@
-export { OpenAIConnector, AnthropicConnector, CohereConnector, OllamaConnector, LocalEmbedder, QdrantRetriever, PineconeRetriever, PgVectorRetriever, createAiSdkAdapter } from "@devilsdev/rag-core/all-types";
+export { OpenAIConnector, AnthropicConnector, CohereConnector, OllamaConnector, LocalEmbedder, QdrantRetriever, PineconeRetriever, PgVectorRetriever, WeaviateRetriever, createAiSdkAdapter } from "@devilsdev/rag-core/all-types";

package/dist/index.mjs

@@ -1,5 +1,5 @@
 import { createRequire } from 'module';
 const require = createRequire(import.meta.url);
 const mod = require('./index.cjs');
-export const { OpenAIConnector, AnthropicConnector, CohereConnector, OllamaConnector, LocalEmbedder, QdrantRetriever, PineconeRetriever, PgVectorRetriever, createAiSdkAdapter } = mod;
+export const { OpenAIConnector, AnthropicConnector, CohereConnector, OllamaConnector, LocalEmbedder, QdrantRetriever, PineconeRetriever, PgVectorRetriever, WeaviateRetriever, createAiSdkAdapter } = mod;
 export default mod;

package/package.json

@@ -1,21 +1,22 @@
 {
   "name": "@devilsdev/rag-connectors",
-  "version": "3.6.0",
-  "description": "Embedders, LLMs and vector stores for @devilsdev/rag-core: OpenAI, Anthropic, Cohere, Ollama, the AI SDK, Qdrant, pgvector and Pinecone.",
+  "version": "3.7.1",
+  "description": "Embedders, LLMs and vector stores for @devilsdev/rag-core: OpenAI, Anthropic, Cohere, Ollama, the AI SDK, Qdrant, pgvector, Pinecone and Weaviate.",
   "keywords": [
     "rag",
     "openai",
     "anthropic",
     "ollama",
     "qdrant",
-    "pgvector"
+    "pgvector",
+    "weaviate"
   ],
   "license": "MIT",
   "author": "Ali Kahwaji",
   "homepage": "https://devilsdev.github.io/rag-pipeline-utils/",
   "repository": {
     "type": "git",
-    "url": "https://github.com/DevilsDev/rag-pipeline-utils.git"
+    "url": "git+https://github.com/DevilsDev/rag-pipeline-utils.git"
   },
   "bugs": {
     "url": "https://github.com/DevilsDev/rag-pipeline-utils/issues"
@@ -44,7 +45,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@devilsdev/rag-core": "3.6.0",
+    "@devilsdev/rag-core": "3.7.1",
     "axios": "^1.21.0"
   },
   "peerDependencies": {