@dudousxd/nestjs-agent-rag 0.10.3 → 0.12.0

package/dist/index.js

@@ -356,6 +356,181 @@ function cosineSimilarity(a, b) {
 }
 __name(cosineSimilarity, "cosineSimilarity");
 
+// src/lexical-query.ts
+var words = /* @__PURE__ */ __name((s) => new Set(s.split(/\s+/).filter(Boolean)), "words");
+var DEFAULT_STOP_WORDS = {
+  english: words(`
+    me my myself we our ours ourselves you your yours yourself yourselves he him his himself she
+    her hers herself it its itself they them their theirs themselves what which who whom this that
+    these those am is are was were be been being have has had having do does did doing an the and
+    but if or because as until while of at by for with about against between into through during
+    before after above below to from up down in out on off over under again further then once here
+    there when where why how all any both each few more most other some such no nor not only own
+    same so than too very can will just don should now`),
+  portuguese: words(`
+    de que do da em um para com n\xE3o uma os no se na por mais as dos como mas ao ele das seu sua ou
+    quando muito nos j\xE1 eu tamb\xE9m s\xF3 pelo pela at\xE9 isso ela entre depois sem mesmo aos seus quem
+    nas me esse eles voc\xEA essa num nem suas meu \xE0s minha numa pelos elas qual n\xF3s lhe deles essas
+    esses pelas este dele tu te voc\xEAs vos lhes meus minhas teu tua teus tuas nosso nossa nossos
+    nossas dela delas esta estes estas aquele aquela aqueles aquelas isto aquilo estou est\xE1 estamos
+    est\xE3o estive esteve estivemos estiveram estava est\xE1vamos estavam estivera estiv\xE9ramos esteja
+    estejamos estejam estivesse estiv\xE9ssemos estivessem estiver estivermos estiverem hei h\xE1 havemos
+    h\xE3o houve houvemos houveram houvera houv\xE9ramos haja hajamos hajam houvesse houv\xE9ssemos
+    houvessem houver houvermos houverem houverei houver\xE1 houveremos houver\xE3o houveria houver\xEDamos
+    houveriam sou somos s\xE3o era \xE9ramos eram fui foi fomos foram fora f\xF4ramos seja sejamos sejam
+    fosse f\xF4ssemos fossem for formos forem serei ser\xE1 seremos ser\xE3o seria ser\xEDamos seriam tenho tem
+    temos t\xE9m tinha t\xEDnhamos tinham tive teve tivemos tiveram tivera tiv\xE9ramos tenha tenhamos
+    tenham tivesse tiv\xE9ssemos tivessem tiver tivermos tiverem terei ter\xE1 teremos ter\xE3o teria
+    ter\xEDamos teriam`),
+  spanish: words(`
+    de la que el en los del se las por un para con no una su al lo como m\xE1s pero sus le ya este s\xED
+    porque esta entre cuando muy sin sobre tambi\xE9n me hasta hay donde quien desde todo nos durante
+    todos uno les ni contra otros ese eso ante ellos esto m\xED antes algunos qu\xE9 unos yo otro otras
+    otra \xE9l tanto esa estos mucho quienes nada muchos cual poco ella estar estas algunas algo
+    nosotros mi mis t\xFA te ti tu tus ellas nosotras vosotros vosotras os m\xEDo m\xEDa m\xEDos m\xEDas tuyo
+    tuya tuyos tuyas suyo suya suyos suyas nuestro nuestra nuestros nuestras vuestro vuestra
+    vuestros vuestras esos esas estoy est\xE1s est\xE1 estamos est\xE1is est\xE1n est\xE9 est\xE9s estemos est\xE9is
+    est\xE9n estar\xE9 estar\xE1s estar\xE1 estaremos estar\xE9is estar\xE1n estar\xEDa estar\xEDas estar\xEDamos estar\xEDais
+    estar\xEDan estaba estabas est\xE1bamos estabais estaban estuve estuviste estuvo estuvimos
+    estuvisteis estuvieron estuviera estuvieras estuvi\xE9ramos estuvierais estuvieran estuviese
+    estuvieses estuvi\xE9semos estuvieseis estuviesen estando estad he has ha hemos hab\xE9is han haya
+    hayas hayamos hay\xE1is hayan habr\xE9 habr\xE1s habr\xE1 habremos habr\xE9is habr\xE1n habr\xEDa habr\xEDas habr\xEDamos
+    habr\xEDais habr\xEDan hab\xEDa hab\xEDas hab\xEDamos hab\xEDais hab\xEDan hube hubiste hubo hubimos hubisteis
+    hubieron hubiera hubieras hubi\xE9ramos hubierais hubieran hubiese hubieses hubi\xE9semos hubieseis
+    hubiesen habiendo habido habida habidos habidas soy eres es somos sois son sea seas seamos
+    se\xE1is sean ser\xE9 ser\xE1s ser\xE1 seremos ser\xE9is ser\xE1n ser\xEDa ser\xEDas ser\xEDamos ser\xEDais ser\xEDan era eras
+    \xE9ramos erais eran fui fuiste fue fuimos fuisteis fueron fuera fueras fu\xE9ramos fuerais fueran
+    fuese fueses fu\xE9semos fueseis fuesen tengo tienes tiene tenemos ten\xE9is tienen tenga tengas
+    tengamos teng\xE1is tengan tendr\xE9 tendr\xE1s tendr\xE1 tendremos tendr\xE9is tendr\xE1n tendr\xEDa tendr\xEDas
+    tendr\xEDamos tendr\xEDais tendr\xEDan ten\xEDa ten\xEDas ten\xEDamos ten\xEDais ten\xEDan tuve tuviste tuvo tuvimos
+    tuvisteis tuvieron tuviera tuvieras tuvi\xE9ramos tuvierais tuvieran tuviese tuvieses tuvi\xE9semos
+    tuvieseis tuviesen teniendo tenido tenida tenidos tenidas tened`)
+};
+var TOKEN = /[\p{L}\p{N}]{2,}/gu;
+function keywordTerms(query, max = 24, stopWords = DEFAULT_STOP_WORDS) {
+  const tokens = [
+    ...new Set(query.toLowerCase().normalize("NFC").match(TOKEN) ?? [])
+  ];
+  const counts = Object.entries(stopWords).map(([, stop2]) => [
+    stop2,
+    tokens.filter((t) => stop2.has(t)).length
+  ]);
+  const best = Math.max(0, ...counts.map(([, n]) => n));
+  if (best === 0) return tokens.slice(0, max);
+  const stop = counts.filter(([, n]) => n === best).map(([s]) => s);
+  const terms = tokens.filter((t) => !stop.some((s) => s.has(t)));
+  return (terms.length > 0 ? terms : tokens).slice(0, max);
+}
+__name(keywordTerms, "keywordTerms");
+function hasSearchSyntax(query) {
+  return /"[^"]+"/.test(query) || /(^|\s)-[\p{L}\p{N}]/u.test(query);
+}
+__name(hasSearchSyntax, "hasSearchSyntax");
+function anyTermTsquery(terms) {
+  return terms.map((t) => `'${t.replace(/'/g, "")}'`).join(" | ");
+}
+__name(anyTermTsquery, "anyTermTsquery");
+
+// src/pg-client.ts
+var ADAPTED = /* @__PURE__ */ Symbol.for("@dudousxd/nestjs-agent-rag/pg-client");
+function toPgClient(source) {
+  if (source[ADAPTED]) {
+    return source;
+  }
+  if (isDrizzle(source)) {
+    return toPgClient(source.$client);
+  }
+  if (typeof source.unsafe === "function") {
+    return fromPostgresJs(source);
+  }
+  return fromQueryable(source);
+}
+__name(toPgClient, "toPgClient");
+function isDrizzle(source) {
+  const client = source.$client;
+  return client !== null && client !== void 0 && typeof client === "object";
+}
+__name(isDrizzle, "isDrizzle");
+function mark(client) {
+  Object.defineProperty(client, ADAPTED, {
+    value: true
+  });
+  return client;
+}
+__name(mark, "mark");
+function rowsOf(result) {
+  if (Array.isArray(result)) {
+    return result;
+  }
+  const rows = result?.rows;
+  return Array.isArray(rows) ? rows : [];
+}
+__name(rowsOf, "rowsOf");
+function fromPostgresJs(sql) {
+  const client = {
+    query: /* @__PURE__ */ __name(async (text, params = []) => (
+      // postgres.js resolves a RowList (an Array subclass); copy it to a plain array.
+      [
+        ...rowsOf(await sql.unsafe(text, params))
+      ]
+    ), "query")
+  };
+  if (typeof sql.begin === "function") {
+    const begin = sql.begin.bind(sql);
+    client.transaction = (work) => Promise.resolve(begin((tx) => work(fromPostgresJs(tx))));
+  }
+  return mark(client);
+}
+__name(fromPostgresJs, "fromPostgresJs");
+function fromQueryable(source) {
+  const client = {
+    query: /* @__PURE__ */ __name(async (text, params) => rowsOf(await source.query(text, params)), "query")
+  };
+  if (typeof source.transaction === "function") {
+    const transaction = source.transaction.bind(source);
+    client.transaction = (work) => transaction((tx) => work(toPgClient(tx)));
+  } else if (isNodePgPool(source)) {
+    client.transaction = async (work) => {
+      const connection = await source.connect();
+      let broken = false;
+      try {
+        return await runInTransaction(connection, work, () => {
+          broken = true;
+        });
+      } finally {
+        connection.release(broken ? new Error("ROLLBACK failed") : void 0);
+      }
+    };
+  } else if (isNodePgClient(source)) {
+    client.transaction = (work) => runInTransaction(source, work);
+  }
+  return mark(client);
+}
+__name(fromQueryable, "fromQueryable");
+async function runInTransaction(connection, work, onRollbackFailed) {
+  const tx = mark({
+    query: /* @__PURE__ */ __name(async (text, params) => rowsOf(await connection.query(text, params)), "query")
+  });
+  await connection.query("BEGIN");
+  try {
+    const result = await work(tx);
+    await connection.query("COMMIT");
+    return result;
+  } catch (error) {
+    await connection.query("ROLLBACK").catch(() => onRollbackFailed?.());
+    throw error;
+  }
+}
+__name(runInTransaction, "runInTransaction");
+function isNodePgPool(source) {
+  return typeof source.connect === "function" && typeof source.totalCount === "number";
+}
+__name(isNodePgPool, "isNodePgPool");
+function isNodePgClient(source) {
+  return typeof source.release === "function" || typeof source.processID === "number" || source.connectionParameters !== void 0;
+}
+__name(isNodePgClient, "isNodePgClient");
+
 // src/pg-vector-store.ts
 var NUL_BYTE = String.fromCharCode(0);
 function stripNulBytes(value) {
@@ -384,11 +559,37 @@ var PgVectorStore = class {
   }
   client;
   table;
+  /** The single width (fixed-dimension table), or `undefined` for a mixed-dimension table. */
   dimensions;
+  /** The widths with a partial HNSW index, for a mixed-dimension table. */
+  indexedDimensions;
+  nullableEmbeddings;
+  iterativeScan;
+  efSearch;
+  upsertBatchSize;
+  iterativeScanSupport;
   constructor(client, options = {}) {
-    this.client = client;
+    this.client = toPgClient(client);
     this.table = options.table ?? "agent_rag_chunks";
-    this.dimensions = options.dimensions ?? 1536;
+    const dimensions = options.dimensions ?? 1536;
+    if (typeof dimensions === "number") {
+      this.dimensions = dimensions;
+      this.indexedDimensions = [];
+    } else {
+      for (const width of dimensions) {
+        if (!Number.isInteger(width) || width <= 0) {
+          throw new Error(`PgVectorStore: invalid embedding width ${String(width)}`);
+        }
+      }
+      this.dimensions = void 0;
+      this.indexedDimensions = [
+        ...new Set(dimensions)
+      ];
+    }
+    this.nullableEmbeddings = options.nullableEmbeddings ?? false;
+    this.iterativeScan = options.iterativeScan;
+    this.efSearch = options.efSearch;
+    this.upsertBatchSize = Math.max(1, Math.floor(options.upsertBatchSize ?? 100));
   }
   /** Telemetry self-description — the table is the namespace a retrieval was served from. */
   describeRetrieval() {
@@ -397,38 +598,77 @@ var PgVectorStore = class {
       collection: this.table
     };
   }
-  /** Idempotent DDL: the `vector` extension, the chunks table, and the cosine HNSW index. */
-  async ensureSchema() {
-    await this.client.query("CREATE EXTENSION IF NOT EXISTS vector");
-    await this.client.query(`CREATE TABLE IF NOT EXISTS ${this.table} (
+  /**
+  * The idempotent DDL {@link PgVectorStore.ensureSchema} runs, in order — for a migration instead
+  * of a boot-time side effect. On a populated table, run the index statements `CONCURRENTLY` there.
+  */
+  schemaStatements() {
+    const mixed = this.dimensions === void 0;
+    const column = mixed ? "vector" : `vector(${this.dimensions})`;
+    const statements = [
+      "CREATE EXTENSION IF NOT EXISTS vector",
+      `CREATE TABLE IF NOT EXISTS ${this.table} (
         id TEXT PRIMARY KEY,
         text TEXT NOT NULL,
         source TEXT,
         metadata JSONB,
-        embedding vector(${this.dimensions}) NOT NULL
-      )`);
-    await this.client.query(`CREATE INDEX IF NOT EXISTS ${this.table}_embedding_idx
+        embedding ${column}${this.nullableEmbeddings ? "" : " NOT NULL"}
+      )`
+    ];
+    if (mixed) {
+      for (const width of this.indexedDimensions) {
+        statements.push(`CREATE INDEX IF NOT EXISTS ${this.table}_embedding_${width}_idx
+        ON ${this.table} USING hnsw ((embedding::vector(${width})) vector_cosine_ops)
+        WHERE vector_dims(embedding) = ${width}`);
+      }
+    } else {
+      statements.push(`CREATE INDEX IF NOT EXISTS ${this.table}_embedding_idx
         ON ${this.table} USING hnsw (embedding vector_cosine_ops)`);
+    }
+    return statements;
+  }
+  /** Idempotent DDL: the `vector` extension, the chunks table, and the cosine HNSW index(es). */
+  async ensureSchema() {
+    for (const statement of this.schemaStatements()) {
+      await this.client.query(statement);
+    }
   }
+  /**
+  * Multi-row `INSERT … ON CONFLICT (id) DO UPDATE`, {@link PgVectorStoreOptions.upsertBatchSize}
+  * rows per statement. An empty `embedding` is written as `NULL` (see
+  * {@link PgVectorStoreOptions.nullableEmbeddings}).
+  */
   async upsert(records) {
+    const byId = /* @__PURE__ */ new Map();
     for (const record of records) {
       const id = stripNulBytes(record.id);
-      const text = stripNulBytes(record.text);
-      const source = record.source !== void 0 ? stripNulBytes(record.source) : void 0;
-      const metadata = record.metadata !== void 0 ? stripNulBytes(record.metadata) : void 0;
+      byId.delete(id);
+      byId.set(id, {
+        ...record,
+        id
+      });
+    }
+    const unique = [
+      ...byId.values()
+    ];
+    for (let start = 0; start < unique.length; start += this.upsertBatchSize) {
+      const batch = unique.slice(start, start + this.upsertBatchSize);
+      const params = [];
+      const rows = batch.map((record) => {
+        const text = stripNulBytes(record.text);
+        const source = record.source !== void 0 ? stripNulBytes(record.source) : void 0;
+        const metadata = record.metadata !== void 0 ? stripNulBytes(record.metadata) : void 0;
+        const base = params.length;
+        params.push(record.id, text, source ?? null, metadata !== void 0 ? JSON.stringify(metadata) : null, record.embedding.length > 0 ? toVectorLiteral(record.embedding) : null);
+        return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}::jsonb, $${base + 5}::vector)`;
+      });
       await this.client.query(`INSERT INTO ${this.table} (id, text, source, metadata, embedding)
-         VALUES ($1, $2, $3, $4, $5)
+         VALUES ${rows.join(", ")}
          ON CONFLICT (id) DO UPDATE SET
            text = EXCLUDED.text,
            source = EXCLUDED.source,
            metadata = EXCLUDED.metadata,
-           embedding = EXCLUDED.embedding`, [
-        id,
-        text,
-        source ?? null,
-        metadata !== void 0 ? JSON.stringify(metadata) : null,
-        toVectorLiteral(record.embedding)
-      ]);
+           embedding = EXCLUDED.embedding`, params);
     }
   }
   async remove(documentId) {
@@ -464,11 +704,12 @@ var PgVectorStore = class {
     return rows.length;
   }
   async listDocuments(filter) {
-    const where = buildWhere(filter, 1);
+    const params = [];
+    const where = this.where(filter, params);
     const rows = await this.client.query(`SELECT DISTINCT ON (${DOCUMENT_ID_FROM_CHUNK}) ${DOCUMENT_ID_FROM_CHUNK} AS doc_id, metadata
        FROM ${this.table}
-       ${where.sql}
-       ORDER BY ${DOCUMENT_ID_FROM_CHUNK}`, where.params);
+       ${where}
+       ORDER BY ${DOCUMENT_ID_FROM_CHUNK}`, params);
     return rows.map((row) => ({
       id: row.doc_id,
       ...row.metadata !== null ? {
@@ -519,11 +760,12 @@ var PgVectorStore = class {
     if (filterMatchesNothing(filter)) {
       return [];
     }
-    const where = buildWhere(filter, 1);
+    const params = [];
+    const where = this.where(filter, params);
     const rows = await this.client.query(`SELECT DISTINCT ${DOCUMENT_ID_FROM_CHUNK} AS doc_id
        FROM ${this.table}
-       ${where.sql}
-       ORDER BY doc_id`, where.params);
+       ${where}
+       ORDER BY doc_id`, params);
     return rows.map((row) => row.doc_id);
   }
   /** N documents in one statement — the set-based form of {@link PgVectorStore.remove}. */
@@ -547,81 +789,250 @@ var PgVectorStore = class {
     if (filterMatchesNothing(filter)) {
       return 0;
     }
-    const where = buildWhere(filter, 1);
-    const rows = await this.client.query(`DELETE FROM ${this.table} ${where.sql} RETURNING id`, where.params);
+    const params = [];
+    const where = this.where(filter, params);
+    const rows = await this.client.query(`DELETE FROM ${this.table} ${where} RETURNING id`, params);
     return rows.length;
   }
   async countChunks(filter) {
     if (filterMatchesNothing(filter)) {
       return 0;
     }
-    const where = buildWhere(filter, 1);
-    const rows = await this.client.query(`SELECT count(*)::int AS count FROM ${this.table} ${where.sql}`, where.params);
+    const params = [];
+    const where = this.where(filter, params);
+    const rows = await this.client.query(`SELECT count(*)::int AS count FROM ${this.table} ${where}`, params);
     return Number(rows[0]?.count ?? 0);
   }
+  /**
+  * Cosine similarity, nearest first. On a mixed-dimension table only vectors as wide as the query
+  * are compared (through that width's partial index when it has one); chunks without an embedding
+  * are never returned.
+  */
   async search(embedding, options) {
-    const vector = toVectorLiteral(embedding);
-    const where = buildWhere(options.filter, 3);
-    const rows = await this.client.query(`SELECT id, text, source, metadata, 1 - (embedding <=> $1::vector) AS score
+    if (embedding.length === 0) {
+      return [];
+    }
+    const params = [
+      toVectorLiteral(embedding),
+      options.topK
+    ];
+    const conditions = this.whereConditions(options.filter, params);
+    let vectorType = "vector";
+    if (this.dimensions === void 0) {
+      const width = embedding.length;
+      if (this.indexedDimensions.includes(width)) {
+        vectorType = `vector(${width})`;
+      }
+      conditions.push("embedding IS NOT NULL", `vector_dims(embedding) = ${width}`);
+    } else if (this.nullableEmbeddings) {
+      conditions.push("embedding IS NOT NULL");
+    }
+    const column = vectorType === "vector" ? "embedding" : `(embedding::${vectorType})`;
+    const sql = `SELECT id, text, source, metadata, 1 - (${column} <=> $1::${vectorType}) AS score
        FROM ${this.table}
-       ${where.sql}
-       ORDER BY embedding <=> $1::vector
-       LIMIT $2`, [
-      vector,
+       ${conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""}
+       ORDER BY ${column} <=> $1::${vectorType}
+       LIMIT $2`;
+    const rows = await this.withSearchSettings((client) => client.query(sql, params));
+    return rows.map(toPassage);
+  }
+  /**
+  * The SQL conditions (ANDed) a metadata `filter` compiles to, appending their bindings to `params`
+  * (placeholders are numbered after what `params` already holds). Every statement that takes a
+  * filter goes through here, so a subclass that maps the filter onto its own columns — a tenant id,
+  * an access list — overrides this one method. Return `['false']` to deny.
+  */
+  whereConditions(filter, params) {
+    return buildWhere(filter, params);
+  }
+  where(filter, params) {
+    const conditions = this.whereConditions(filter, params);
+    return conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+  }
+  /** Run a read with the configured HNSW settings `SET LOCAL` in one transaction, when possible. */
+  async withSearchSettings(read) {
+    const transaction = this.client.transaction?.bind(this.client);
+    if (transaction === void 0) {
+      return read(this.client);
+    }
+    const iterative = this.iterativeScan !== void 0 && await this.supportsIterativeScan();
+    if (!iterative && this.efSearch === void 0) {
+      return read(this.client);
+    }
+    return transaction(async (tx) => {
+      if (iterative) {
+        await tx.query(`SELECT set_config('hnsw.iterative_scan', $1, true)`, [
+          this.iterativeScan
+        ]);
+      }
+      if (this.efSearch !== void 0) {
+        await tx.query(`SELECT set_config('hnsw.ef_search', $1, true)`, [
+          String(this.efSearch)
+        ]);
+      }
+      return read(tx);
+    });
+  }
+  /** `hnsw.iterative_scan` exists from pgvector 0.8.0; setting it on an older one is an error. */
+  supportsIterativeScan() {
+    this.iterativeScanSupport ??= this.client.query(`SELECT extversion FROM pg_extension WHERE extname = 'vector'`).then((rows) => {
+      const [major = 0, minor = 0] = (rows[0]?.extversion ?? "0.0").split(".").map(Number);
+      return major > 0 || minor >= 8;
+    }).catch(() => false);
+    return this.iterativeScanSupport;
+  }
+};
+var PgLexicalVectorStore = class extends PgVectorStore {
+  static {
+    __name(this, "PgLexicalVectorStore");
+  }
+  tsConfig;
+  tsColumn;
+  anyTermFallback;
+  stopWords;
+  constructor(client, options = {}) {
+    super(client, options);
+    const config = options.fullText?.config ?? "simple";
+    if (!SQL_IDENTIFIER.test(config)) {
+      throw new Error(`PgLexicalVectorStore: invalid text search configuration "${config}"`);
+    }
+    const column = options.fullText?.column;
+    if (column !== void 0 && !SQL_IDENTIFIER.test(column)) {
+      throw new Error(`PgLexicalVectorStore: invalid tsvector column "${column}"`);
+    }
+    this.tsConfig = config;
+    this.tsColumn = column;
+    this.anyTermFallback = options.fullText?.anyTermFallback ?? true;
+    const stopWords = options.fullText?.stopWords;
+    this.stopWords = stopWords === false ? {} : stopWords ?? DEFAULT_STOP_WORDS;
+  }
+  schemaStatements() {
+    const statements = super.schemaStatements();
+    if (this.tsColumn === void 0) {
+      statements.push(`CREATE INDEX IF NOT EXISTS ${this.table}_text_tsv_idx
+        ON ${this.table} USING gin (${this.tsvector()})`);
+    }
+    return statements;
+  }
+  async searchText(query, options) {
+    if (filterMatchesNothing(options.filter) || query.trim() === "") {
+      return [];
+    }
+    if (!this.anyTermFallback || hasSearchSyntax(query)) {
+      const strict = await this.runTextSearch(query, options);
+      if (strict.length > 0 || !this.anyTermFallback) {
+        return strict;
+      }
+    }
+    const terms = keywordTerms(query, 24, this.stopWords);
+    return terms.length === 0 ? [] : this.runRankedTermSearch(terms, options);
+  }
+  /** `websearch_to_tsquery` on the query as written: every word, quoted phrases, `-exclusions`. */
+  async runTextSearch(query, options) {
+    const tsquery = `websearch_to_tsquery('${this.tsConfig}', $1)`;
+    const params = [
+      query,
+      options.topK
+    ];
+    const conditions = [
+      `${this.tsvector()} @@ ${tsquery}`,
+      ...this.whereConditions(options.filter, params)
+    ];
+    const rows = await this.client.query(`SELECT id, text, source, metadata, ts_rank_cd(${this.tsvector()}, ${tsquery}) AS score
+       FROM ${this.table}
+       WHERE ${conditions.join(" AND ")}
+       ORDER BY score DESC, id
+       LIMIT $2`, params);
+    return rows.map(toPassage);
+  }
+  /**
+  * Rows holding any of `terms`, ranked by the sum of the matched terms' IDF among those rows,
+  * `ln(1 + (N - df + 0.5) / (df + 0.5))`, then `ts_rank_cd`. Each (row, term) match is evaluated
+  * once, and `ts_rank_cd` only for the rows ranked within `topK` (ties included): a question full
+  * of common words can match most of the table.
+  */
+  async runRankedTermSearch(terms, options) {
+    const any = `to_tsquery('${this.tsConfig}', $1)`;
+    const params = [
+      anyTermTsquery(terms),
       options.topK,
-      ...where.params
-    ]);
-    return rows.map((row) => ({
-      id: row.id,
-      text: row.text,
-      score: Number(row.score),
-      ...row.source !== null ? {
-        source: row.source
-      } : {},
-      ...row.metadata !== null ? {
-        metadata: row.metadata
-      } : {}
-    }));
+      terms
+    ];
+    const conditions = [
+      `${this.tsvector()} @@ ${any}`,
+      ...this.whereConditions(options.filter, params)
+    ];
+    const rows = await this.client.query(`WITH __terms AS (
+         SELECT DISTINCT to_tsquery('${this.tsConfig}', quote_literal(t)) AS q
+         FROM unnest($3::text[]) AS t
+       ), __cand AS MATERIALIZED (
+         SELECT id, text, source, metadata, ${this.tsvector()} AS __tsv
+         FROM ${this.table}
+         WHERE ${conditions.join(" AND ")}
+       ), __hits AS MATERIALIZED (
+         SELECT __cand.id AS __id, __terms.q FROM __cand JOIN __terms ON __cand.__tsv @@ __terms.q
+       ), __idf AS (
+         SELECT q, ln(1 + ((SELECT count(*) FROM __cand) - count(*) + 0.5) / (count(*) + 0.5)) AS w
+         FROM __hits GROUP BY q
+       ), __scored AS (
+         SELECT __hits.__id, sum(__idf.w) AS s, rank() OVER (ORDER BY sum(__idf.w) DESC) AS r
+         FROM __hits JOIN __idf USING (q) GROUP BY __hits.__id
+       )
+       SELECT __cand.id, __cand.text, __cand.source, __cand.metadata,
+              __scored.s + ts_rank_cd(__cand.__tsv, ${any}, 32) AS score
+       FROM __scored JOIN __cand ON __cand.id = __scored.__id
+       WHERE __scored.r <= $2
+       ORDER BY __scored.s DESC, ts_rank_cd(__cand.__tsv, ${any}, 32) DESC, __cand.id
+       LIMIT $2`, params);
+    return rows.map(toPassage);
+  }
+  /** Must be character-for-character the index expression, or the planner won't use the index. */
+  tsvector() {
+    return this.tsColumn ?? `to_tsvector('${this.tsConfig}'::regconfig, text)`;
   }
 };
-function buildWhere(filter, startIndex) {
+var SQL_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?$/;
+function toPassage(row) {
+  return {
+    id: row.id,
+    text: row.text,
+    score: Number(row.score),
+    ...row.source !== null ? {
+      source: row.source
+    } : {},
+    ...row.metadata !== null ? {
+      metadata: row.metadata
+    } : {}
+  };
+}
+__name(toPassage, "toPassage");
+function buildWhere(filter, params) {
   if (filter === void 0 || Object.keys(filter).length === 0) {
-    return {
-      sql: "",
-      params: []
-    };
+    return [];
   }
   const stripped = stripNulBytes(filter);
   const clauses = [];
-  const params = [];
   const scalar = {};
-  let index = startIndex;
   for (const [key, value] of Object.entries(stripped)) {
     if (Array.isArray(value)) {
       if (value.length === 0) {
         clauses.push("false");
         continue;
       }
-      const keyParam = `$${index}`;
       params.push(key);
-      index += 1;
-      const arrParam = `$${index}`;
+      const keyParam = `$${params.length}`;
       params.push(value.map(String));
-      index += 1;
+      const arrParam = `$${params.length}`;
       clauses.push(`(CASE WHEN jsonb_typeof(metadata->${keyParam}) = 'array' THEN metadata->${keyParam} ELSE jsonb_build_array(metadata->${keyParam}) END) ?| ${arrParam}::text[]`);
     } else {
       scalar[key] = value;
     }
   }
   if (Object.keys(scalar).length > 0) {
-    clauses.push(`metadata @> $${index}::jsonb`);
     params.push(JSON.stringify(scalar));
-    index += 1;
+    clauses.push(`metadata @> $${params.length}::jsonb`);
   }
-  return {
-    sql: `WHERE ${clauses.join(" AND ")}`,
-    params
-  };
+  return clauses;
 }
 __name(buildWhere, "buildWhere");
 var DOCUMENT_ID_FROM_CHUNK = "regexp_replace(id, '#[0-9]+$', '')";
@@ -1431,7 +1842,7 @@ function parseSearchReply(reply, prefix, scoreSource = "vector") {
           attrs[toStr(rawFields[field])] = toStr(rawFields[field + 1]);
         }
         const inlineScore = scoreSource === "lexical" ? toStr(reply[index + 1]) : void 0;
-        passages.push(toPassage(toStr(reply[index]), attrs, prefix, scoreSource, inlineScore));
+        passages.push(toPassage2(toStr(reply[index]), attrs, prefix, scoreSource, inlineScore));
       }
     }
     return passages;
@@ -1439,7 +1850,7 @@ function parseSearchReply(reply, prefix, scoreSource = "vector") {
   if (typeof reply === "object" && reply !== null && "results" in reply) {
     const results = reply.results;
     if (Array.isArray(results)) {
-      return results.map((result) => toPassage(readId(result), readAttributes(result), prefix, scoreSource, scoreSource === "lexical" ? readScore(result) : void 0));
+      return results.map((result) => toPassage2(readId(result), readAttributes(result), prefix, scoreSource, scoreSource === "lexical" ? readScore(result) : void 0));
     }
   }
   return [];
@@ -1472,7 +1883,7 @@ function readAttributes(result) {
   return attrs;
 }
 __name(readAttributes, "readAttributes");
-function toPassage(rawId, attrs, prefix, scoreSource = "vector", inlineScore) {
+function toPassage2(rawId, attrs, prefix, scoreSource = "vector", inlineScore) {
   const score = scoreSource === "lexical" ? Number(inlineScore ?? "0") : 1 - Number(attrs.vector_score ?? "1");
   const source = attrs.source;
   return {
@@ -1485,7 +1896,7 @@ function toPassage(rawId, attrs, prefix, scoreSource = "vector", inlineScore) {
     ...parseMetadata(attrs.metadata_json)
   };
 }
-__name(toPassage, "toPassage");
+__name(toPassage2, "toPassage");
 function parseMetadata(raw) {
   if (raw === void 0) {
     return {};
@@ -1878,7 +2289,7 @@ async function ingestDocuments(documents, options) {
 __name(ingestDocuments, "ingestDocuments");
 
 // src/retrieval-telemetry.ts
-import { AsyncLocalStorage } from "node:async_hooks";
+import { AsyncLocalStorage } from "async_hooks";
 import { channelName, emit, getChannel } from "@dudousxd/nestjs-diagnostics";
 var RAG_DIAGNOSTIC_LIB = "rag";
 var RAG_RETRIEVAL_EVENT = "retrieval";
@@ -2030,13 +2441,181 @@ function createRetrievalTool(retriever, options = {}) {
   };
 }
 __name(createRetrievalTool, "createRetrievalTool");
+
+// src/openai-embeddings.ts
+var HttpModelError = class extends Error {
+  static {
+    __name(this, "HttpModelError");
+  }
+  status;
+  constructor(status, message) {
+    super(message), this.status = status;
+    this.name = "HttpModelError";
+  }
+};
+function openAiEmbeddings(options) {
+  const url = `${(options.baseUrl ?? "https://api.openai.com/v1").replace(/\/+$/, "")}/embeddings`;
+  const batchSize = Math.max(1, Math.floor(options.batchSize ?? 64));
+  const maxInputChars = options.maxInputChars ?? 24e3;
+  const timeoutMs = options.timeoutMs ?? 12e4;
+  const doFetch = options.fetch ?? fetch;
+  return {
+    async embed(texts) {
+      const vectors = [];
+      for (let start = 0; start < texts.length; start += batchSize) {
+        const input = texts.slice(start, start + batchSize).map((text) => (text.length > maxInputChars ? text.slice(0, maxInputChars) : text) || " ");
+        const response = await doFetch(url, {
+          method: "POST",
+          headers: {
+            "content-type": "application/json",
+            ...options.apiKey ? {
+              authorization: `Bearer ${options.apiKey}`
+            } : {},
+            ...options.headers
+          },
+          body: JSON.stringify({
+            model: options.model,
+            input,
+            ...options.dimensions !== void 0 ? {
+              dimensions: options.dimensions
+            } : {}
+          }),
+          signal: AbortSignal.timeout(timeoutMs)
+        });
+        const body = await response.text();
+        if (!response.ok) {
+          throw new HttpModelError(response.status, `Embeddings request failed (${response.status}): ${errorMessageOf(body)}`);
+        }
+        const json = parseJson(body);
+        const items = [
+          ...json?.data ?? []
+        ].sort((a, b) => (a.index ?? 0) - (b.index ?? 0));
+        if (items.length !== input.length || !items.every((item) => Array.isArray(item.embedding))) {
+          throw new HttpModelError(502, `Embeddings returned ${items.length} vectors for ${input.length} inputs`);
+        }
+        for (const item of items) {
+          vectors.push(item.embedding);
+        }
+        const tokens = json?.usage?.total_tokens ?? json?.usage?.prompt_tokens;
+        if (typeof tokens === "number" && Number.isFinite(tokens)) {
+          options.onUsage?.(tokens);
+        }
+      }
+      return vectors;
+    }
+  };
+}
+__name(openAiEmbeddings, "openAiEmbeddings");
+function parseJson(body) {
+  try {
+    return JSON.parse(body);
+  } catch {
+    return null;
+  }
+}
+__name(parseJson, "parseJson");
+function errorMessageOf(body) {
+  const json = parseJson(body);
+  const message = typeof json?.error === "string" ? json.error : json?.error?.message ?? json?.message ?? void 0;
+  return typeof message === "string" ? message : body.slice(0, 300);
+}
+__name(errorMessageOf, "errorMessageOf");
+
+// src/http-reranker.ts
+var HttpReranker = class {
+  static {
+    __name(this, "HttpReranker");
+  }
+  options;
+  format;
+  doFetch;
+  constructor(options) {
+    this.options = options;
+    this.format = options.format ?? "auto";
+    this.doFetch = options.fetch ?? fetch;
+  }
+  async rerank(query, passages, options = {}) {
+    if (passages.length === 0) {
+      return [];
+    }
+    const topK = options.topK ?? passages.length;
+    const documents = passages.map((passage) => passage.text);
+    const response = await this.doFetch(this.options.url, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        ...this.options.apiKey ? {
+          authorization: `Bearer ${this.options.apiKey}`
+        } : {},
+        ...this.options.headers
+      },
+      body: JSON.stringify({
+        ...this.options.model !== void 0 ? {
+          model: this.options.model
+        } : {},
+        query,
+        ...this.format !== "tei" ? {
+          documents,
+          top_n: topK
+        } : {},
+        ...this.format !== "cohere" ? {
+          texts: documents
+        } : {}
+      }),
+      signal: AbortSignal.timeout(this.options.timeoutMs ?? 3e4)
+    });
+    const body = await response.text();
+    if (!response.ok) {
+      throw new HttpModelError(response.status, `Rerank request failed (${response.status}): ${errorMessageOf(body)}`);
+    }
+    const items = itemsOf(body);
+    if (items === void 0) {
+      throw new HttpModelError(502, "Rerank response has no results");
+    }
+    const seen = /* @__PURE__ */ new Set();
+    const reranked = [];
+    for (const item of items) {
+      const index = item.index;
+      const passage = typeof index === "number" ? passages[index] : void 0;
+      if (passage === void 0 || seen.has(index)) {
+        continue;
+      }
+      seen.add(index);
+      reranked.push({
+        ...passage,
+        score: Number(item.relevance_score ?? item.score ?? 0)
+      });
+    }
+    reranked.sort((a, b) => b.score - a.score);
+    return reranked.slice(0, topK);
+  }
+};
+function itemsOf(body) {
+  let json;
+  try {
+    json = JSON.parse(body);
+  } catch {
+    return void 0;
+  }
+  if (Array.isArray(json)) {
+    return json;
+  }
+  const wrapped = json;
+  const items = wrapped?.results ?? wrapped?.data;
+  return Array.isArray(items) ? items : void 0;
+}
+__name(itemsOf, "itemsOf");
 export {
+  DEFAULT_STOP_WORDS,
   EmbeddingRetriever,
   FilteredRetriever,
+  HttpModelError,
+  HttpReranker,
   HybridRetriever,
   KeywordRetriever,
   LexicalRetriever,
   MemoryVectorStore,
+  PgLexicalVectorStore,
   PgVectorStore,
   RAG_DIAGNOSTIC_LIB,
   RAG_RETRIEVAL_CHANNEL,
@@ -2045,6 +2624,7 @@ export {
   RedisVectorStore,
   RerankingRetriever,
   UnsafeRemovalError,
+  anyTermTsquery,
   applyMetadataPatch,
   chunkDocuments,
   chunkIndexOf,
@@ -2056,9 +2636,13 @@ export {
   describesRetrieval,
   documentIdOf,
   emitRagRetrieval,
+  hasSearchSyntax,
   ingestChunks,
   ingestDocuments,
   instrumentRetriever,
-  isLexicalVectorStore
+  isLexicalVectorStore,
+  keywordTerms,
+  openAiEmbeddings,
+  toPgClient
 };
 //# sourceMappingURL=index.js.map