@adonis-agora/agent 0.34.1 → 0.36.0

package/dist/src/rag/pg-vector-store.d.ts

@@ -2,7 +2,7 @@ import type { EmbeddingProvider } from '../spi/embedding-provider.js';
 import type { Passage } from '../spi/retriever.js';
 import type { LucidRawRunner } from '../stores/lucid.js';
 import { EmbeddingRetriever } from './embedding-retriever.js';
-import type { IndexedDocument, MetadataPatch, VectorRecord, VectorSearchOptions, VectorStore } from './vector-store.js';
+import type { IndexedDocument, LexicalVectorStore, MetadataPatch, VectorRecord, VectorSearchOptions, VectorStore } from './vector-store.js';
 /**
  * The similarity metric — which pgvector distance operator ranks results (and which index opclass the
  * migration should build). `cosine` (the default) is scale-invariant and the right choice for most
@@ -30,8 +30,40 @@ export interface PgVectorColumns {
 export interface PgVectorStoreOptions {
     /** Table name. Default `agent_rag_chunks`. Validated against the identifier regex. */
     table?: string;
-    /** Embedding width — must match your model (e.g. 1536 for text-embedding-3-small). Default 1536. */
-    dimension?: number;
+    /**
+     * Embedding width — must match your model (e.g. 1536 for text-embedding-3-small). Default 1536.
+     *
+     * Pass a **list** to let several models' vectors share one table (mixed dimensions — say, while
+     * migrating from a 768-wide local model to a 1536-wide hosted one): the column becomes an untyped
+     * `vector`, the schema gets one *partial* HNSW index per listed width
+     * (`((embedding::vector(n))) … WHERE vector_dims(embedding) = n`), and `search` only compares the
+     * query against vectors of its own width, through that width's index. A width that is not listed is
+     * still answered — exactly, by a sequential scan. Put the model in the chunk metadata and filter on
+     * it when two models share a width.
+     */
+    dimension?: number | readonly number[];
+    /**
+     * Allow chunks with no embedding (`embedding: []` on upsert is stored as `NULL`): index text first
+     * and embed later, or keep full-text-only chunks. `search` skips them; the lexical leg of
+     * {@link PgLexicalVectorStore} still finds them. Default `false` — the column stays `NOT NULL`. Only
+     * changes the DDL for a table that does not exist yet.
+     */
+    nullableEmbeddings?: boolean;
+    /**
+     * pgvector ≥ 0.8 iterative index scans for `search`: keep walking the HNSW graph until enough rows
+     * pass the metadata filter, instead of returning fewer than `topK` when a selective filter (one
+     * tenant of many) rejects most of what the index produced. Applied with `SET LOCAL` inside a
+     * transaction, so it needs a handle with `transaction` (Lucid's `db` has one); silently skipped on an
+     * older pgvector or a runner without transactions. Default off.
+     */
+    iterativeScan?: 'strict_order' | 'relaxed_order';
+    /** `hnsw.ef_search` for `search` (candidate list size; pgvector's default is 40). Needs `transaction`. */
+    efSearch?: number;
+    /**
+     * Rows per multi-row `INSERT … ON CONFLICT` statement in `upsert`. Default 100. Duplicate ids in one
+     * `upsert` call collapse to the last occurrence (as the old row-by-row loop left them).
+     */
+    upsertBatchSize?: number;
     /** Similarity metric / distance operator. Default `cosine`. */
     metric?: PgVectorMetric;
     /** Override the physical column names (each validated). */
@@ -78,12 +110,20 @@ export declare function stripNulBytes<T>(value: T): T;
  * `vector` extension, the chunk table, and the metric's index.
  */
 export declare class PgVectorStore implements VectorStore {
-    private readonly db;
-    private readonly table;
+    protected readonly db: LucidRawRunner;
+    protected readonly table: string;
+    /** The single width (fixed-dimension table), or `undefined` for a mixed-dimension table. */
     private readonly dimension;
+    /** The widths with a partial HNSW index, for a mixed-dimension table. */
+    private readonly indexedDimensions;
+    private readonly nullableEmbeddings;
+    private readonly iterativeScan;
+    private readonly efSearch;
+    private readonly upsertBatchSize;
+    private iterativeScanSupport;
     private readonly metric;
     private readonly metricName;
-    private readonly col;
+    protected readonly col: Required<PgVectorColumns>;
     constructor(db: LucidRawRunner, options?: PgVectorStoreOptions);
     /**
      * Idempotent DDL — the `vector` extension, the chunk table, and the metric's index. Handy for tests
@@ -98,6 +138,9 @@ export declare class PgVectorStore implements VectorStore {
      * (`0x00`) in `text`/`jsonb` outright, while upstream sources (PDF extraction, Qdrant) happily carry
      * it, so an unstripped chunk would otherwise fail this INSERT with `invalid byte sequence for encoding
      * "UTF8": 0x00`. The embedding is untouched — it is never text/jsonb.
+     *
+     * Written as multi-row `INSERT … ON CONFLICT` statements of {@link PgVectorStoreOptions.upsertBatchSize}
+     * rows each (one round trip per batch instead of per chunk). An empty `embedding` is written as `NULL`.
      */
     upsert(records: VectorRecord[]): Promise<void>;
     /**
@@ -144,7 +187,91 @@ export declare class PgVectorStore implements VectorStore {
      * clause here as it does there, and a scalar the same `@> ?::jsonb` containment.
      */
     removeWhere(filter: Record<string, unknown>): Promise<number>;
+    /**
+     * Nearest first by the configured metric. On a mixed-dimension table only vectors as wide as the
+     * query are compared (through that width's partial index when it has one); a chunk without an
+     * embedding is never returned.
+     */
     search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
+    /** Run a read with the configured HNSW settings `SET LOCAL` in one transaction, when possible. */
+    private withSearchSettings;
+    /** `hnsw.iterative_scan` exists from pgvector 0.8.0; setting it on an older one is an error. */
+    private supportsIterativeScan;
+}
+/** Where the lexical leg of a {@link PgLexicalVectorStore} reads its `tsvector` from. */
+export interface PgFullTextOptions {
+    /**
+     * Text-search configuration (`simple`, `english`, `portuguese`, …). Default `simple` — no stemming
+     * or stop words, so it works for any language and matches exact terms. Must be the configuration
+     * the index was built with, or Postgres cannot use the index and scans the table.
+     */
+    config?: string;
+    /**
+     * A `tsvector` column you maintain yourself (typically `GENERATED ALWAYS AS (…) STORED` in a
+     * migration, e.g. weighting a title above the body) with its own GIN index. When omitted the store
+     * searches `to_tsvector(config, text)`, and {@link PgVectorStore.schemaStatements} adds a GIN
+     * expression index over exactly that.
+     */
+    column?: string;
+    /**
+     * Search natural-language questions by their meaningful terms (default `true`): the question's
+     * stop words are dropped (see {@link PgFullTextOptions.stopWords}) and rows holding any remaining
+     * term (at most 24) are ranked by the sum of the matched terms' IDF among those rows (BM25 without
+     * term frequencies), `ts_rank_cd` breaking ties; a question with explicit search syntax (a quoted
+     * phrase, `-word`) is first tried as written. `false`: only `websearch_to_tsquery` on the query as
+     * written (every word must appear).
+     */
+    anyTermFallback?: boolean;
+    /**
+     * Stop word lists dropped from questions, by language; only the list(s) with the most words in the
+     * question apply, so a word that is a stop word in another language stays a term. Default
+     * {@link DEFAULT_STOP_WORDS} (English, Portuguese, Spanish: Postgres' Snowball lists). `false` keeps
+     * every word. With a language `config` (`english`) Postgres drops that language's stop words too.
+     */
+    stopWords?: Readonly<Record<string, ReadonlySet<string>>> | false;
+}
+/**
+ * {@link PgVectorStore} plus Postgres full-text search: a {@link LexicalVectorStore}, so a
+ * {@link import('./lexical-retriever.js').LexicalRetriever} and a
+ * {@link import('./hybrid-retriever.js').HybridRetriever} can fuse the dense and lexical legs over the
+ * same rows and ids. `retrievers.pgvector({ fullText: {} })` wires exactly that.
+ *
+ * A question is searched by its meaningful terms: its stop words dropped, rows holding any remaining
+ * term ranked by IDF-weighted coverage, `ts_rank_cd` (cover density) breaking ties (see
+ * {@link PgFullTextOptions.anyTermFallback}); explicit syntax (quoted phrases, `-exclusions`) goes
+ * through `websearch_to_tsquery` first.
+ *
+ * A separate class rather than a method on {@link PgVectorStore} on purpose: full-text search needs a
+ * GIN index, which on a populated table belongs in a migration you run and watch (`CREATE INDEX` in
+ * `ensureSchema` takes a write lock at boot), and a store that silently grew `searchText` on upgrade
+ * would be picked up by every `isLexicalVectorStore` check and scan an unindexed table.
+ */
+export declare class PgLexicalVectorStore extends PgVectorStore implements LexicalVectorStore {
+    private readonly tsConfig;
+    private readonly tsColumn;
+    private readonly anyTermFallback;
+    private readonly stopWords;
+    constructor(db: LucidRawRunner, options?: PgVectorStoreOptions & {
+        fullText?: PgFullTextOptions;
+    });
+    schemaStatements(): string[];
+    /**
+     * Full-text search over the chunk text. `score` is an IDF / `ts_rank_cd` rank — not a similarity,
+     * and not on the vector leg's scale — so `minScore` is deliberately not applied here.
+     */
+    searchText(query: string, options: VectorSearchOptions): Promise<Passage[]>;
+    /** `websearch_to_tsquery` on the query as written: every word, quoted phrases, `-exclusions`. */
+    private runTextSearch;
+    /**
+     * Rows holding any of `terms`, ranked by the sum of the matched terms' IDF among those rows,
+     * `ln(1 + (N - df + 0.5) / (df + 0.5))`, then `ts_rank_cd`. Each (row, term) match is evaluated
+     * once, and `ts_rank_cd` only for the rows ranked within `topK` (ties included): a question full of
+     * common words can match most of the table. Terms are letter/digit runs: space-joined in one
+     * binding (no array binding), each quoted as a literal lexeme.
+     */
+    private runRankedTermSearch;
+    /** Must be character-for-character the index expression, or the planner won't use the index. */
+    private tsvector;
 }
 /**
  * A {@link import('../spi/retriever.js').Retriever} over a {@link PgVectorStore}: embed the query, then

package/dist/src/rag/pg-vector-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pg-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/pg-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EACV,eAAe,EACf,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,IAAI,GAAG,OAAO,CAAC;AAEvD;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,6EAA6E;IAC7E,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,sFAAsF;IACtF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oGAAoG;IACpG,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAsDD,yGAAyG;AACzG,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAE3D;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAuB5C;AAyGD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAc,YAAW,WAAW;IAQ7C,OAAO,CAAC,QAAQ,CAAC,EAAE;IAPrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA4B;IAEhD,YACmB,EAAE,EAAE,cAAc,EACnC,OAAO,GAAE,oBAAyB,EAcnC;IAED;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAIlC;IAED,uGAAuG;IACvG,gBAAgB,IAAI,MAAM,EAAE,CAc3B;IAED;;;;;;OAMG;IACG,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAoBnD;IAED;;;;;OAKG;IACG,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI9C;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CA4B9E;IAEK,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAkBhF;IAED;;;;;OAKG;IACG,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAWzE;IAED;;;;;;;OAOG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAWlE;IAEK,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CA8ClF;CACF;AAED;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,kBAAkB;IAEvD,YAAY,QAAQ,EAAE,iBAAiB,EAAE,KAAK,EAAE,aAAa,EAE5D;CACF"}
+{"version":3,"file":"pg-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/pg-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAO9D,OAAO,KAAK,EACV,eAAe,EACf,kBAAkB,EAClB,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,IAAI,GAAG,OAAO,CAAC;AAEvD;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,6EAA6E;IAC7E,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,sFAAsF;IACtF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;;;OAUG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;IACvC;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,cAAc,GAAG,eAAe,CAAC;IACjD,0GAA0G;IAC1G,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAsDD,yGAAyG;AACzG,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAE3D;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAuB5C;AAyGD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAc,YAAW,WAAW;IAgB7C,SAAS,CAAC,QAAQ,CAAC,EAAE,EAAE,cAAc;IAfvC,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACjC,4FAA4F;IAC5F,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAC/C,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoB;IACtD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAC7C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAwC;IACtE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,oBAAoB,CAA+B;IAC3D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiB;IAC5C,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,eAAe,CAAC,CAAC;IAElD,YACqB,EAAE,EAAE,cAAc,EACrC,OAAO,GAAE,oBAAyB,EAgCnC;IAED;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAIlC;IAED,uGAAuG;IACvG,gBAAgB,IAAI,MAAM,EAAE,CA4B3B;IAED;;;;;;;;;OASG;IACG,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAqCnD;IAED;;;;;OAKG;IACG,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI9C;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CA4B9E;IAEK,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAkBhF;IAED;;;;;OAKG;IACG,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAWzE;IAED;;;;;;;OAOG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAWlE;IAED;;;;OAIG;IACG,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAoDlF;IAED,kGAAkG;YACpF,kBAAkB;IA+BhC,gGAAgG;IAChG,OAAO,CAAC,qBAAqB;CAW9B;AAED,yFAAyF;AACzF,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;CACnE;AAED;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,oBAAqB,SAAQ,aAAc,YAAW,kBAAkB;IACnF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAU;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgD;IAE1E,YACE,EAAE,EAAE,cAAc,EAClB,OAAO,GAAE,oBAAoB,GAAG;QAAE,QAAQ,CAAC,EAAE,iBAAiB,CAAA;KAAO,EAWtE;IAEQ,gBAAgB,IAAI,MAAM,EAAE,CASpC;IAED;;;OAGG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAYhF;IAED,iGAAiG;YACnF,aAAa;IAoB3B;;;;;;OAMG;YACW,mBAAmB;IA2CjC,gGAAgG;IAChG,OAAO,CAAC,QAAQ;CAGjB;AAmBD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,kBAAkB;IAEvD,YAAY,QAAQ,EAAE,iBAAiB,EAAE,KAAK,EAAE,aAAa,EAE5D;CACF"}

package/dist/src/rag/pg-vector-store.js

@@ -1,22 +1,23 @@
 import { EmbeddingRetriever } from './embedding-retriever.js';
+import { anyTermTsquery, DEFAULT_STOP_WORDS, hasSearchSyntax, keywordTerms, } from './lexical-query.js';
 import { assertRemovalFilter, effectivePatchKeys, filterDeniesAll } from './vector-store.js';
 const METRICS = {
     cosine: {
         operator: '<=>',
         opclass: 'vector_cosine_ops',
-        score: (col) => `1 - (${col} <=> ?::vector)`,
+        score: (col, type = 'vector') => `1 - (${col} <=> ?::${type})`,
     },
     l2: {
         operator: '<->',
         opclass: 'vector_l2_ops',
-        score: (col) => `-(${col} <-> ?::vector)`,
+        score: (col, type = 'vector') => `-(${col} <-> ?::${type})`,
     },
     inner: {
         // pgvector's `<#>` returns the NEGATIVE inner product, so `-(a <#> b)` recovers the inner product
         // and `ORDER BY a <#> b ASC` ranks the largest inner product first — consistent with the others.
         operator: '<#>',
         opclass: 'vector_ip_ops',
-        score: (col) => `-(${col} <#> ?::vector)`,
+        score: (col, type = 'vector') => `-(${col} <#> ?::${type})`,
     },
 };
 /** Only unqualified SQL identifiers (letter/underscore start, then letters/digits/underscores) are allowed. */
@@ -192,14 +193,39 @@ function documentIdExpr(col) {
 export class PgVectorStore {
     db;
     table;
+    /** The single width (fixed-dimension table), or `undefined` for a mixed-dimension table. */
     dimension;
+    /** The widths with a partial HNSW index, for a mixed-dimension table. */
+    indexedDimensions;
+    nullableEmbeddings;
+    iterativeScan;
+    efSearch;
+    upsertBatchSize;
+    iterativeScanSupport;
     metric;
     metricName;
     col;
     constructor(db, options = {}) {
         this.db = db;
         this.table = assertIdentifier(options.table ?? 'agent_rag_chunks', 'table');
-        this.dimension = options.dimension ?? 1536;
+        const dimension = options.dimension ?? 1536;
+        if (typeof dimension === 'number') {
+            this.dimension = dimension;
+            this.indexedDimensions = [];
+        }
+        else {
+            for (const width of dimension) {
+                if (!Number.isInteger(width) || width <= 0) {
+                    throw new Error(`Invalid pgvector dimension ${String(width)}: must be a positive integer`);
+                }
+            }
+            this.dimension = undefined;
+            this.indexedDimensions = [...new Set(dimension)];
+        }
+        this.nullableEmbeddings = options.nullableEmbeddings ?? false;
+        this.iterativeScan = options.iterativeScan;
+        this.efSearch = options.efSearch;
+        this.upsertBatchSize = Math.max(1, Math.floor(options.upsertBatchSize ?? 100));
         this.metricName = options.metric ?? 'cosine';
         this.metric = METRICS[this.metricName];
         const columns = options.columns ?? {};
@@ -223,18 +249,29 @@ export class PgVectorStore {
     /** The `CREATE EXTENSION` / `CREATE TABLE` / `CREATE INDEX` statements {@link ensureSchema} issues. */
     schemaStatements() {
         const c = this.col;
-        return [
+        const mixed = this.dimension === undefined;
+        const statements = [
             'CREATE EXTENSION IF NOT EXISTS vector',
             `CREATE TABLE IF NOT EXISTS ${this.table} (
         ${c.id} TEXT PRIMARY KEY,
         ${c.text} TEXT NOT NULL,
         ${c.source} TEXT,
         ${c.metadata} JSONB,
-        ${c.embedding} vector(${this.dimension}) NOT NULL
+        ${c.embedding} ${mixed ? 'vector' : `vector(${this.dimension})`}${this.nullableEmbeddings ? '' : ' NOT NULL'}
       )`,
-            `CREATE INDEX IF NOT EXISTS ${this.table}_${c.embedding}_idx
-        ON ${this.table} USING hnsw (${c.embedding} ${this.metric.opclass})`,
         ];
+        if (mixed) {
+            for (const width of this.indexedDimensions) {
+                statements.push(`CREATE INDEX IF NOT EXISTS ${this.table}_${c.embedding}_${width}_idx
+        ON ${this.table} USING hnsw ((${c.embedding}::vector(${width})) ${this.metric.opclass})
+        WHERE vector_dims(${c.embedding}) = ${width}`);
+            }
+        }
+        else {
+            statements.push(`CREATE INDEX IF NOT EXISTS ${this.table}_${c.embedding}_idx
+        ON ${this.table} USING hnsw (${c.embedding} ${this.metric.opclass})`);
+        }
+        return statements;
     }
     /**
      * Every `text`/`jsonb` binding (id, text, source, metadata — including nested metadata values, array
@@ -242,23 +279,38 @@ export class PgVectorStore {
      * (`0x00`) in `text`/`jsonb` outright, while upstream sources (PDF extraction, Qdrant) happily carry
      * it, so an unstripped chunk would otherwise fail this INSERT with `invalid byte sequence for encoding
      * "UTF8": 0x00`. The embedding is untouched — it is never text/jsonb.
+     *
+     * Written as multi-row `INSERT … ON CONFLICT` statements of {@link PgVectorStoreOptions.upsertBatchSize}
+     * rows each (one round trip per batch instead of per chunk). An empty `embedding` is written as `NULL`.
      */
     async upsert(records) {
-        const c = this.col;
+        // The id is stripped before de-duplication, so two ids differing only by a NUL byte are the one
+        // row they would be stored as. One statement can't touch a row twice (`ON CONFLICT DO UPDATE
+        // command cannot affect row a second time`), so duplicates collapse to the LAST occurrence — the
+        // same last-write-wins the old row-by-row loop had.
+        const byId = new Map();
         for (const record of records) {
+            const id = stripNulBytes(record.id);
+            byId.delete(id);
+            byId.set(id, { ...record, id });
+        }
+        const unique = [...byId.values()];
+        const c = this.col;
+        for (let start = 0; start < unique.length; start += this.upsertBatchSize) {
+            const batch = unique.slice(start, start + this.upsertBatchSize);
+            const bindings = [];
+            for (const record of batch) {
+                bindings.push(record.id, stripNulBytes(record.text), stripNulBytes(record.source ?? null), record.metadata !== undefined ? JSON.stringify(stripNulBytes(record.metadata)) : null, 
+                // An empty embedding is a chunk with no vector yet — NULL (see `nullableEmbeddings`).
+                record.embedding.length > 0 ? toVectorLiteral(record.embedding) : null);
+            }
             await this.db.rawQuery(`INSERT INTO ${this.table} (${c.id}, ${c.text}, ${c.source}, ${c.metadata}, ${c.embedding})
-         VALUES (?, ?, ?, ?::jsonb, ?::vector)
+         VALUES ${batch.map(() => '(?, ?, ?, ?::jsonb, ?::vector)').join(', ')}
          ON CONFLICT (${c.id}) DO UPDATE SET
            ${c.text} = EXCLUDED.${c.text},
            ${c.source} = EXCLUDED.${c.source},
            ${c.metadata} = EXCLUDED.${c.metadata},
-           ${c.embedding} = EXCLUDED.${c.embedding}`, [
-                stripNulBytes(record.id),
-                stripNulBytes(record.text),
-                stripNulBytes(record.source ?? null),
-                record.metadata !== undefined ? JSON.stringify(stripNulBytes(record.metadata)) : null,
-                toVectorLiteral(record.embedding),
-            ]);
+           ${c.embedding} = EXCLUDED.${c.embedding}`, bindings);
         }
     }
     /**
@@ -366,10 +418,33 @@ export class PgVectorStore {
         const raw = await this.db.rawQuery(`DELETE FROM ${this.table} ${where.sql} RETURNING ${this.col.id} AS id`, where.bindings);
         return normalizeRows(raw).length;
     }
+    /**
+     * Nearest first by the configured metric. On a mixed-dimension table only vectors as wide as the
+     * query are compared (through that width's partial index when it has one); a chunk without an
+     * embedding is never returned.
+     */
     async search(embedding, options) {
+        if (embedding.length === 0) {
+            return [];
+        }
         const c = this.col;
         const vector = toVectorLiteral(embedding);
-        const scoreExpr = this.metric.score(c.embedding);
+        // A fixed-width table keeps the exact expression its index was built on (`embedding`); a mixed one
+        // casts both sides to the query's width so the planner can match that width's partial index.
+        let vectorType = 'vector';
+        const shapeClauses = [];
+        if (this.dimension === undefined) {
+            const width = embedding.length;
+            if (this.indexedDimensions.includes(width)) {
+                vectorType = `vector(${width})`;
+            }
+            shapeClauses.push(`${c.embedding} IS NOT NULL`, `vector_dims(${c.embedding}) = ${width}`);
+        }
+        else if (this.nullableEmbeddings) {
+            shapeClauses.push(`${c.embedding} IS NOT NULL`);
+        }
+        const column = vectorType === 'vector' ? c.embedding : `(${c.embedding}::${vectorType})`;
+        const scoreExpr = this.metric.score(column, vectorType);
         const meta = buildMetadataWhere(options.filter, c.metadata);
         // Combine the metadata predicate with an optional minScore relevance floor. Filtering the score
         // expression in SQL (not in JS after the fact) keeps the floor applied BEFORE `LIMIT`, so the K
@@ -382,6 +457,7 @@ export class PgVectorStore {
             clauses.push(meta.sql.slice('WHERE '.length));
             whereBindings.push(...meta.bindings);
         }
+        clauses.push(...shapeClauses);
         if (options.minScore !== undefined) {
             // The score expression carries its own `?::vector`, so the embedding is bound again here.
             clauses.push(`${scoreExpr} >= ?`);
@@ -392,24 +468,189 @@ export class PgVectorStore {
         // then the optional filter/minScore clause(s), then the ORDER BY embedding, then the LIMIT. The
         // embedding is bound once per `?::vector` because a positional `?` cannot be reused like a `$1`.
         const bindings = [vector, ...whereBindings, vector, options.topK];
-        const raw = await this.db.rawQuery(`SELECT ${c.id} AS id, ${c.text} AS text, ${c.source} AS source, ${c.metadata} AS metadata,
+        const sql = `SELECT ${c.id} AS id, ${c.text} AS text, ${c.source} AS source, ${c.metadata} AS metadata,
               ${scoreExpr} AS score
        FROM ${this.table}
        ${whereSql}
-       ORDER BY ${c.embedding} ${this.metric.operator} ?::vector
-       LIMIT ?`, bindings);
-        return normalizeRows(raw).map((row) => {
-            const metadata = parseMetadata(row.metadata);
-            const source = row.source;
-            return {
-                id: String(row.id),
-                text: String(row.text),
-                score: Number(row.score),
-                ...(source !== null && source !== undefined ? { source: String(source) } : {}),
-                ...(metadata !== undefined ? { metadata } : {}),
-            };
+       ORDER BY ${column} ${this.metric.operator} ?::${vectorType}
+       LIMIT ?`;
+        const raw = await this.withSearchSettings((runner) => runner.rawQuery(sql, bindings));
+        return normalizeRows(raw).map(toPassage);
+    }
+    /** Run a read with the configured HNSW settings `SET LOCAL` in one transaction, when possible. */
+    async withSearchSettings(read) {
+        const transaction = this.db.transaction;
+        if (typeof transaction !== 'function') {
+            return read(this.db);
+        }
+        const iterative = this.iterativeScan !== undefined && (await this.supportsIterativeScan());
+        if (!iterative && this.efSearch === undefined) {
+            return read(this.db);
+        }
+        return transaction.call(this.db, async (trx) => {
+            // Lucid's transaction client is a full query client (it has `rawQuery`); the structural type
+            // this file sees only promises `from`/`table`, hence the check.
+            const runner = trx;
+            if (typeof runner.rawQuery !== 'function') {
+                return read(this.db);
+            }
+            const tx = runner;
+            if (iterative) {
+                await tx.rawQuery(`SELECT set_config('hnsw.iterative_scan', ?, true)`, [
+                    this.iterativeScan,
+                ]);
+            }
+            if (this.efSearch !== undefined) {
+                await tx.rawQuery(`SELECT set_config('hnsw.ef_search', ?, true)`, [String(this.efSearch)]);
+            }
+            return read(tx);
         });
     }
+    /** `hnsw.iterative_scan` exists from pgvector 0.8.0; setting it on an older one is an error. */
+    supportsIterativeScan() {
+        this.iterativeScanSupport ??= this.db
+            .rawQuery(`SELECT extversion FROM pg_extension WHERE extname = 'vector'`)
+            .then((raw) => {
+            const version = String(normalizeRows(raw)[0]?.extversion ?? '0.0');
+            const [major = 0, minor = 0] = version.split('.').map(Number);
+            return major > 0 || minor >= 8;
+        })
+            .catch(() => false);
+        return this.iterativeScanSupport;
+    }
+}
+/**
+ * {@link PgVectorStore} plus Postgres full-text search: a {@link LexicalVectorStore}, so a
+ * {@link import('./lexical-retriever.js').LexicalRetriever} and a
+ * {@link import('./hybrid-retriever.js').HybridRetriever} can fuse the dense and lexical legs over the
+ * same rows and ids. `retrievers.pgvector({ fullText: {} })` wires exactly that.
+ *
+ * A question is searched by its meaningful terms: its stop words dropped, rows holding any remaining
+ * term ranked by IDF-weighted coverage, `ts_rank_cd` (cover density) breaking ties (see
+ * {@link PgFullTextOptions.anyTermFallback}); explicit syntax (quoted phrases, `-exclusions`) goes
+ * through `websearch_to_tsquery` first.
+ *
+ * A separate class rather than a method on {@link PgVectorStore} on purpose: full-text search needs a
+ * GIN index, which on a populated table belongs in a migration you run and watch (`CREATE INDEX` in
+ * `ensureSchema` takes a write lock at boot), and a store that silently grew `searchText` on upgrade
+ * would be picked up by every `isLexicalVectorStore` check and scan an unindexed table.
+ */
+export class PgLexicalVectorStore extends PgVectorStore {
+    tsConfig;
+    tsColumn;
+    anyTermFallback;
+    stopWords;
+    constructor(db, options = {}) {
+        super(db, options);
+        this.tsConfig = assertIdentifier(options.fullText?.config ?? 'simple', 'text search config');
+        this.tsColumn =
+            options.fullText?.column !== undefined
+                ? assertIdentifier(options.fullText.column, 'column')
+                : undefined;
+        this.anyTermFallback = options.fullText?.anyTermFallback ?? true;
+        const stopWords = options.fullText?.stopWords;
+        this.stopWords = stopWords === false ? {} : (stopWords ?? DEFAULT_STOP_WORDS);
+    }
+    schemaStatements() {
+        const statements = super.schemaStatements();
+        if (this.tsColumn === undefined) {
+            statements.push(`CREATE INDEX IF NOT EXISTS ${this.table}_${this.col.text}_tsv_idx
+        ON ${this.table} USING gin (${this.tsvector()})`);
+        }
+        return statements;
+    }
+    /**
+     * Full-text search over the chunk text. `score` is an IDF / `ts_rank_cd` rank — not a similarity,
+     * and not on the vector leg's scale — so `minScore` is deliberately not applied here.
+     */
+    async searchText(query, options) {
+        if ((options.filter !== undefined && filterDeniesAll(options.filter)) || query.trim() === '') {
+            return [];
+        }
+        if (!this.anyTermFallback || hasSearchSyntax(query)) {
+            const strict = await this.runTextSearch(query, options);
+            if (strict.length > 0 || !this.anyTermFallback) {
+                return strict;
+            }
+        }
+        const terms = keywordTerms(query, 24, this.stopWords);
+        return terms.length === 0 ? [] : this.runRankedTermSearch(terms, options);
+    }
+    /** `websearch_to_tsquery` on the query as written: every word, quoted phrases, `-exclusions`. */
+    async runTextSearch(query, options) {
+        const c = this.col;
+        const where = buildMetadataWhere(options.filter, c.metadata);
+        // The parsed query is a lateral relation, so the one `?` binding feeds both the match and the rank.
+        const clauses = [`${this.tsvector()} @@ __tsq.q`];
+        if (where.sql !== '') {
+            clauses.push(where.sql.slice('WHERE '.length));
+        }
+        const raw = await this.db.rawQuery(`SELECT ${c.id} AS id, ${c.text} AS text, ${c.source} AS source, ${c.metadata} AS metadata,
+              ts_rank_cd(${this.tsvector()}, __tsq.q) AS score
+       FROM ${this.table}, websearch_to_tsquery('${this.tsConfig}', ?) AS __tsq(q)
+       WHERE ${clauses.join(' AND ')}
+       ORDER BY score DESC, ${c.id}
+       LIMIT ?`, [query, ...where.bindings, options.topK]);
+        return normalizeRows(raw).map(toPassage);
+    }
+    /**
+     * Rows holding any of `terms`, ranked by the sum of the matched terms' IDF among those rows,
+     * `ln(1 + (N - df + 0.5) / (df + 0.5))`, then `ts_rank_cd`. Each (row, term) match is evaluated
+     * once, and `ts_rank_cd` only for the rows ranked within `topK` (ties included): a question full of
+     * common words can match most of the table. Terms are letter/digit runs: space-joined in one
+     * binding (no array binding), each quoted as a literal lexeme.
+     */
+    async runRankedTermSearch(terms, options) {
+        const c = this.col;
+        const cfg = this.tsConfig;
+        const where = buildMetadataWhere(options.filter, c.metadata);
+        const clauses = [`${this.tsvector()} @@ __any.q`];
+        if (where.sql !== '') {
+            clauses.push(where.sql.slice('WHERE '.length));
+        }
+        const raw = await this.db.rawQuery(`SELECT id, text, source, metadata, score FROM (
+         WITH __terms AS (
+           SELECT DISTINCT to_tsquery('${cfg}', quote_literal(t)) AS q
+           FROM regexp_split_to_table(?, ' ') AS t
+         ), __cand AS MATERIALIZED (
+           SELECT ${c.id} AS id, ${c.text} AS text, ${c.source} AS source, ${c.metadata} AS metadata,
+                  ${this.tsvector()} AS __tsv, __any.q AS __q
+           FROM ${this.table}, to_tsquery('${cfg}', ?) AS __any(q)
+           WHERE ${clauses.join(' AND ')}
+         ), __hits AS MATERIALIZED (
+           SELECT __cand.id AS __id, __terms.q FROM __cand JOIN __terms ON __cand.__tsv @@ __terms.q
+         ), __idf AS (
+           SELECT q, ln(1 + ((SELECT count(*) FROM __cand) - count(*) + 0.5) / (count(*) + 0.5)) AS w
+           FROM __hits GROUP BY q
+         ), __scored AS (
+           SELECT __hits.__id, sum(__idf.w) AS s, rank() OVER (ORDER BY sum(__idf.w) DESC) AS r
+           FROM __hits JOIN __idf USING (q) GROUP BY __hits.__id
+         )
+         SELECT __cand.id, __cand.text, __cand.source, __cand.metadata,
+                __scored.s AS __s, ts_rank_cd(__cand.__tsv, __cand.__q, 32) AS __tr,
+                __scored.s + ts_rank_cd(__cand.__tsv, __cand.__q, 32) AS score
+         FROM __scored JOIN __cand ON __cand.id = __scored.__id
+         WHERE __scored.r <= ?
+       ) AS __ranked
+       ORDER BY __s DESC, __tr DESC, id
+       LIMIT ?`, [terms.join(' '), anyTermTsquery(terms), ...where.bindings, options.topK, options.topK]);
+        return normalizeRows(raw).map(toPassage);
+    }
+    /** Must be character-for-character the index expression, or the planner won't use the index. */
+    tsvector() {
+        return this.tsColumn ?? `to_tsvector('${this.tsConfig}'::regconfig, ${this.col.text})`;
+    }
+}
+function toPassage(row) {
+    const metadata = parseMetadata(row.metadata);
+    const source = row.source;
+    return {
+        id: String(row.id),
+        text: String(row.text),
+        score: Number(row.score),
+        ...(source !== null && source !== undefined ? { source: String(source) } : {}),
+        ...(metadata !== undefined ? { metadata } : {}),
+    };
 }
 /**
  * A {@link import('../spi/retriever.js').Retriever} over a {@link PgVectorStore}: embed the query, then