@statewalker/indexer-pglite 0.2.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mikhail Kotelnikov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,53 @@
+# @statewalker/indexer-pglite
+
+PGlite-backed implementation of `@statewalker/indexer-api`: full-text search via PostgreSQL's native `tsvector` + GIN index, vector search via [pgvector](https://github.com/pgvector/pgvector) with an HNSW cosine index. Runs entirely in-process (in the browser, Node, or anywhere [PGlite](https://github.com/electric-sql/pglite) does).
+
+## Installation
+
+```sh
+pnpm add @statewalker/indexer-pglite @electric-sql/pglite
+```
+
+## Usage
+
+```ts
+import { createPGLiteIndexer } from "@statewalker/indexer-pglite";
+
+// Uses an in-memory PGlite instance by default.
+const indexer = await createPGLiteIndexer();
+
+// Or pass an existing PGlite (with the `vector` extension already loaded):
+// import { PGlite } from "@electric-sql/pglite";
+// import { vector } from "@electric-sql/pglite/vector";
+// const db = await PGlite.create({ dataDir: "idb://myapp", extensions: { vector } });
+// const indexer = await createPGLiteIndexer({ db });
+
+const index = await indexer.createIndex({
+  name: "docs",
+  fulltext: { language: "en" },
+  vector: { dimensionality: 384, model: "all-MiniLM-L6-v2" },
+});
+```
+
+## Runtime extension
+
+`createPGLiteIndexer` runs `CREATE EXTENSION IF NOT EXISTS vector` at init. When the factory creates its own PGlite (no `db` passed), it auto-loads the `vector` extension via PGlite's `extensions: { vector }` option; when a caller-supplied `db` is passed, the caller is responsible for having the extension available (pass `extensions: { vector }` to `PGlite.create`).
+
+## SQL shape
+
+Per named index:
+
+- `idx_<prefix>_docs(doc_id SERIAL PRIMARY KEY, path TEXT UNIQUE)` — path ↔ doc_id mapping.
+- `idx_<prefix>_fts(doc_id, block_id, content, content_tsv TSVECTOR GENERATED ..., metadata)` + `USING GIN (content_tsv)`.
+- `idx_<prefix>_vec(doc_id, block_id, embedding vector(dim))` + `USING hnsw (embedding vector_cosine_ops)`.
+
+Search uses `ts_rank_cd` + `to_tsquery(<lang>, ...)` for FTS, and the `<=>` cosine-distance operator for vector search.
+
+## API
+
+- `createPGLiteIndexer(options?)` — returns `Promise<Indexer>`. Options: `db?: PGlite` (bring your own; otherwise a fresh in-memory instance is created, and the returned indexer owns and will close it on `indexer.close()`).
+- `PGLiteIndexerOptions` — option type.
+
+## Related
+
+- `@statewalker/indexer-api`, `@electric-sql/pglite`, `pgvector`.

package/dist/dialect.js

@@ -0,0 +1,124 @@
+import { buildPathPrefixesSql } from "@statewalker/indexer-core";
+//#region src/dialect.ts
+/** Adapt `@electric-sql/pglite`'s `PGlite` to `@statewalker/indexer-core`'s minimal `SqlDb`. */
+function wrapDbAsSqlDb(db) {
+	return {
+		exec: (sql) => db.exec(sql).then(() => void 0),
+		async query(sql, params) {
+			const { rows } = await db.query(sql, params ?? []);
+			return rows;
+		}
+	};
+}
+const LANGUAGE_MAP = {
+	en: "english",
+	fr: "french",
+	de: "german",
+	es: "spanish",
+	it: "italian",
+	pt: "portuguese",
+	nl: "dutch",
+	ru: "russian",
+	sv: "swedish",
+	no: "norwegian",
+	da: "danish",
+	fi: "finnish",
+	hu: "hungarian",
+	ro: "romanian",
+	tr: "turkish",
+	simple: "simple"
+};
+const KNOWN_PG_CONFIGS = new Set(Object.values(LANGUAGE_MAP));
+function resolvePgLanguage(lang) {
+	const mapped = LANGUAGE_MAP[lang];
+	if (mapped) return mapped;
+	if (KNOWN_PG_CONFIGS.has(lang)) return lang;
+	throw new Error(`pglite FTS: unsupported language "${lang}" (must be an ISO-639-1 code or a Postgres text-search config name)`);
+}
+/**
+* pgvector's `vector(dim)` column accepts either a JSON-array text literal (`"[1,2,3]"`) cast with
+* `::vector(dim)`, or a JS array bound by the driver. The text-literal path is currently used
+* because it works uniformly across the pglite vector extension versions available in the catalog;
+* swapping to native array binding is a follow-up when driver support is confirmed.
+*/
+function embeddingToLiteral(embedding) {
+	return `[${Array.from(embedding).join(",")}]`;
+}
+/**
+* PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring,
+* and `to_tsquery` with a language-specific stemmer.
+*/
+const pgliteFtsDialect = {
+	createTableDdl({ tableName, info }) {
+		return [`CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('${resolvePgLanguage(info.language)}', content)) STORED, metadata TEXT, PRIMARY KEY (doc_id, block_id))`, `CREATE INDEX IF NOT EXISTS ${tableName}_tsv_idx ON ${tableName} USING GIN (content_tsv)`];
+	},
+	async search({ db, tableName, docsTable, info, query, paths, topK }) {
+		const pgLang = resolvePgLanguage(info.language);
+		const validWords = query.toLowerCase().split(/\s+/).map((w) => w.replace(/[&|!():'"\\]/g, "")).filter((w) => w.length > 0);
+		if (validWords.length === 0) return [];
+		const allParams = [validWords.join(" | ")];
+		let pathClause = "";
+		if (paths && paths.length > 0) {
+			const cond = buildPathPrefixesSql("d.path", paths, allParams.length + 1);
+			pathClause = ` AND ${cond.sql}`;
+			allParams.push(...cond.params);
+		}
+		const topKParam = `$${allParams.length + 1}`;
+		allParams.push(topK);
+		const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`;
+		return (await db.query(sql, allParams)).map((row) => ({
+			path: row.path,
+			blockId: row.block_id,
+			content: row.content,
+			score: row.score
+		}));
+	}
+};
+/**
+* PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.
+* Embeddings are currently bound as stringified array literals — Phase 9 switches to parameter-bound
+* driver-native arrays.
+*/
+const pgliteVectorDialect = {
+	createTableDdl({ tableName, indexName, info }) {
+		return [`CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding vector(${info.dimensionality}) NOT NULL, PRIMARY KEY (doc_id, block_id))`, `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING hnsw (embedding vector_cosine_ops)`];
+	},
+	bindEmbedding: embeddingToLiteral,
+	embeddingCastSuffix: (dim) => `::vector(${dim})`,
+	async search({ db, tableName, docsTable, queryEmbedding, paths, topK, info, bindEmbedding, embeddingCastSuffix }) {
+		const vecLiteral = bindEmbedding(queryEmbedding);
+		const dim = info.dimensionality;
+		const allParams = [vecLiteral];
+		let pathClause = "";
+		if (paths && paths.length > 0) {
+			const cond = buildPathPrefixesSql("d.path", paths, allParams.length + 1);
+			pathClause = `WHERE ${cond.sql} `;
+			allParams.push(...cond.params);
+		}
+		const topKParam = `$${allParams.length + 1}`;
+		allParams.push(topK);
+		return (await db.query(`SELECT d.path, b.block_id, (b.embedding <=> $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`, allParams)).map((row) => ({
+			path: row.path,
+			blockId: row.block_id,
+			score: 1 - row.dist
+		}));
+	},
+	decodeEmbedding(raw) {
+		return new Float32Array(JSON.parse(raw));
+	}
+};
+/** Aggregate PGlite dialect for `createSqlBackedIndexer`. */
+const pgliteDialect = {
+	extensionInit: ["CREATE EXTENSION IF NOT EXISTS vector"],
+	docsTableDdl(prefix) {
+		return [`CREATE TABLE IF NOT EXISTS idx_${prefix}_docs (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`];
+	},
+	unionAliasSuffix: " AS combined",
+	supportsDDLInTransaction: true,
+	fts: pgliteFtsDialect,
+	vec: pgliteVectorDialect
+};
+//#endregion
+export { pgliteDialect, pgliteFtsDialect, pgliteVectorDialect, wrapDbAsSqlDb };
+
+//# sourceMappingURL=dialect.js.map

package/dist/dialect.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dialect.js","names":[],"sources":["../src/dialect.ts"],"sourcesContent":["import type { PGlite } from \"@electric-sql/pglite\";\nimport type {\n  SqlBackedDialect,\n  SqlDb,\n  SqlFtsDialect,\n  SqlVectorDialect,\n} from \"@statewalker/indexer-core\";\nimport { buildPathPrefixesSql } from \"@statewalker/indexer-core\";\n\n/** Adapt `@electric-sql/pglite`'s `PGlite` to `@statewalker/indexer-core`'s minimal `SqlDb`. */\nexport function wrapDbAsSqlDb(db: PGlite): SqlDb {\n  return {\n    exec: (sql) => db.exec(sql).then(() => undefined),\n    async query<T>(sql: string, params?: unknown[]): Promise<T[]> {\n      const { rows } = await db.query<T>(sql, params ?? []);\n      return rows;\n    },\n  };\n}\n\nconst LANGUAGE_MAP: Record<string, string> = {\n  en: \"english\",\n  fr: \"french\",\n  de: \"german\",\n  es: \"spanish\",\n  it: \"italian\",\n  pt: \"portuguese\",\n  nl: \"dutch\",\n  ru: \"russian\",\n  sv: \"swedish\",\n  no: \"norwegian\",\n  da: \"danish\",\n  fi: \"finnish\",\n  hu: \"hungarian\",\n  ro: \"romanian\",\n  tr: \"turkish\",\n  simple: \"simple\",\n};\n\n// Postgres text-search configuration names that have always-on availability;\n// any value outside this set is interpolated into DDL/queries and must be a\n// whitelisted identifier — never user-controlled free-form text.\nconst KNOWN_PG_CONFIGS: ReadonlySet<string> = new Set(Object.values(LANGUAGE_MAP));\n\nfunction resolvePgLanguage(lang: string): string {\n  const mapped = LANGUAGE_MAP[lang];\n  if (mapped) return mapped;\n  if (KNOWN_PG_CONFIGS.has(lang)) return lang;\n  throw new Error(\n    `pglite FTS: unsupported language \"${lang}\" (must be an ISO-639-1 code or a Postgres text-search config name)`,\n  );\n}\n\n/**\n * pgvector's `vector(dim)` column accepts either a JSON-array text literal (`\"[1,2,3]\"`) cast with\n * `::vector(dim)`, or a JS array bound by the driver. The text-literal path is currently used\n * because it works uniformly across the pglite vector extension versions available in the catalog;\n * swapping to native array binding is a follow-up when driver support is confirmed.\n */\nfunction embeddingToLiteral(embedding: Float32Array): string {\n  return `[${Array.from(embedding).join(\",\")}]`;\n}\n\n/**\n * PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring,\n * and `to_tsquery` with a language-specific stemmer.\n */\nexport const pgliteFtsDialect: SqlFtsDialect = {\n  createTableDdl({ tableName, info }) {\n    const pgLang = resolvePgLanguage(info.language);\n    return [\n      `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('${pgLang}', content)) STORED, metadata TEXT, PRIMARY KEY (doc_id, block_id))`,\n      `CREATE INDEX IF NOT EXISTS ${tableName}_tsv_idx ON ${tableName} USING GIN (content_tsv)`,\n    ];\n  },\n\n  async search({ db, tableName, docsTable, info, query, paths, topK }) {\n    const pgLang = resolvePgLanguage(info.language);\n    // Strip only tsquery-significant punctuation; keep Unicode letters/digits so\n    // non-ASCII languages (Russian, French, CJK, …) reach the FTS engine intact.\n    const validWords = query\n      .toLowerCase()\n      .split(/\\s+/)\n      .map((w) => w.replace(/[&|!():'\"\\\\]/g, \"\"))\n      .filter((w) => w.length > 0);\n    if (validWords.length === 0) return [];\n\n    const orTerms = validWords.join(\" | \");\n\n    const allParams: unknown[] = [orTerms];\n    let pathClause = \"\";\n    if (paths && paths.length > 0) {\n      const cond = buildPathPrefixesSql(\"d.path\", paths as string[], allParams.length + 1);\n      pathClause = ` AND ${cond.sql}`;\n      allParams.push(...cond.params);\n    }\n\n    const topKParam = `$${allParams.length + 1}`;\n    allParams.push(topK);\n\n    const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`;\n\n    const rows = await db.query<{\n      path: string;\n      block_id: string;\n      content: string;\n      score: number;\n    }>(sql, allParams);\n\n    return rows.map((row) => ({\n      path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n      blockId: row.block_id,\n      content: row.content,\n      score: row.score,\n    }));\n  },\n};\n\n/**\n * PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.\n * Embeddings are currently bound as stringified array literals — Phase 9 switches to parameter-bound\n * driver-native arrays.\n */\nexport const pgliteVectorDialect: SqlVectorDialect = {\n  createTableDdl({ tableName, indexName, info }) {\n    const dim = info.dimensionality;\n    return [\n      `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding vector(${dim}) NOT NULL, PRIMARY KEY (doc_id, block_id))`,\n      `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING hnsw (embedding vector_cosine_ops)`,\n    ];\n  },\n\n  bindEmbedding: embeddingToLiteral,\n  embeddingCastSuffix: (dim) => `::vector(${dim})`,\n\n  async search({\n    db,\n    tableName,\n    docsTable,\n    queryEmbedding,\n    paths,\n    topK,\n    info,\n    bindEmbedding,\n    embeddingCastSuffix,\n  }) {\n    const vecLiteral = bindEmbedding(queryEmbedding);\n    const dim = info.dimensionality;\n\n    const allParams: unknown[] = [vecLiteral];\n    let pathClause = \"\";\n    if (paths && paths.length > 0) {\n      const cond = buildPathPrefixesSql(\"d.path\", paths as string[], allParams.length + 1);\n      pathClause = `WHERE ${cond.sql} `;\n      allParams.push(...cond.params);\n    }\n\n    const topKParam = `$${allParams.length + 1}`;\n    allParams.push(topK);\n\n    const rows = await db.query<{ path: string; block_id: string; dist: number }>(\n      `SELECT d.path, b.block_id, (b.embedding <=> $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`,\n      allParams,\n    );\n\n    return rows.map((row) => ({\n      path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n      blockId: row.block_id,\n      score: 1 - row.dist,\n    }));\n  },\n\n  decodeEmbedding(raw) {\n    // PGlite returns vector as a string \"[1,2,3]\".\n    return new Float32Array(JSON.parse(raw as string) as number[]);\n  },\n};\n\n/** Aggregate PGlite dialect for `createSqlBackedIndexer`. */\nexport const pgliteDialect: SqlBackedDialect = {\n  extensionInit: [\"CREATE EXTENSION IF NOT EXISTS vector\"],\n  docsTableDdl(prefix) {\n    return [\n      `CREATE TABLE IF NOT EXISTS idx_${prefix}_docs (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`,\n    ];\n  },\n  unionAliasSuffix: \" AS combined\",\n  supportsDDLInTransaction: true,\n  fts: pgliteFtsDialect,\n  vec: pgliteVectorDialect,\n};\n"],"mappings":";;;AAUA,SAAgB,cAAc,IAAmB;CAC/C,OAAO;EACL,OAAO,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,WAAW,KAAA,CAAS;EAChD,MAAM,MAAS,KAAa,QAAkC;GAC5D,MAAM,EAAE,SAAS,MAAM,GAAG,MAAS,KAAK,UAAU,CAAC,CAAC;GACpD,OAAO;EACT;CACF;AACF;AAEA,MAAM,eAAuC;CAC3C,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,QAAQ;AACV;AAKA,MAAM,mBAAwC,IAAI,IAAI,OAAO,OAAO,YAAY,CAAC;AAEjF,SAAS,kBAAkB,MAAsB;CAC/C,MAAM,SAAS,aAAa;CAC5B,IAAI,QAAQ,OAAO;CACnB,IAAI,iBAAiB,IAAI,IAAI,GAAG,OAAO;CACvC,MAAM,IAAI,MACR,qCAAqC,KAAK,oEAC5C;AACF;;;;;;;AAQA,SAAS,mBAAmB,WAAiC;CAC3D,OAAO,IAAI,MAAM,KAAK,SAAS,CAAC,CAAC,KAAK,GAAG,EAAE;AAC7C;;;;;AAMA,MAAa,mBAAkC;CAC7C,eAAe,EAAE,WAAW,QAAQ;EAElC,OAAO,CACL,8BAA8B,UAAU,mIAF3B,kBAAkB,KAAK,QAE4I,EAAE,sEAClL,8BAA8B,UAAU,cAAc,UAAU,yBAClE;CACF;CAEA,MAAM,OAAO,EAAE,IAAI,WAAW,WAAW,MAAM,OAAO,OAAO,QAAQ;EACnE,MAAM,SAAS,kBAAkB,KAAK,QAAQ;EAG9C,MAAM,aAAa,MAChB,YAAY,CAAC,CACb,MAAM,KAAK,CAAC,CACZ,KAAK,MAAM,EAAE,QAAQ,iBAAiB,EAAE,CAAC,CAAC,CAC1C,QAAQ,MAAM,EAAE,SAAS,CAAC;EAC7B,IAAI,WAAW,WAAW,GAAG,OAAO,CAAC;EAIrC,MAAM,YAAuB,CAFb,WAAW,KAAK,KAEI,CAAC;EACrC,IAAI,aAAa;EACjB,IAAI,SAAS,MAAM,SAAS,GAAG;GAC7B,MAAM,OAAO,qBAAqB,UAAU,OAAmB,UAAU,SAAS,CAAC;GACnF,aAAa,QAAQ,KAAK;GAC1B,UAAU,KAAK,GAAG,KAAK,MAAM;EAC/B;EAEA,MAAM,YAAY,IAAI,UAAU,SAAS;EACzC,UAAU,KAAK,IAAI;EAEnB,MAAM,MAAM,+EAA+E,OAAO,wBAAwB,UAAU,UAAU,UAAU,+DAA+D,OAAO,QAAQ,WAAW,6BAA6B;EAS9Q,QAAO,MAPY,GAAG,MAKnB,KAAK,SAAS,EAAA,CAEL,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,SAAS,IAAI;GACb,OAAO,IAAI;EACb,EAAE;CACJ;AACF;;;;;;AAOA,MAAa,sBAAwC;CACnD,eAAe,EAAE,WAAW,WAAW,QAAQ;EAE7C,OAAO,CACL,8BAA8B,UAAU,sEAF9B,KAAK,eAEmG,8CAClH,8BAA8B,UAAU,MAAM,UAAU,0CAC1D;CACF;CAEA,eAAe;CACf,sBAAsB,QAAQ,YAAY,IAAI;CAE9C,MAAM,OAAO,EACX,IACA,WACA,WACA,gBACA,OACA,MACA,MACA,eACA,uBACC;EACD,MAAM,aAAa,cAAc,cAAc;EAC/C,MAAM,MAAM,KAAK;EAEjB,MAAM,YAAuB,CAAC,UAAU;EACxC,IAAI,aAAa;EACjB,IAAI,SAAS,MAAM,SAAS,GAAG;GAC7B,MAAM,OAAO,qBAAqB,UAAU,OAAmB,UAAU,SAAS,CAAC;GACnF,aAAa,SAAS,KAAK,IAAI;GAC/B,UAAU,KAAK,GAAG,KAAK,MAAM;EAC/B;EAEA,MAAM,YAAY,IAAI,UAAU,SAAS;EACzC,UAAU,KAAK,IAAI;EAOnB,QAAO,MALY,GAAG,MACpB,iDAAiD,oBAAoB,GAAG,EAAE,iBAAiB,UAAU,UAAU,UAAU,4BAA4B,WAAW,0BAA0B,aAC1L,SACF,EAAA,CAEY,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,OAAO,IAAI,IAAI;EACjB,EAAE;CACJ;CAEA,gBAAgB,KAAK;EAEnB,OAAO,IAAI,aAAa,KAAK,MAAM,GAAa,CAAa;CAC/D;AACF;;AAGA,MAAa,gBAAkC;CAC7C,eAAe,CAAC,uCAAuC;CACvD,aAAa,QAAQ;EACnB,OAAO,CACL,kCAAkC,OAAO,6DAC3C;CACF;CACA,kBAAkB;CAClB,0BAA0B;CAC1B,KAAK;CACL,KAAK;AACP"}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+import { PGLiteIndexerOptions, createPGLiteIndexer } from "./pglite-indexer.js";
+export { type PGLiteIndexerOptions, createPGLiteIndexer };

package/dist/index.js

@@ -0,0 +1,2 @@
+import { createPGLiteIndexer } from "./pglite-indexer.js";
+export { createPGLiteIndexer };

package/dist/pglite-indexer.d.ts

@@ -0,0 +1,17 @@
+import { PGlite } from "@electric-sql/pglite";
+import { Indexer } from "@statewalker/indexer-api";
+//#region src/pglite-indexer.d.ts
+export interface PGLiteIndexerOptions {
+  db?: PGlite;
+}
+/**
+ * PGlite-backed `Indexer` over the open-registry contract.
+ *
+ * Wires the per-Index docs table, per-sub-index FTS / vector tables, and the
+ * `__indexer_manifest` table via `createSqlIndexer`. Each Index is an open
+ * registry of named sub-indexes; the FTS modality uses PostgreSQL `tsvector`
+ * with GIN; the vector modality uses pgvector's `vector(dim)` with HNSW.
+ */
+export declare function createPGLiteIndexer(options?: PGLiteIndexerOptions): Promise<Indexer>;
+//#endregion
+//# sourceMappingURL=pglite-indexer.d.ts.map

package/dist/pglite-indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pglite-indexer.d.ts","names":[],"sources":["../src/pglite-indexer.ts"],"mappings":";;;iBAMiB;EACf,KAAK;;;;;;;;;;wBAWe,oBAAoB,UAAU,uBAAuB,QAAQ"}

package/dist/pglite-indexer.js

@@ -0,0 +1,28 @@
+import { pgliteDialect, wrapDbAsSqlDb } from "./dialect.js";
+import { PGlite } from "@electric-sql/pglite";
+import { vector } from "@electric-sql/pglite-pgvector";
+import { createSqlIndexer } from "@statewalker/indexer-core";
+//#region src/pglite-indexer.ts
+/**
+* PGlite-backed `Indexer` over the open-registry contract.
+*
+* Wires the per-Index docs table, per-sub-index FTS / vector tables, and the
+* `__indexer_manifest` table via `createSqlIndexer`. Each Index is an open
+* registry of named sub-indexes; the FTS modality uses PostgreSQL `tsvector`
+* with GIN; the vector modality uses pgvector's `vector(dim)` with HNSW.
+*/
+async function createPGLiteIndexer(options) {
+	const ownsDb = !options?.db;
+	const db = options?.db ?? await PGlite.create({ extensions: { vector } });
+	return createSqlIndexer({
+		db: wrapDbAsSqlDb(db),
+		dialect: pgliteDialect,
+		onClose: ownsDb ? async () => {
+			await db.close();
+		} : void 0
+	});
+}
+//#endregion
+export { createPGLiteIndexer };
+
+//# sourceMappingURL=pglite-indexer.js.map

package/dist/pglite-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pglite-indexer.js","names":[],"sources":["../src/pglite-indexer.ts"],"sourcesContent":["import { PGlite } from \"@electric-sql/pglite\";\nimport { vector } from \"@electric-sql/pglite-pgvector\";\nimport type { Indexer } from \"@statewalker/indexer-api\";\nimport { createSqlIndexer } from \"@statewalker/indexer-core\";\nimport { pgliteDialect, wrapDbAsSqlDb } from \"./dialect.js\";\n\nexport interface PGLiteIndexerOptions {\n  db?: PGlite;\n}\n\n/**\n * PGlite-backed `Indexer` over the open-registry contract.\n *\n * Wires the per-Index docs table, per-sub-index FTS / vector tables, and the\n * `__indexer_manifest` table via `createSqlIndexer`. Each Index is an open\n * registry of named sub-indexes; the FTS modality uses PostgreSQL `tsvector`\n * with GIN; the vector modality uses pgvector's `vector(dim)` with HNSW.\n */\nexport async function createPGLiteIndexer(options?: PGLiteIndexerOptions): Promise<Indexer> {\n  const ownsDb = !options?.db;\n  const db = options?.db ?? (await PGlite.create({ extensions: { vector } }));\n\n  return createSqlIndexer({\n    db: wrapDbAsSqlDb(db),\n    dialect: pgliteDialect,\n    onClose: ownsDb\n      ? async () => {\n          await db.close();\n        }\n      : undefined,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;AAkBA,eAAsB,oBAAoB,SAAkD;CAC1F,MAAM,SAAS,CAAC,SAAS;CACzB,MAAM,KAAK,SAAS,MAAO,MAAM,OAAO,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;CAEzE,OAAO,iBAAiB;EACtB,IAAI,cAAc,EAAE;EACpB,SAAS;EACT,SAAS,SACL,YAAY;GACV,MAAM,GAAG,MAAM;EACjB,IACA,KAAA;CACN,CAAC;AACH"}

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@statewalker/indexer-pglite",
+  "version": "0.2.1",
+  "private": false,
+  "type": "module",
+  "description": "PGlite-backed indexer implementation: full-text + vector search using in-browser Postgres.",
+  "homepage": "https://github.com/statewalker/statewalker-indexer",
+  "author": {
+    "name": "Mikhail Kotelnikov",
+    "email": "mikhail.kotelnikov@gmail.com"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/statewalker/statewalker-indexer.git"
+  },
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@electric-sql/pglite": "^0.5.8",
+    "@electric-sql/pglite-pgvector": "^0.0.9",
+    "@statewalker/indexer-api": "^0.1.1",
+    "@statewalker/indexer-core": "^0.1.1",
+    "@statewalker/indexer-fulltext": "^0.1.1",
+    "@statewalker/indexer-vector": "^0.1.1"
+  },
+  "devDependencies": {
+    "rimraf": "^6.1.3",
+    "tsdown": "^0.23.0",
+    "typescript": "^7.0.2",
+    "vitest": "^5.0.3",
+    "@statewalker/indexer-tests": "^0.0.0"
+  },
+  "sideEffects": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "dev": "tsdown --watch",
+    "test": "vitest run --passWithNoTests",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "clean": "rimraf dist",
+    "lint": "biome check --write .",
+    "format": "biome format --write ."
+  }
+}

package/src/dialect.ts

@@ -0,0 +1,191 @@
+import type { PGlite } from "@electric-sql/pglite";
+import type {
+  SqlBackedDialect,
+  SqlDb,
+  SqlFtsDialect,
+  SqlVectorDialect,
+} from "@statewalker/indexer-core";
+import { buildPathPrefixesSql } from "@statewalker/indexer-core";
+
+/** Adapt `@electric-sql/pglite`'s `PGlite` to `@statewalker/indexer-core`'s minimal `SqlDb`. */
+export function wrapDbAsSqlDb(db: PGlite): SqlDb {
+  return {
+    exec: (sql) => db.exec(sql).then(() => undefined),
+    async query<T>(sql: string, params?: unknown[]): Promise<T[]> {
+      const { rows } = await db.query<T>(sql, params ?? []);
+      return rows;
+    },
+  };
+}
+
+const LANGUAGE_MAP: Record<string, string> = {
+  en: "english",
+  fr: "french",
+  de: "german",
+  es: "spanish",
+  it: "italian",
+  pt: "portuguese",
+  nl: "dutch",
+  ru: "russian",
+  sv: "swedish",
+  no: "norwegian",
+  da: "danish",
+  fi: "finnish",
+  hu: "hungarian",
+  ro: "romanian",
+  tr: "turkish",
+  simple: "simple",
+};
+
+// Postgres text-search configuration names that have always-on availability;
+// any value outside this set is interpolated into DDL/queries and must be a
+// whitelisted identifier — never user-controlled free-form text.
+const KNOWN_PG_CONFIGS: ReadonlySet<string> = new Set(Object.values(LANGUAGE_MAP));
+
+function resolvePgLanguage(lang: string): string {
+  const mapped = LANGUAGE_MAP[lang];
+  if (mapped) return mapped;
+  if (KNOWN_PG_CONFIGS.has(lang)) return lang;
+  throw new Error(
+    `pglite FTS: unsupported language "${lang}" (must be an ISO-639-1 code or a Postgres text-search config name)`,
+  );
+}
+
+/**
+ * pgvector's `vector(dim)` column accepts either a JSON-array text literal (`"[1,2,3]"`) cast with
+ * `::vector(dim)`, or a JS array bound by the driver. The text-literal path is currently used
+ * because it works uniformly across the pglite vector extension versions available in the catalog;
+ * swapping to native array binding is a follow-up when driver support is confirmed.
+ */
+function embeddingToLiteral(embedding: Float32Array): string {
+  return `[${Array.from(embedding).join(",")}]`;
+}
+
+/**
+ * PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring,
+ * and `to_tsquery` with a language-specific stemmer.
+ */
+export const pgliteFtsDialect: SqlFtsDialect = {
+  createTableDdl({ tableName, info }) {
+    const pgLang = resolvePgLanguage(info.language);
+    return [
+      `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('${pgLang}', content)) STORED, metadata TEXT, PRIMARY KEY (doc_id, block_id))`,
+      `CREATE INDEX IF NOT EXISTS ${tableName}_tsv_idx ON ${tableName} USING GIN (content_tsv)`,
+    ];
+  },
+
+  async search({ db, tableName, docsTable, info, query, paths, topK }) {
+    const pgLang = resolvePgLanguage(info.language);
+    // Strip only tsquery-significant punctuation; keep Unicode letters/digits so
+    // non-ASCII languages (Russian, French, CJK, …) reach the FTS engine intact.
+    const validWords = query
+      .toLowerCase()
+      .split(/\s+/)
+      .map((w) => w.replace(/[&|!():'"\\]/g, ""))
+      .filter((w) => w.length > 0);
+    if (validWords.length === 0) return [];
+
+    const orTerms = validWords.join(" | ");
+
+    const allParams: unknown[] = [orTerms];
+    let pathClause = "";
+    if (paths && paths.length > 0) {
+      const cond = buildPathPrefixesSql("d.path", paths as string[], allParams.length + 1);
+      pathClause = ` AND ${cond.sql}`;
+      allParams.push(...cond.params);
+    }
+
+    const topKParam = `$${allParams.length + 1}`;
+    allParams.push(topK);
+
+    const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`;
+
+    const rows = await db.query<{
+      path: string;
+      block_id: string;
+      content: string;
+      score: number;
+    }>(sql, allParams);
+
+    return rows.map((row) => ({
+      path: row.path as import("@statewalker/indexer-api").DocumentPath,
+      blockId: row.block_id,
+      content: row.content,
+      score: row.score,
+    }));
+  },
+};
+
+/**
+ * PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.
+ * Embeddings are currently bound as stringified array literals — Phase 9 switches to parameter-bound
+ * driver-native arrays.
+ */
+export const pgliteVectorDialect: SqlVectorDialect = {
+  createTableDdl({ tableName, indexName, info }) {
+    const dim = info.dimensionality;
+    return [
+      `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding vector(${dim}) NOT NULL, PRIMARY KEY (doc_id, block_id))`,
+      `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING hnsw (embedding vector_cosine_ops)`,
+    ];
+  },
+
+  bindEmbedding: embeddingToLiteral,
+  embeddingCastSuffix: (dim) => `::vector(${dim})`,
+
+  async search({
+    db,
+    tableName,
+    docsTable,
+    queryEmbedding,
+    paths,
+    topK,
+    info,
+    bindEmbedding,
+    embeddingCastSuffix,
+  }) {
+    const vecLiteral = bindEmbedding(queryEmbedding);
+    const dim = info.dimensionality;
+
+    const allParams: unknown[] = [vecLiteral];
+    let pathClause = "";
+    if (paths && paths.length > 0) {
+      const cond = buildPathPrefixesSql("d.path", paths as string[], allParams.length + 1);
+      pathClause = `WHERE ${cond.sql} `;
+      allParams.push(...cond.params);
+    }
+
+    const topKParam = `$${allParams.length + 1}`;
+    allParams.push(topK);
+
+    const rows = await db.query<{ path: string; block_id: string; dist: number }>(
+      `SELECT d.path, b.block_id, (b.embedding <=> $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`,
+      allParams,
+    );
+
+    return rows.map((row) => ({
+      path: row.path as import("@statewalker/indexer-api").DocumentPath,
+      blockId: row.block_id,
+      score: 1 - row.dist,
+    }));
+  },
+
+  decodeEmbedding(raw) {
+    // PGlite returns vector as a string "[1,2,3]".
+    return new Float32Array(JSON.parse(raw as string) as number[]);
+  },
+};
+
+/** Aggregate PGlite dialect for `createSqlBackedIndexer`. */
+export const pgliteDialect: SqlBackedDialect = {
+  extensionInit: ["CREATE EXTENSION IF NOT EXISTS vector"],
+  docsTableDdl(prefix) {
+    return [
+      `CREATE TABLE IF NOT EXISTS idx_${prefix}_docs (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`,
+    ];
+  },
+  unionAliasSuffix: " AS combined",
+  supportsDDLInTransaction: true,
+  fts: pgliteFtsDialect,
+  vec: pgliteVectorDialect,
+};

package/src/index.ts

@@ -0,0 +1,2 @@
+export type { PGLiteIndexerOptions } from "./pglite-indexer.js";
+export { createPGLiteIndexer } from "./pglite-indexer.js";

package/src/pglite-indexer.ts

@@ -0,0 +1,32 @@
+import { PGlite } from "@electric-sql/pglite";
+import { vector } from "@electric-sql/pglite-pgvector";
+import type { Indexer } from "@statewalker/indexer-api";
+import { createSqlIndexer } from "@statewalker/indexer-core";
+import { pgliteDialect, wrapDbAsSqlDb } from "./dialect.js";
+
+export interface PGLiteIndexerOptions {
+  db?: PGlite;
+}
+
+/**
+ * PGlite-backed `Indexer` over the open-registry contract.
+ *
+ * Wires the per-Index docs table, per-sub-index FTS / vector tables, and the
+ * `__indexer_manifest` table via `createSqlIndexer`. Each Index is an open
+ * registry of named sub-indexes; the FTS modality uses PostgreSQL `tsvector`
+ * with GIN; the vector modality uses pgvector's `vector(dim)` with HNSW.
+ */
+export async function createPGLiteIndexer(options?: PGLiteIndexerOptions): Promise<Indexer> {
+  const ownsDb = !options?.db;
+  const db = options?.db ?? (await PGlite.create({ extensions: { vector } }));
+
+  return createSqlIndexer({
+    db: wrapDbAsSqlDb(db),
+    dialect: pgliteDialect,
+    onClose: ownsDb
+      ? async () => {
+          await db.close();
+        }
+      : undefined,
+  });
+}

package/src/text-codec.d.ts

@@ -0,0 +1,3 @@
+// See ../../indexer-core/src/text-codec.d.ts for rationale.
+declare const TextEncoder: { new (): { encode(input: string): Uint8Array } };
+declare const TextDecoder: { new (label?: string): { decode(input?: Uint8Array): string } };