@statewalker/indexer-pglite 0.3.0 → 0.3.2

package/README.md

@@ -85,7 +85,7 @@ idx_<index>_<sub>_vec(doc_id, block_id, embedding vector(dim))
 
 ### Queries, scores and languages
 
-- A full-text query is lowercased, split on whitespace, stripped of tsquery operators (`& | ! ( ) : ' " \`) and joined with `|` (OR). Score is `ts_rank_cd`. With several `queries` in one sub-query, each block keeps its best score across them (scores are not summed).
+- A full-text query is lowercased, split into words on every character that is not a Unicode letter, combining mark, digit or `_` (so no tsquery operator such as `&`, `|`, `<->` or `:` gets through, and `foo:bar` becomes `foo | bar`), and joined with `|` (OR). Score is `ts_rank_cd`. With several `queries` in one sub-query, each block keeps its best score across them (scores are not summed).
 - Vector `score` is `1 - (embedding <=> query)` (cosine). Embeddings are sent as text literals cast to `vector(dim)`.
 - `language` accepts ISO-639-1 codes (`en`, `fr`, `de`, `es`, `it`, `pt`, `nl`, `ru`, `sv`, `no`, `da`, `fi`, `hu`, `ro`, `tr`), `simple`, or the matching Postgres config names (`english`, ...). Anything else throws `pglite FTS: unsupported language "<x>" (must be an ISO-639-1 code or a Postgres text-search config name)`. The value is put into SQL, which is why it is checked against this list.
 - Default retrieval depth is `topK = 100` per sub-query.

package/dist/dialect.js

@@ -1,21 +1,7 @@
-import { buildPathPrefixesSql } from "@statewalker/indexer-core";
+import { ISO_639_1_LANGUAGE_NAMES, buildSearchFilterSql, embeddingToLiteral, sqlDocsTableName } from "@statewalker/indexer-core";
 //#region src/dialect.ts
 const LANGUAGE_MAP = {
-	en: "english",
-	fr: "french",
-	de: "german",
-	es: "spanish",
-	it: "italian",
-	pt: "portuguese",
-	nl: "dutch",
-	ru: "russian",
-	sv: "swedish",
-	no: "norwegian",
-	da: "danish",
-	fi: "finnish",
-	hu: "hungarian",
-	ro: "romanian",
-	tr: "turkish",
+	...ISO_639_1_LANGUAGE_NAMES,
 	simple: "simple"
 };
 const KNOWN_PG_CONFIGS = new Set(Object.values(LANGUAGE_MAP));
@@ -26,15 +12,6 @@ function resolvePgLanguage(lang) {
 	throw new Error(`pglite FTS: unsupported language "${lang}" (must be an ISO-639-1 code or a Postgres text-search config name)`);
 }
 /**
-* pgvector's `vector(dim)` column accepts either a JSON-array text literal (`"[1,2,3]"`) cast with
-* `::vector(dim)`, or a JS array bound by the driver. The text-literal path is currently used
-* because it works uniformly across the pglite vector extension versions available in the catalog;
-* swapping to native array binding is a follow-up when driver support is confirmed.
-*/
-function embeddingToLiteral(embedding) {
-	return `[${Array.from(embedding).join(",")}]`;
-}
-/**
 * PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring,
 * and `to_tsquery` with a language-specific stemmer.
 */
@@ -44,18 +21,11 @@ const pgliteFtsDialect = {
 	},
 	async search({ db, tableName, docsTable, info, query, paths, topK }) {
 		const pgLang = resolvePgLanguage(info.language);
-		const validWords = query.toLowerCase().split(/\s+/).map((w) => w.replace(/[&|!():'"\\]/g, "")).filter((w) => w.length > 0);
+		const validWords = query.toLowerCase().split(/[^\p{L}\p{M}\p{N}_]+/u).filter((w) => w.length > 0);
 		if (validWords.length === 0) return [];
-		const allParams = [validWords.join(" | ")];
-		let pathClause = "";
-		if (paths && paths.length > 0) {
-			const cond = buildPathPrefixesSql("d.path", paths, allParams.length + 1);
-			pathClause = ` AND ${cond.sql}`;
-			allParams.push(...cond.params);
-		}
-		const topKParam = `$${allParams.length + 1}`;
-		allParams.push(topK);
-		const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`;
+		const orTerms = validWords.join(" | ");
+		const { pathCondition, topKParam, params: allParams } = buildSearchFilterSql("d.path", paths, topK, [orTerms]);
+		const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathCondition ? ` AND ${pathCondition}` : ""} ORDER BY score DESC LIMIT ${topKParam}`;
 		return (await db.query(sql, allParams)).map((row) => ({
 			path: row.path,
 			blockId: row.block_id,
@@ -66,8 +36,7 @@ const pgliteFtsDialect = {
 };
 /**
 * PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.
-* Embeddings are currently bound as stringified array literals — Phase 9 switches to parameter-bound
-* driver-native arrays.
+* Embeddings are bound as stringified array literals.
 */
 const pgliteVectorDialect = {
 	createTableDdl({ tableName, indexName, info }) {
@@ -78,15 +47,8 @@ const pgliteVectorDialect = {
 	async search({ db, tableName, docsTable, queryEmbedding, paths, topK, info, bindEmbedding, embeddingCastSuffix }) {
 		const vecLiteral = bindEmbedding(queryEmbedding);
 		const dim = info.dimensionality;
-		const allParams = [vecLiteral];
-		let pathClause = "";
-		if (paths && paths.length > 0) {
-			const cond = buildPathPrefixesSql("d.path", paths, allParams.length + 1);
-			pathClause = `WHERE ${cond.sql} `;
-			allParams.push(...cond.params);
-		}
-		const topKParam = `$${allParams.length + 1}`;
-		allParams.push(topK);
+		const { pathCondition, topKParam, params: allParams } = buildSearchFilterSql("d.path", paths, topK, [vecLiteral]);
+		const pathClause = pathCondition ? `WHERE ${pathCondition} ` : "";
 		return (await db.query(`SELECT d.path, b.block_id, (b.embedding <=> $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`, allParams)).map((row) => ({
 			path: row.path,
 			blockId: row.block_id,
@@ -101,7 +63,7 @@ const pgliteVectorDialect = {
 const pgliteDialect = {
 	extensionInit: ["CREATE EXTENSION IF NOT EXISTS vector"],
 	docsTableDdl(prefix) {
-		return [`CREATE TABLE IF NOT EXISTS idx_${prefix}_docs (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`];
+		return [`CREATE TABLE IF NOT EXISTS ${sqlDocsTableName(prefix)} (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`];
 	},
 	unionAliasSuffix: " AS combined",
 	supportsDDLInTransaction: true,

package/dist/dialect.js.map

@@ -1 +1 @@
-{"version":3,"file":"dialect.js","names":[],"sources":["../src/dialect.ts"],"sourcesContent":["import type { SqlBackedDialect, SqlFtsDialect, SqlVectorDialect } from \"@statewalker/indexer-core\";\nimport { buildPathPrefixesSql } from \"@statewalker/indexer-core\";\n\nconst LANGUAGE_MAP: Record<string, string> = {\n  en: \"english\",\n  fr: \"french\",\n  de: \"german\",\n  es: \"spanish\",\n  it: \"italian\",\n  pt: \"portuguese\",\n  nl: \"dutch\",\n  ru: \"russian\",\n  sv: \"swedish\",\n  no: \"norwegian\",\n  da: \"danish\",\n  fi: \"finnish\",\n  hu: \"hungarian\",\n  ro: \"romanian\",\n  tr: \"turkish\",\n  simple: \"simple\",\n};\n\n// Postgres text-search configuration names that have always-on availability;\n// any value outside this set is interpolated into DDL/queries and must be a\n// whitelisted identifier — never user-controlled free-form text.\nconst KNOWN_PG_CONFIGS: ReadonlySet<string> = new Set(Object.values(LANGUAGE_MAP));\n\nfunction resolvePgLanguage(lang: string): string {\n  const mapped = LANGUAGE_MAP[lang];\n  if (mapped) return mapped;\n  if (KNOWN_PG_CONFIGS.has(lang)) return lang;\n  throw new Error(\n    `pglite FTS: unsupported language \"${lang}\" (must be an ISO-639-1 code or a Postgres text-search config name)`,\n  );\n}\n\n/**\n * pgvector's `vector(dim)` column accepts either a JSON-array text literal (`\"[1,2,3]\"`) cast with\n * `::vector(dim)`, or a JS array bound by the driver. The text-literal path is currently used\n * because it works uniformly across the pglite vector extension versions available in the catalog;\n * swapping to native array binding is a follow-up when driver support is confirmed.\n */\nfunction embeddingToLiteral(embedding: Float32Array): string {\n  return `[${Array.from(embedding).join(\",\")}]`;\n}\n\n/**\n * PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring,\n * and `to_tsquery` with a language-specific stemmer.\n */\nexport const pgliteFtsDialect: SqlFtsDialect = {\n  createTableDdl({ tableName, info }) {\n    const pgLang = resolvePgLanguage(info.language);\n    return [\n      `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('${pgLang}', content)) STORED, metadata TEXT, PRIMARY KEY (doc_id, block_id))`,\n      `CREATE INDEX IF NOT EXISTS ${tableName}_tsv_idx ON ${tableName} USING GIN (content_tsv)`,\n    ];\n  },\n\n  async search({ db, tableName, docsTable, info, query, paths, topK }) {\n    const pgLang = resolvePgLanguage(info.language);\n    // Strip only tsquery-significant punctuation; keep Unicode letters/digits so\n    // non-ASCII languages (Russian, French, CJK, …) reach the FTS engine intact.\n    const validWords = query\n      .toLowerCase()\n      .split(/\\s+/)\n      .map((w) => w.replace(/[&|!():'\"\\\\]/g, \"\"))\n      .filter((w) => w.length > 0);\n    if (validWords.length === 0) return [];\n\n    const orTerms = validWords.join(\" | \");\n\n    const allParams: unknown[] = [orTerms];\n    let pathClause = \"\";\n    if (paths && paths.length > 0) {\n      const cond = buildPathPrefixesSql(\"d.path\", paths as string[], allParams.length + 1);\n      pathClause = ` AND ${cond.sql}`;\n      allParams.push(...cond.params);\n    }\n\n    const topKParam = `$${allParams.length + 1}`;\n    allParams.push(topK);\n\n    const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`;\n\n    const rows = await db.query<{\n      path: string;\n      block_id: string;\n      content: string;\n      score: number;\n    }>(sql, allParams);\n\n    return rows.map((row) => ({\n      path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n      blockId: row.block_id,\n      content: row.content,\n      score: row.score,\n    }));\n  },\n};\n\n/**\n * PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.\n * Embeddings are currently bound as stringified array literals — Phase 9 switches to parameter-bound\n * driver-native arrays.\n */\nexport const pgliteVectorDialect: SqlVectorDialect = {\n  createTableDdl({ tableName, indexName, info }) {\n    const dim = info.dimensionality;\n    return [\n      `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding vector(${dim}) NOT NULL, PRIMARY KEY (doc_id, block_id))`,\n      `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING hnsw (embedding vector_cosine_ops)`,\n    ];\n  },\n\n  bindEmbedding: embeddingToLiteral,\n  embeddingCastSuffix: (dim) => `::vector(${dim})`,\n\n  async search({\n    db,\n    tableName,\n    docsTable,\n    queryEmbedding,\n    paths,\n    topK,\n    info,\n    bindEmbedding,\n    embeddingCastSuffix,\n  }) {\n    const vecLiteral = bindEmbedding(queryEmbedding);\n    const dim = info.dimensionality;\n\n    const allParams: unknown[] = [vecLiteral];\n    let pathClause = \"\";\n    if (paths && paths.length > 0) {\n      const cond = buildPathPrefixesSql(\"d.path\", paths as string[], allParams.length + 1);\n      pathClause = `WHERE ${cond.sql} `;\n      allParams.push(...cond.params);\n    }\n\n    const topKParam = `$${allParams.length + 1}`;\n    allParams.push(topK);\n\n    const rows = await db.query<{ path: string; block_id: string; dist: number }>(\n      `SELECT d.path, b.block_id, (b.embedding <=> $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`,\n      allParams,\n    );\n\n    return rows.map((row) => ({\n      path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n      blockId: row.block_id,\n      score: 1 - row.dist,\n    }));\n  },\n\n  decodeEmbedding(raw) {\n    // PGlite returns vector as a string \"[1,2,3]\".\n    return new Float32Array(JSON.parse(raw as string) as number[]);\n  },\n};\n\n/** Aggregate PGlite dialect for `createSqlBackedIndexer`. */\nexport const pgliteDialect: SqlBackedDialect = {\n  extensionInit: [\"CREATE EXTENSION IF NOT EXISTS vector\"],\n  docsTableDdl(prefix) {\n    return [\n      `CREATE TABLE IF NOT EXISTS idx_${prefix}_docs (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`,\n    ];\n  },\n  unionAliasSuffix: \" AS combined\",\n  supportsDDLInTransaction: true,\n  fts: pgliteFtsDialect,\n  vec: pgliteVectorDialect,\n};\n"],"mappings":";;AAGA,MAAM,eAAuC;CAC3C,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,QAAQ;AACV;AAKA,MAAM,mBAAwC,IAAI,IAAI,OAAO,OAAO,YAAY,CAAC;AAEjF,SAAS,kBAAkB,MAAsB;CAC/C,MAAM,SAAS,aAAa;CAC5B,IAAI,QAAQ,OAAO;CACnB,IAAI,iBAAiB,IAAI,IAAI,GAAG,OAAO;CACvC,MAAM,IAAI,MACR,qCAAqC,KAAK,oEAC5C;AACF;;;;;;;AAQA,SAAS,mBAAmB,WAAiC;CAC3D,OAAO,IAAI,MAAM,KAAK,SAAS,CAAC,CAAC,KAAK,GAAG,EAAE;AAC7C;;;;;AAMA,MAAa,mBAAkC;CAC7C,eAAe,EAAE,WAAW,QAAQ;EAElC,OAAO,CACL,8BAA8B,UAAU,mIAF3B,kBAAkB,KAAK,QAE4I,EAAE,sEAClL,8BAA8B,UAAU,cAAc,UAAU,yBAClE;CACF;CAEA,MAAM,OAAO,EAAE,IAAI,WAAW,WAAW,MAAM,OAAO,OAAO,QAAQ;EACnE,MAAM,SAAS,kBAAkB,KAAK,QAAQ;EAG9C,MAAM,aAAa,MAChB,YAAY,CAAC,CACb,MAAM,KAAK,CAAC,CACZ,KAAK,MAAM,EAAE,QAAQ,iBAAiB,EAAE,CAAC,CAAC,CAC1C,QAAQ,MAAM,EAAE,SAAS,CAAC;EAC7B,IAAI,WAAW,WAAW,GAAG,OAAO,CAAC;EAIrC,MAAM,YAAuB,CAFb,WAAW,KAAK,KAEI,CAAC;EACrC,IAAI,aAAa;EACjB,IAAI,SAAS,MAAM,SAAS,GAAG;GAC7B,MAAM,OAAO,qBAAqB,UAAU,OAAmB,UAAU,SAAS,CAAC;GACnF,aAAa,QAAQ,KAAK;GAC1B,UAAU,KAAK,GAAG,KAAK,MAAM;EAC/B;EAEA,MAAM,YAAY,IAAI,UAAU,SAAS;EACzC,UAAU,KAAK,IAAI;EAEnB,MAAM,MAAM,+EAA+E,OAAO,wBAAwB,UAAU,UAAU,UAAU,+DAA+D,OAAO,QAAQ,WAAW,6BAA6B;EAS9Q,QAAO,MAPY,GAAG,MAKnB,KAAK,SAAS,EAAA,CAEL,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,SAAS,IAAI;GACb,OAAO,IAAI;EACb,EAAE;CACJ;AACF;;;;;;AAOA,MAAa,sBAAwC;CACnD,eAAe,EAAE,WAAW,WAAW,QAAQ;EAE7C,OAAO,CACL,8BAA8B,UAAU,sEAF9B,KAAK,eAEmG,8CAClH,8BAA8B,UAAU,MAAM,UAAU,0CAC1D;CACF;CAEA,eAAe;CACf,sBAAsB,QAAQ,YAAY,IAAI;CAE9C,MAAM,OAAO,EACX,IACA,WACA,WACA,gBACA,OACA,MACA,MACA,eACA,uBACC;EACD,MAAM,aAAa,cAAc,cAAc;EAC/C,MAAM,MAAM,KAAK;EAEjB,MAAM,YAAuB,CAAC,UAAU;EACxC,IAAI,aAAa;EACjB,IAAI,SAAS,MAAM,SAAS,GAAG;GAC7B,MAAM,OAAO,qBAAqB,UAAU,OAAmB,UAAU,SAAS,CAAC;GACnF,aAAa,SAAS,KAAK,IAAI;GAC/B,UAAU,KAAK,GAAG,KAAK,MAAM;EAC/B;EAEA,MAAM,YAAY,IAAI,UAAU,SAAS;EACzC,UAAU,KAAK,IAAI;EAOnB,QAAO,MALY,GAAG,MACpB,iDAAiD,oBAAoB,GAAG,EAAE,iBAAiB,UAAU,UAAU,UAAU,4BAA4B,WAAW,0BAA0B,aAC1L,SACF,EAAA,CAEY,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,OAAO,IAAI,IAAI;EACjB,EAAE;CACJ;CAEA,gBAAgB,KAAK;EAEnB,OAAO,IAAI,aAAa,KAAK,MAAM,GAAa,CAAa;CAC/D;AACF;;AAGA,MAAa,gBAAkC;CAC7C,eAAe,CAAC,uCAAuC;CACvD,aAAa,QAAQ;EACnB,OAAO,CACL,kCAAkC,OAAO,6DAC3C;CACF;CACA,kBAAkB;CAClB,0BAA0B;CAC1B,KAAK;CACL,KAAK;AACP"}
+{"version":3,"file":"dialect.js","names":[],"sources":["../src/dialect.ts"],"sourcesContent":["import type { SqlBackedDialect, SqlFtsDialect, SqlVectorDialect } from \"@statewalker/indexer-core\";\nimport {\n  buildSearchFilterSql,\n  embeddingToLiteral,\n  ISO_639_1_LANGUAGE_NAMES,\n  sqlDocsTableName,\n} from \"@statewalker/indexer-core\";\n\n// ISO-639-1 codes map to the Postgres text-search configuration of the same language name.\nconst LANGUAGE_MAP: Record<string, string> = {\n  ...ISO_639_1_LANGUAGE_NAMES,\n  simple: \"simple\",\n};\n\n// Postgres text-search configuration names that have always-on availability;\n// any value outside this set is interpolated into DDL/queries and must be a\n// whitelisted identifier — never user-controlled free-form text.\nconst KNOWN_PG_CONFIGS: ReadonlySet<string> = new Set(Object.values(LANGUAGE_MAP));\n\nfunction resolvePgLanguage(lang: string): string {\n  const mapped = LANGUAGE_MAP[lang];\n  if (mapped) return mapped;\n  if (KNOWN_PG_CONFIGS.has(lang)) return lang;\n  throw new Error(\n    `pglite FTS: unsupported language \"${lang}\" (must be an ISO-639-1 code or a Postgres text-search config name)`,\n  );\n}\n\n/**\n * PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring,\n * and `to_tsquery` with a language-specific stemmer.\n */\nexport const pgliteFtsDialect: SqlFtsDialect = {\n  createTableDdl({ tableName, info }) {\n    const pgLang = resolvePgLanguage(info.language);\n    return [\n      `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('${pgLang}', content)) STORED, metadata TEXT, PRIMARY KEY (doc_id, block_id))`,\n      `CREATE INDEX IF NOT EXISTS ${tableName}_tsv_idx ON ${tableName} USING GIN (content_tsv)`,\n    ];\n  },\n\n  async search({ db, tableName, docsTable, info, query, paths, topK }) {\n    const pgLang = resolvePgLanguage(info.language);\n    // Keep only words: split on everything that is not a Unicode letter, combining mark, digit\n    // or `_`, so no tsquery operator (`& | ! ( ) : < > <-> <N>`, quotes, backslash) reaches\n    // `to_tsquery`, and non-ASCII words (Russian, Hindi, decomposed accents, …) stay whole.\n    const validWords = query\n      .toLowerCase()\n      .split(/[^\\p{L}\\p{M}\\p{N}_]+/u)\n      .filter((w) => w.length > 0);\n    if (validWords.length === 0) return [];\n\n    const orTerms = validWords.join(\" | \");\n\n    const {\n      pathCondition,\n      topKParam,\n      params: allParams,\n    } = buildSearchFilterSql(\"d.path\", paths, topK, [orTerms]);\n    const pathClause = pathCondition ? ` AND ${pathCondition}` : \"\";\n\n    const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`;\n\n    const rows = await db.query<{\n      path: string;\n      block_id: string;\n      content: string;\n      score: number;\n    }>(sql, allParams);\n\n    return rows.map((row) => ({\n      path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n      blockId: row.block_id,\n      content: row.content,\n      score: row.score,\n    }));\n  },\n};\n\n/**\n * PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.\n * Embeddings are bound as stringified array literals.\n */\nexport const pgliteVectorDialect: SqlVectorDialect = {\n  createTableDdl({ tableName, indexName, info }) {\n    const dim = info.dimensionality;\n    return [\n      `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding vector(${dim}) NOT NULL, PRIMARY KEY (doc_id, block_id))`,\n      `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING hnsw (embedding vector_cosine_ops)`,\n    ];\n  },\n\n  // pgvector's `vector(dim)` column accepts either a JSON-array text literal (`\"[1,2,3]\"`) cast with\n  // `::vector(dim)`, or a JS array bound by the driver. The text-literal path is used because it\n  // works uniformly across the pglite vector extension versions available in the catalog; swapping\n  // to native array binding is a follow-up when driver support is confirmed.\n  bindEmbedding: embeddingToLiteral,\n  embeddingCastSuffix: (dim) => `::vector(${dim})`,\n\n  async search({\n    db,\n    tableName,\n    docsTable,\n    queryEmbedding,\n    paths,\n    topK,\n    info,\n    bindEmbedding,\n    embeddingCastSuffix,\n  }) {\n    const vecLiteral = bindEmbedding(queryEmbedding);\n    const dim = info.dimensionality;\n\n    const {\n      pathCondition,\n      topKParam,\n      params: allParams,\n    } = buildSearchFilterSql(\"d.path\", paths, topK, [vecLiteral]);\n    const pathClause = pathCondition ? `WHERE ${pathCondition} ` : \"\";\n\n    const rows = await db.query<{ path: string; block_id: string; dist: number }>(\n      `SELECT d.path, b.block_id, (b.embedding <=> $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`,\n      allParams,\n    );\n\n    return rows.map((row) => ({\n      path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n      blockId: row.block_id,\n      score: 1 - row.dist,\n    }));\n  },\n\n  decodeEmbedding(raw) {\n    // PGlite returns vector as a string \"[1,2,3]\".\n    return new Float32Array(JSON.parse(raw as string) as number[]);\n  },\n};\n\n/** Aggregate PGlite dialect for `createSqlBackedIndexer`. */\nexport const pgliteDialect: SqlBackedDialect = {\n  extensionInit: [\"CREATE EXTENSION IF NOT EXISTS vector\"],\n  docsTableDdl(prefix) {\n    return [\n      `CREATE TABLE IF NOT EXISTS ${sqlDocsTableName(prefix)} (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`,\n    ];\n  },\n  unionAliasSuffix: \" AS combined\",\n  supportsDDLInTransaction: true,\n  fts: pgliteFtsDialect,\n  vec: pgliteVectorDialect,\n};\n"],"mappings":";;AASA,MAAM,eAAuC;CAC3C,GAAG;CACH,QAAQ;AACV;AAKA,MAAM,mBAAwC,IAAI,IAAI,OAAO,OAAO,YAAY,CAAC;AAEjF,SAAS,kBAAkB,MAAsB;CAC/C,MAAM,SAAS,aAAa;CAC5B,IAAI,QAAQ,OAAO;CACnB,IAAI,iBAAiB,IAAI,IAAI,GAAG,OAAO;CACvC,MAAM,IAAI,MACR,qCAAqC,KAAK,oEAC5C;AACF;;;;;AAMA,MAAa,mBAAkC;CAC7C,eAAe,EAAE,WAAW,QAAQ;EAElC,OAAO,CACL,8BAA8B,UAAU,mIAF3B,kBAAkB,KAAK,QAE4I,EAAE,sEAClL,8BAA8B,UAAU,cAAc,UAAU,yBAClE;CACF;CAEA,MAAM,OAAO,EAAE,IAAI,WAAW,WAAW,MAAM,OAAO,OAAO,QAAQ;EACnE,MAAM,SAAS,kBAAkB,KAAK,QAAQ;EAI9C,MAAM,aAAa,MAChB,YAAY,CAAC,CACb,MAAM,uBAAuB,CAAC,CAC9B,QAAQ,MAAM,EAAE,SAAS,CAAC;EAC7B,IAAI,WAAW,WAAW,GAAG,OAAO,CAAC;EAErC,MAAM,UAAU,WAAW,KAAK,KAAK;EAErC,MAAM,EACJ,eACA,WACA,QAAQ,cACN,qBAAqB,UAAU,OAAO,MAAM,CAAC,OAAO,CAAC;EAGzD,MAAM,MAAM,+EAA+E,OAAO,wBAAwB,UAAU,UAAU,UAAU,+DAA+D,OAAO,QAF3M,gBAAgB,QAAQ,kBAAkB,GAEoL,6BAA6B;EAS9Q,QAAO,MAPY,GAAG,MAKnB,KAAK,SAAS,EAAA,CAEL,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,SAAS,IAAI;GACb,OAAO,IAAI;EACb,EAAE;CACJ;AACF;;;;;AAMA,MAAa,sBAAwC;CACnD,eAAe,EAAE,WAAW,WAAW,QAAQ;EAE7C,OAAO,CACL,8BAA8B,UAAU,sEAF9B,KAAK,eAEmG,8CAClH,8BAA8B,UAAU,MAAM,UAAU,0CAC1D;CACF;CAMA,eAAe;CACf,sBAAsB,QAAQ,YAAY,IAAI;CAE9C,MAAM,OAAO,EACX,IACA,WACA,WACA,gBACA,OACA,MACA,MACA,eACA,uBACC;EACD,MAAM,aAAa,cAAc,cAAc;EAC/C,MAAM,MAAM,KAAK;EAEjB,MAAM,EACJ,eACA,WACA,QAAQ,cACN,qBAAqB,UAAU,OAAO,MAAM,CAAC,UAAU,CAAC;EAC5D,MAAM,aAAa,gBAAgB,SAAS,cAAc,KAAK;EAO/D,QAAO,MALY,GAAG,MACpB,iDAAiD,oBAAoB,GAAG,EAAE,iBAAiB,UAAU,UAAU,UAAU,4BAA4B,WAAW,0BAA0B,aAC1L,SACF,EAAA,CAEY,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,OAAO,IAAI,IAAI;EACjB,EAAE;CACJ;CAEA,gBAAgB,KAAK;EAEnB,OAAO,IAAI,aAAa,KAAK,MAAM,GAAa,CAAa;CAC/D;AACF;;AAGA,MAAa,gBAAkC;CAC7C,eAAe,CAAC,uCAAuC;CACvD,aAAa,QAAQ;EACnB,OAAO,CACL,8BAA8B,iBAAiB,MAAM,EAAE,wDACzD;CACF;CACA,kBAAkB;CAClB,0BAA0B;CAC1B,KAAK;CACL,KAAK;AACP"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@statewalker/indexer-pglite",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "private": false,
   "type": "module",
   "description": "PGlite-backed indexer implementation: full-text + vector search over a @statewalker/db-api Db.",
@@ -32,10 +32,10 @@
   ],
   "dependencies": {
     "@statewalker/db-api": "^0.1.0",
-    "@statewalker/indexer-api": "^0.1.3",
-    "@statewalker/indexer-core": "^0.1.2",
-    "@statewalker/indexer-fulltext": "^0.1.3",
-    "@statewalker/indexer-vector": "^0.1.3"
+    "@statewalker/indexer-api": "^0.1.4",
+    "@statewalker/indexer-core": "^0.1.4",
+    "@statewalker/indexer-fulltext": "^0.1.5",
+    "@statewalker/indexer-vector": "^0.1.5"
   },
   "devDependencies": {
     "@electric-sql/pglite-pgvector": "^0.0.9",

package/src/dialect.ts

@@ -1,22 +1,14 @@
 import type { SqlBackedDialect, SqlFtsDialect, SqlVectorDialect } from "@statewalker/indexer-core";
-import { buildPathPrefixesSql } from "@statewalker/indexer-core";
-
+import {
+  buildSearchFilterSql,
+  embeddingToLiteral,
+  ISO_639_1_LANGUAGE_NAMES,
+  sqlDocsTableName,
+} from "@statewalker/indexer-core";
+
+// ISO-639-1 codes map to the Postgres text-search configuration of the same language name.
 const LANGUAGE_MAP: Record<string, string> = {
-  en: "english",
-  fr: "french",
-  de: "german",
-  es: "spanish",
-  it: "italian",
-  pt: "portuguese",
-  nl: "dutch",
-  ru: "russian",
-  sv: "swedish",
-  no: "norwegian",
-  da: "danish",
-  fi: "finnish",
-  hu: "hungarian",
-  ro: "romanian",
-  tr: "turkish",
+  ...ISO_639_1_LANGUAGE_NAMES,
   simple: "simple",
 };
 
@@ -34,16 +26,6 @@ function resolvePgLanguage(lang: string): string {
   );
 }
 
-/**
- * pgvector's `vector(dim)` column accepts either a JSON-array text literal (`"[1,2,3]"`) cast with
- * `::vector(dim)`, or a JS array bound by the driver. The text-literal path is currently used
- * because it works uniformly across the pglite vector extension versions available in the catalog;
- * swapping to native array binding is a follow-up when driver support is confirmed.
- */
-function embeddingToLiteral(embedding: Float32Array): string {
-  return `[${Array.from(embedding).join(",")}]`;
-}
-
 /**
  * PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring,
  * and `to_tsquery` with a language-specific stemmer.
@@ -59,27 +41,23 @@ export const pgliteFtsDialect: SqlFtsDialect = {
 
   async search({ db, tableName, docsTable, info, query, paths, topK }) {
     const pgLang = resolvePgLanguage(info.language);
-    // Strip only tsquery-significant punctuation; keep Unicode letters/digits so
-    // non-ASCII languages (Russian, French, CJK, …) reach the FTS engine intact.
+    // Keep only words: split on everything that is not a Unicode letter, combining mark, digit
+    // or `_`, so no tsquery operator (`& | ! ( ) : < > <-> <N>`, quotes, backslash) reaches
+    // `to_tsquery`, and non-ASCII words (Russian, Hindi, decomposed accents, …) stay whole.
     const validWords = query
       .toLowerCase()
-      .split(/\s+/)
-      .map((w) => w.replace(/[&|!():'"\\]/g, ""))
+      .split(/[^\p{L}\p{M}\p{N}_]+/u)
       .filter((w) => w.length > 0);
     if (validWords.length === 0) return [];
 
     const orTerms = validWords.join(" | ");
 
-    const allParams: unknown[] = [orTerms];
-    let pathClause = "";
-    if (paths && paths.length > 0) {
-      const cond = buildPathPrefixesSql("d.path", paths as string[], allParams.length + 1);
-      pathClause = ` AND ${cond.sql}`;
-      allParams.push(...cond.params);
-    }
-
-    const topKParam = `$${allParams.length + 1}`;
-    allParams.push(topK);
+    const {
+      pathCondition,
+      topKParam,
+      params: allParams,
+    } = buildSearchFilterSql("d.path", paths, topK, [orTerms]);
+    const pathClause = pathCondition ? ` AND ${pathCondition}` : "";
 
     const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`;
 
@@ -101,8 +79,7 @@ export const pgliteFtsDialect: SqlFtsDialect = {
 
 /**
  * PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.
- * Embeddings are currently bound as stringified array literals — Phase 9 switches to parameter-bound
- * driver-native arrays.
+ * Embeddings are bound as stringified array literals.
  */
 export const pgliteVectorDialect: SqlVectorDialect = {
   createTableDdl({ tableName, indexName, info }) {
@@ -113,6 +90,10 @@ export const pgliteVectorDialect: SqlVectorDialect = {
     ];
   },
 
+  // pgvector's `vector(dim)` column accepts either a JSON-array text literal (`"[1,2,3]"`) cast with
+  // `::vector(dim)`, or a JS array bound by the driver. The text-literal path is used because it
+  // works uniformly across the pglite vector extension versions available in the catalog; swapping
+  // to native array binding is a follow-up when driver support is confirmed.
   bindEmbedding: embeddingToLiteral,
   embeddingCastSuffix: (dim) => `::vector(${dim})`,
 
@@ -130,16 +111,12 @@ export const pgliteVectorDialect: SqlVectorDialect = {
     const vecLiteral = bindEmbedding(queryEmbedding);
     const dim = info.dimensionality;
 
-    const allParams: unknown[] = [vecLiteral];
-    let pathClause = "";
-    if (paths && paths.length > 0) {
-      const cond = buildPathPrefixesSql("d.path", paths as string[], allParams.length + 1);
-      pathClause = `WHERE ${cond.sql} `;
-      allParams.push(...cond.params);
-    }
-
-    const topKParam = `$${allParams.length + 1}`;
-    allParams.push(topK);
+    const {
+      pathCondition,
+      topKParam,
+      params: allParams,
+    } = buildSearchFilterSql("d.path", paths, topK, [vecLiteral]);
+    const pathClause = pathCondition ? `WHERE ${pathCondition} ` : "";
 
     const rows = await db.query<{ path: string; block_id: string; dist: number }>(
       `SELECT d.path, b.block_id, (b.embedding <=> $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`,
@@ -164,7 +141,7 @@ export const pgliteDialect: SqlBackedDialect = {
   extensionInit: ["CREATE EXTENSION IF NOT EXISTS vector"],
   docsTableDdl(prefix) {
     return [
-      `CREATE TABLE IF NOT EXISTS idx_${prefix}_docs (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`,
+      `CREATE TABLE IF NOT EXISTS ${sqlDocsTableName(prefix)} (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`,
     ];
   },
   unionAliasSuffix: " AS combined",