@statewalker/indexer-pglite 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dialect.js +1 -2
- package/dist/dialect.js.map +1 -1
- package/package.json +4 -4
- package/src/dialect.ts +1 -2
package/dist/dialect.js
CHANGED
|
@@ -66,8 +66,7 @@ const pgliteFtsDialect = {
|
|
|
66
66
|
};
|
|
67
67
|
/**
|
|
68
68
|
* PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.
|
|
69
|
-
* Embeddings are
|
|
70
|
-
* driver-native arrays.
|
|
69
|
+
* Embeddings are bound as stringified array literals.
|
|
71
70
|
*/
|
|
72
71
|
const pgliteVectorDialect = {
|
|
73
72
|
createTableDdl({ tableName, indexName, info }) {
|
package/dist/dialect.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dialect.js","names":[],"sources":["../src/dialect.ts"],"sourcesContent":["import type { SqlBackedDialect, SqlFtsDialect, SqlVectorDialect } from \"@statewalker/indexer-core\";\nimport { buildPathPrefixesSql } from \"@statewalker/indexer-core\";\n\nconst LANGUAGE_MAP: Record<string, string> = {\n en: \"english\",\n fr: \"french\",\n de: \"german\",\n es: \"spanish\",\n it: \"italian\",\n pt: \"portuguese\",\n nl: \"dutch\",\n ru: \"russian\",\n sv: \"swedish\",\n no: \"norwegian\",\n da: \"danish\",\n fi: \"finnish\",\n hu: \"hungarian\",\n ro: \"romanian\",\n tr: \"turkish\",\n simple: \"simple\",\n};\n\n// Postgres text-search configuration names that have always-on availability;\n// any value outside this set is interpolated into DDL/queries and must be a\n// whitelisted identifier — never user-controlled free-form text.\nconst KNOWN_PG_CONFIGS: ReadonlySet<string> = new Set(Object.values(LANGUAGE_MAP));\n\nfunction resolvePgLanguage(lang: string): string {\n const mapped = LANGUAGE_MAP[lang];\n if (mapped) return mapped;\n if (KNOWN_PG_CONFIGS.has(lang)) return lang;\n throw new Error(\n `pglite FTS: unsupported language \"${lang}\" (must be an ISO-639-1 code or a Postgres text-search config name)`,\n );\n}\n\n/**\n * pgvector's `vector(dim)` column accepts either a JSON-array text literal (`\"[1,2,3]\"`) cast with\n * `::vector(dim)`, or a JS array bound by the driver. The text-literal path is currently used\n * because it works uniformly across the pglite vector extension versions available in the catalog;\n * swapping to native array binding is a follow-up when driver support is confirmed.\n */\nfunction embeddingToLiteral(embedding: Float32Array): string {\n return `[${Array.from(embedding).join(\",\")}]`;\n}\n\n/**\n * PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring,\n * and `to_tsquery` with a language-specific stemmer.\n */\nexport const pgliteFtsDialect: SqlFtsDialect = {\n createTableDdl({ tableName, info }) {\n const pgLang = resolvePgLanguage(info.language);\n return [\n `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('${pgLang}', content)) STORED, metadata TEXT, PRIMARY KEY (doc_id, block_id))`,\n `CREATE INDEX IF NOT EXISTS ${tableName}_tsv_idx ON ${tableName} USING GIN (content_tsv)`,\n ];\n },\n\n async search({ db, tableName, docsTable, info, query, paths, topK }) {\n const pgLang = resolvePgLanguage(info.language);\n // Strip only tsquery-significant punctuation; keep Unicode letters/digits so\n // non-ASCII languages (Russian, French, CJK, …) reach the FTS engine intact.\n const validWords = query\n .toLowerCase()\n .split(/\\s+/)\n .map((w) => w.replace(/[&|!():'\"\\\\]/g, \"\"))\n .filter((w) => w.length > 0);\n if (validWords.length === 0) return [];\n\n const orTerms = validWords.join(\" | \");\n\n const allParams: unknown[] = [orTerms];\n let pathClause = \"\";\n if (paths && paths.length > 0) {\n const cond = buildPathPrefixesSql(\"d.path\", paths as string[], allParams.length + 1);\n pathClause = ` AND ${cond.sql}`;\n allParams.push(...cond.params);\n }\n\n const topKParam = `$${allParams.length + 1}`;\n allParams.push(topK);\n\n const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`;\n\n const rows = await db.query<{\n path: string;\n block_id: string;\n content: string;\n score: number;\n }>(sql, allParams);\n\n return rows.map((row) => ({\n path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n blockId: row.block_id,\n content: row.content,\n score: row.score,\n }));\n },\n};\n\n/**\n * PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.\n * Embeddings are
|
|
1
|
+
{"version":3,"file":"dialect.js","names":[],"sources":["../src/dialect.ts"],"sourcesContent":["import type { SqlBackedDialect, SqlFtsDialect, SqlVectorDialect } from \"@statewalker/indexer-core\";\nimport { buildPathPrefixesSql } from \"@statewalker/indexer-core\";\n\nconst LANGUAGE_MAP: Record<string, string> = {\n en: \"english\",\n fr: \"french\",\n de: \"german\",\n es: \"spanish\",\n it: \"italian\",\n pt: \"portuguese\",\n nl: \"dutch\",\n ru: \"russian\",\n sv: \"swedish\",\n no: \"norwegian\",\n da: \"danish\",\n fi: \"finnish\",\n hu: \"hungarian\",\n ro: \"romanian\",\n tr: \"turkish\",\n simple: \"simple\",\n};\n\n// Postgres text-search configuration names that have always-on availability;\n// any value outside this set is interpolated into DDL/queries and must be a\n// whitelisted identifier — never user-controlled free-form text.\nconst KNOWN_PG_CONFIGS: ReadonlySet<string> = new Set(Object.values(LANGUAGE_MAP));\n\nfunction resolvePgLanguage(lang: string): string {\n const mapped = LANGUAGE_MAP[lang];\n if (mapped) return mapped;\n if (KNOWN_PG_CONFIGS.has(lang)) return lang;\n throw new Error(\n `pglite FTS: unsupported language \"${lang}\" (must be an ISO-639-1 code or a Postgres text-search config name)`,\n );\n}\n\n/**\n * pgvector's `vector(dim)` column accepts either a JSON-array text literal (`\"[1,2,3]\"`) cast with\n * `::vector(dim)`, or a JS array bound by the driver. The text-literal path is currently used\n * because it works uniformly across the pglite vector extension versions available in the catalog;\n * swapping to native array binding is a follow-up when driver support is confirmed.\n */\nfunction embeddingToLiteral(embedding: Float32Array): string {\n return `[${Array.from(embedding).join(\",\")}]`;\n}\n\n/**\n * PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring,\n * and `to_tsquery` with a language-specific stemmer.\n */\nexport const pgliteFtsDialect: SqlFtsDialect = {\n createTableDdl({ tableName, info }) {\n const pgLang = resolvePgLanguage(info.language);\n return [\n `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('${pgLang}', content)) STORED, metadata TEXT, PRIMARY KEY (doc_id, block_id))`,\n `CREATE INDEX IF NOT EXISTS ${tableName}_tsv_idx ON ${tableName} USING GIN (content_tsv)`,\n ];\n },\n\n async search({ db, tableName, docsTable, info, query, paths, topK }) {\n const pgLang = resolvePgLanguage(info.language);\n // Strip only tsquery-significant punctuation; keep Unicode letters/digits so\n // non-ASCII languages (Russian, French, CJK, …) reach the FTS engine intact.\n const validWords = query\n .toLowerCase()\n .split(/\\s+/)\n .map((w) => w.replace(/[&|!():'\"\\\\]/g, \"\"))\n .filter((w) => w.length > 0);\n if (validWords.length === 0) return [];\n\n const orTerms = validWords.join(\" | \");\n\n const allParams: unknown[] = [orTerms];\n let pathClause = \"\";\n if (paths && paths.length > 0) {\n const cond = buildPathPrefixesSql(\"d.path\", paths as string[], allParams.length + 1);\n pathClause = ` AND ${cond.sql}`;\n allParams.push(...cond.params);\n }\n\n const topKParam = `$${allParams.length + 1}`;\n allParams.push(topK);\n\n const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`;\n\n const rows = await db.query<{\n path: string;\n block_id: string;\n content: string;\n score: number;\n }>(sql, allParams);\n\n return rows.map((row) => ({\n path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n blockId: row.block_id,\n content: row.content,\n score: row.score,\n }));\n },\n};\n\n/**\n * PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.\n * Embeddings are bound as stringified array literals.\n */\nexport const pgliteVectorDialect: SqlVectorDialect = {\n createTableDdl({ tableName, indexName, info }) {\n const dim = info.dimensionality;\n return [\n `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding vector(${dim}) NOT NULL, PRIMARY KEY (doc_id, block_id))`,\n `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING hnsw (embedding vector_cosine_ops)`,\n ];\n },\n\n bindEmbedding: embeddingToLiteral,\n embeddingCastSuffix: (dim) => `::vector(${dim})`,\n\n async search({\n db,\n tableName,\n docsTable,\n queryEmbedding,\n paths,\n topK,\n info,\n bindEmbedding,\n embeddingCastSuffix,\n }) {\n const vecLiteral = bindEmbedding(queryEmbedding);\n const dim = info.dimensionality;\n\n const allParams: unknown[] = [vecLiteral];\n let pathClause = \"\";\n if (paths && paths.length > 0) {\n const cond = buildPathPrefixesSql(\"d.path\", paths as string[], allParams.length + 1);\n pathClause = `WHERE ${cond.sql} `;\n allParams.push(...cond.params);\n }\n\n const topKParam = `$${allParams.length + 1}`;\n allParams.push(topK);\n\n const rows = await db.query<{ path: string; block_id: string; dist: number }>(\n `SELECT d.path, b.block_id, (b.embedding <=> $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`,\n allParams,\n );\n\n return rows.map((row) => ({\n path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n blockId: row.block_id,\n score: 1 - row.dist,\n }));\n },\n\n decodeEmbedding(raw) {\n // PGlite returns vector as a string \"[1,2,3]\".\n return new Float32Array(JSON.parse(raw as string) as number[]);\n },\n};\n\n/** Aggregate PGlite dialect for `createSqlBackedIndexer`. */\nexport const pgliteDialect: SqlBackedDialect = {\n extensionInit: [\"CREATE EXTENSION IF NOT EXISTS vector\"],\n docsTableDdl(prefix) {\n return [\n `CREATE TABLE IF NOT EXISTS idx_${prefix}_docs (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`,\n ];\n },\n unionAliasSuffix: \" AS combined\",\n supportsDDLInTransaction: true,\n fts: pgliteFtsDialect,\n vec: pgliteVectorDialect,\n};\n"],"mappings":";;AAGA,MAAM,eAAuC;CAC3C,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,QAAQ;AACV;AAKA,MAAM,mBAAwC,IAAI,IAAI,OAAO,OAAO,YAAY,CAAC;AAEjF,SAAS,kBAAkB,MAAsB;CAC/C,MAAM,SAAS,aAAa;CAC5B,IAAI,QAAQ,OAAO;CACnB,IAAI,iBAAiB,IAAI,IAAI,GAAG,OAAO;CACvC,MAAM,IAAI,MACR,qCAAqC,KAAK,oEAC5C;AACF;;;;;;;AAQA,SAAS,mBAAmB,WAAiC;CAC3D,OAAO,IAAI,MAAM,KAAK,SAAS,CAAC,CAAC,KAAK,GAAG,EAAE;AAC7C;;;;;AAMA,MAAa,mBAAkC;CAC7C,eAAe,EAAE,WAAW,QAAQ;EAElC,OAAO,CACL,8BAA8B,UAAU,mIAF3B,kBAAkB,KAAK,QAE4I,EAAE,sEAClL,8BAA8B,UAAU,cAAc,UAAU,yBAClE;CACF;CAEA,MAAM,OAAO,EAAE,IAAI,WAAW,WAAW,MAAM,OAAO,OAAO,QAAQ;EACnE,MAAM,SAAS,kBAAkB,KAAK,QAAQ;EAG9C,MAAM,aAAa,MAChB,YAAY,CAAC,CACb,MAAM,KAAK,CAAC,CACZ,KAAK,MAAM,EAAE,QAAQ,iBAAiB,EAAE,CAAC,CAAC,CAC1C,QAAQ,MAAM,EAAE,SAAS,CAAC;EAC7B,IAAI,WAAW,WAAW,GAAG,OAAO,CAAC;EAIrC,MAAM,YAAuB,CAFb,WAAW,KAAK,KAEI,CAAC;EACrC,IAAI,aAAa;EACjB,IAAI,SAAS,MAAM,SAAS,GAAG;GAC7B,MAAM,OAAO,qBAAqB,UAAU,OAAmB,UAAU,SAAS,CAAC;GACnF,aAAa,QAAQ,KAAK;GAC1B,UAAU,KAAK,GAAG,KAAK,MAAM;EAC/B;EAEA,MAAM,YAAY,IAAI,UAAU,SAAS;EACzC,UAAU,KAAK,IAAI;EAEnB,MAAM,MAAM,+EAA+E,OAAO,wBAAwB,UAAU,UAAU,UAAU,+DAA+D,OAAO,QAAQ,WAAW,6BAA6B;EAS9Q,QAAO,MAPY,GAAG,MAKnB,KAAK,SAAS,EAAA,CAEL,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,SAAS,IAAI;GACb,OAAO,IAAI;EACb,EAAE;CACJ;AACF;;;;;AAMA,MAAa,sBAAwC;CACnD,eAAe,EAAE,WAAW,WAAW,QAAQ;EAE7C,OAAO,CACL,8BAA8B,UAAU,sEAF9B,KAAK,eAEmG,8CAClH,8BAA8B,UAAU,MAAM,UAAU,0CAC1D;CACF;CAEA,eAAe;CACf,sBAAsB,QAAQ,YAAY,IAAI;CAE9C,MAAM,OAAO,EACX,IACA,WACA,WACA,gBACA,OACA,MACA,MACA,eACA,uBACC;EACD,MAAM,aAAa,cAAc,cAAc;EAC/C,MAAM,MAAM,KAAK;EAEjB,MAAM,YAAuB,CAAC,UAAU;EACxC,IAAI,aAAa;EACjB,IAAI,SAAS,MAAM,SAAS,GAAG;GAC7B,MAAM,OAAO,qBAAqB,UAAU,OAAmB,UAAU,SAAS,CAAC;GACnF,aAAa,SAAS,KAAK,IAAI;GAC/B,UAAU,KAAK,GAAG,KAAK,MAAM;EAC/B;EAEA,MAAM,YAAY,IAAI,UAAU,SAAS;EACzC,UAAU,KAAK,IAAI;EAOnB,QAAO,MALY,GAAG,MACpB,iDAAiD,oBAAoB,GAAG,EAAE,iBAAiB,UAAU,UAAU,UAAU,4BAA4B,WAAW,0BAA0B,aAC1L,SACF,EAAA,CAEY,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,OAAO,IAAI,IAAI;EACjB,EAAE;CACJ;CAEA,gBAAgB,KAAK;EAEnB,OAAO,IAAI,aAAa,KAAK,MAAM,GAAa,CAAa;CAC/D;AACF;;AAGA,MAAa,gBAAkC;CAC7C,eAAe,CAAC,uCAAuC;CACvD,aAAa,QAAQ;EACnB,OAAO,CACL,kCAAkC,OAAO,6DAC3C;CACF;CACA,kBAAkB;CAClB,0BAA0B;CAC1B,KAAK;CACL,KAAK;AACP"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/indexer-pglite",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "PGlite-backed indexer implementation: full-text + vector search over a @statewalker/db-api Db.",
|
|
@@ -33,9 +33,9 @@
|
|
|
33
33
|
"dependencies": {
|
|
34
34
|
"@statewalker/db-api": "^0.1.0",
|
|
35
35
|
"@statewalker/indexer-api": "^0.1.3",
|
|
36
|
-
"@statewalker/indexer-
|
|
37
|
-
"@statewalker/indexer-
|
|
38
|
-
"@statewalker/indexer-vector": "^0.1.
|
|
36
|
+
"@statewalker/indexer-fulltext": "^0.1.4",
|
|
37
|
+
"@statewalker/indexer-core": "^0.1.3",
|
|
38
|
+
"@statewalker/indexer-vector": "^0.1.4"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
41
|
"@electric-sql/pglite-pgvector": "^0.0.9",
|
package/src/dialect.ts
CHANGED
|
@@ -101,8 +101,7 @@ export const pgliteFtsDialect: SqlFtsDialect = {
|
|
|
101
101
|
|
|
102
102
|
/**
|
|
103
103
|
* PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.
|
|
104
|
-
* Embeddings are
|
|
105
|
-
* driver-native arrays.
|
|
104
|
+
* Embeddings are bound as stringified array literals.
|
|
106
105
|
*/
|
|
107
106
|
export const pgliteVectorDialect: SqlVectorDialect = {
|
|
108
107
|
createTableDdl({ tableName, indexName, info }) {
|