@statewalker/indexer-pglite 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/dialect.js +25 -3
- package/dist/dialect.js.map +1 -1
- package/package.json +4 -4
- package/src/dialect.ts +30 -1
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
## What it is
|
|
4
4
|
|
|
5
|
-
An `Indexer` stored in [PGlite](https://github.com/electric-sql/pglite) (PostgreSQL compiled to WASM). Full-text sub-indexes use PostgreSQL `tsvector` columns with a GIN index and `ts_rank_cd` scoring; vector sub-indexes use pgvector with an HNSW cosine index. It runs in the browser, in Node and in workers.
|
|
5
|
+
An `Indexer` stored in [PGlite](https://github.com/electric-sql/pglite) (PostgreSQL compiled to WASM). Full-text sub-indexes use PostgreSQL `tsvector` columns with a GIN index and `ts_rank_cd` scoring; vector sub-indexes use pgvector with an HNSW cosine index; n-gram (grep) sub-indexes (`{ type: "ngram" }`, `@statewalker/indexer-ngram`) keep each block's case-folded trigrams in a `text[]` column under a GIN index (store `gin`, core Postgres, no extension to load) and verify candidates in JavaScript. It runs in the browser, in Node and in workers.
|
|
6
6
|
|
|
7
7
|
## Why it exists
|
|
8
8
|
|
package/dist/dialect.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ISO_639_1_LANGUAGE_NAMES, buildSearchFilterSql, embeddingToLiteral, sqlDocsTableName } from "@statewalker/indexer-core";
|
|
1
|
+
import { ISO_639_1_LANGUAGE_NAMES, buildSearchFilterSql, embeddingToLiteral, ngramTrigrams, sqlDocsTableName } from "@statewalker/indexer-core";
|
|
2
2
|
//#region src/dialect.ts
|
|
3
3
|
const LANGUAGE_MAP = {
|
|
4
4
|
...ISO_639_1_LANGUAGE_NAMES,
|
|
@@ -60,6 +60,27 @@ const pgliteVectorDialect = {
|
|
|
60
60
|
}
|
|
61
61
|
};
|
|
62
62
|
/** Aggregate PGlite dialect for `createSqlBackedIndexer`. */
|
|
63
|
+
/**
|
|
64
|
+
* PGlite n-gram store (`gin`): each row keeps `grams`, the distinct trigrams of its case-folded
|
|
65
|
+
* content, under a GIN index (core Postgres, no extension). A branch's candidates are the rows
|
|
66
|
+
* whose `grams` contain every trigram of its literals.
|
|
67
|
+
*/
|
|
68
|
+
const pgliteNgramDialect = {
|
|
69
|
+
store: "gin",
|
|
70
|
+
createTableDdl({ tableName }) {
|
|
71
|
+
return [`CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, metadata TEXT, grams TEXT[] NOT NULL, PRIMARY KEY (doc_id, block_id))`, `CREATE INDEX IF NOT EXISTS ${tableName}_grams_idx ON ${tableName} USING GIN (grams)`];
|
|
72
|
+
},
|
|
73
|
+
storeColumns(content) {
|
|
74
|
+
return { grams: ngramTrigrams(content) };
|
|
75
|
+
},
|
|
76
|
+
branchSql(literals, startParam) {
|
|
77
|
+
const grams = [...new Set(literals.flatMap((literal) => ngramTrigrams(literal)))];
|
|
78
|
+
return {
|
|
79
|
+
sql: `b.grams @> $${startParam}::text[]`,
|
|
80
|
+
params: [grams]
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
};
|
|
63
84
|
const pgliteDialect = {
|
|
64
85
|
extensionInit: ["CREATE EXTENSION IF NOT EXISTS vector"],
|
|
65
86
|
docsTableDdl(prefix) {
|
|
@@ -68,9 +89,10 @@ const pgliteDialect = {
|
|
|
68
89
|
unionAliasSuffix: " AS combined",
|
|
69
90
|
supportsDDLInTransaction: true,
|
|
70
91
|
fts: pgliteFtsDialect,
|
|
71
|
-
vec: pgliteVectorDialect
|
|
92
|
+
vec: pgliteVectorDialect,
|
|
93
|
+
ngram: pgliteNgramDialect
|
|
72
94
|
};
|
|
73
95
|
//#endregion
|
|
74
|
-
export { pgliteDialect, pgliteFtsDialect, pgliteVectorDialect };
|
|
96
|
+
export { pgliteDialect, pgliteFtsDialect, pgliteNgramDialect, pgliteVectorDialect };
|
|
75
97
|
|
|
76
98
|
//# sourceMappingURL=dialect.js.map
|
package/dist/dialect.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dialect.js","names":[],"sources":["../src/dialect.ts"],"sourcesContent":["import type {
|
|
1
|
+
{"version":3,"file":"dialect.js","names":[],"sources":["../src/dialect.ts"],"sourcesContent":["import type {\n SqlBackedDialect,\n SqlFtsDialect,\n SqlNgramDialect,\n SqlVectorDialect,\n} from \"@statewalker/indexer-core\";\nimport {\n buildSearchFilterSql,\n embeddingToLiteral,\n ISO_639_1_LANGUAGE_NAMES,\n ngramTrigrams,\n sqlDocsTableName,\n} from \"@statewalker/indexer-core\";\n\n// ISO-639-1 codes map to the Postgres text-search configuration of the same language name.\nconst LANGUAGE_MAP: Record<string, string> = {\n ...ISO_639_1_LANGUAGE_NAMES,\n simple: \"simple\",\n};\n\n// Postgres text-search configuration names that have always-on availability;\n// any value outside this set is interpolated into DDL/queries and must be a\n// whitelisted identifier — never user-controlled free-form text.\nconst KNOWN_PG_CONFIGS: ReadonlySet<string> = new Set(Object.values(LANGUAGE_MAP));\n\nfunction resolvePgLanguage(lang: string): string {\n const mapped = LANGUAGE_MAP[lang];\n if (mapped) return mapped;\n if (KNOWN_PG_CONFIGS.has(lang)) return lang;\n throw new Error(\n `pglite FTS: unsupported language \"${lang}\" (must be an ISO-639-1 code or a Postgres text-search config name)`,\n );\n}\n\n/**\n * PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring,\n * and `to_tsquery` with a language-specific stemmer.\n */\nexport const pgliteFtsDialect: SqlFtsDialect = {\n createTableDdl({ tableName, info }) {\n const pgLang = resolvePgLanguage(info.language);\n return [\n `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('${pgLang}', content)) STORED, metadata TEXT, PRIMARY KEY (doc_id, block_id))`,\n `CREATE INDEX IF NOT EXISTS ${tableName}_tsv_idx ON ${tableName} USING GIN (content_tsv)`,\n ];\n },\n\n async search({ db, tableName, docsTable, info, query, paths, topK }) {\n const pgLang = resolvePgLanguage(info.language);\n // Keep only words: split on everything that is not a Unicode letter, combining mark, digit\n // or `_`, so no tsquery operator (`& | ! ( ) : < > <-> <N>`, quotes, backslash) reaches\n // `to_tsquery`, and non-ASCII words (Russian, Hindi, decomposed accents, …) stay whole.\n const validWords = query\n .toLowerCase()\n .split(/[^\\p{L}\\p{M}\\p{N}_]+/u)\n .filter((w) => w.length > 0);\n if (validWords.length === 0) return [];\n\n const orTerms = validWords.join(\" | \");\n\n const {\n pathCondition,\n topKParam,\n params: allParams,\n } = buildSearchFilterSql(\"d.path\", paths, topK, [orTerms]);\n const pathClause = pathCondition ? ` AND ${pathCondition}` : \"\";\n\n const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`;\n\n const rows = await db.query<{\n path: string;\n block_id: string;\n content: string;\n score: number;\n }>(sql, allParams);\n\n return rows.map((row) => ({\n path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n blockId: row.block_id,\n content: row.content,\n score: row.score,\n }));\n },\n};\n\n/**\n * PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.\n * Embeddings are bound as stringified array literals.\n */\nexport const pgliteVectorDialect: SqlVectorDialect = {\n createTableDdl({ tableName, indexName, info }) {\n const dim = info.dimensionality;\n return [\n `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding vector(${dim}) NOT NULL, PRIMARY KEY (doc_id, block_id))`,\n `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING hnsw (embedding vector_cosine_ops)`,\n ];\n },\n\n // pgvector's `vector(dim)` column accepts either a JSON-array text literal (`\"[1,2,3]\"`) cast with\n // `::vector(dim)`, or a JS array bound by the driver. The text-literal path is used because it\n // works uniformly across the pglite vector extension versions available in the catalog; swapping\n // to native array binding is a follow-up when driver support is confirmed.\n bindEmbedding: embeddingToLiteral,\n embeddingCastSuffix: (dim) => `::vector(${dim})`,\n\n async search({\n db,\n tableName,\n docsTable,\n queryEmbedding,\n paths,\n topK,\n info,\n bindEmbedding,\n embeddingCastSuffix,\n }) {\n const vecLiteral = bindEmbedding(queryEmbedding);\n const dim = info.dimensionality;\n\n const {\n pathCondition,\n topKParam,\n params: allParams,\n } = buildSearchFilterSql(\"d.path\", paths, topK, [vecLiteral]);\n const pathClause = pathCondition ? `WHERE ${pathCondition} ` : \"\";\n\n const rows = await db.query<{ path: string; block_id: string; dist: number }>(\n `SELECT d.path, b.block_id, (b.embedding <=> $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`,\n allParams,\n );\n\n return rows.map((row) => ({\n path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n blockId: row.block_id,\n score: 1 - row.dist,\n }));\n },\n\n decodeEmbedding(raw) {\n // PGlite returns vector as a string \"[1,2,3]\".\n return new Float32Array(JSON.parse(raw as string) as number[]);\n },\n};\n\n/** Aggregate PGlite dialect for `createSqlBackedIndexer`. */\n/**\n * PGlite n-gram store (`gin`): each row keeps `grams`, the distinct trigrams of its case-folded\n * content, under a GIN index (core Postgres, no extension). A branch's candidates are the rows\n * whose `grams` contain every trigram of its literals.\n */\nexport const pgliteNgramDialect: SqlNgramDialect = {\n store: \"gin\",\n createTableDdl({ tableName }) {\n return [\n `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, metadata TEXT, grams TEXT[] NOT NULL, PRIMARY KEY (doc_id, block_id))`,\n `CREATE INDEX IF NOT EXISTS ${tableName}_grams_idx ON ${tableName} USING GIN (grams)`,\n ];\n },\n storeColumns(content) {\n return { grams: ngramTrigrams(content) };\n },\n branchSql(literals, startParam) {\n const grams = [...new Set(literals.flatMap((literal) => ngramTrigrams(literal)))];\n return { sql: `b.grams @> $${startParam}::text[]`, params: [grams] };\n },\n};\n\nexport const pgliteDialect: SqlBackedDialect = {\n extensionInit: [\"CREATE EXTENSION IF NOT EXISTS vector\"],\n docsTableDdl(prefix) {\n return [\n `CREATE TABLE IF NOT EXISTS ${sqlDocsTableName(prefix)} (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`,\n ];\n },\n unionAliasSuffix: \" AS combined\",\n supportsDDLInTransaction: true,\n fts: pgliteFtsDialect,\n vec: pgliteVectorDialect,\n ngram: pgliteNgramDialect,\n};\n"],"mappings":";;AAeA,MAAM,eAAuC;CAC3C,GAAG;CACH,QAAQ;AACV;AAKA,MAAM,mBAAwC,IAAI,IAAI,OAAO,OAAO,YAAY,CAAC;AAEjF,SAAS,kBAAkB,MAAsB;CAC/C,MAAM,SAAS,aAAa;CAC5B,IAAI,QAAQ,OAAO;CACnB,IAAI,iBAAiB,IAAI,IAAI,GAAG,OAAO;CACvC,MAAM,IAAI,MACR,qCAAqC,KAAK,oEAC5C;AACF;;;;;AAMA,MAAa,mBAAkC;CAC7C,eAAe,EAAE,WAAW,QAAQ;EAElC,OAAO,CACL,8BAA8B,UAAU,mIAF3B,kBAAkB,KAAK,QAE4I,EAAE,sEAClL,8BAA8B,UAAU,cAAc,UAAU,yBAClE;CACF;CAEA,MAAM,OAAO,EAAE,IAAI,WAAW,WAAW,MAAM,OAAO,OAAO,QAAQ;EACnE,MAAM,SAAS,kBAAkB,KAAK,QAAQ;EAI9C,MAAM,aAAa,MAChB,YAAY,CAAC,CACb,MAAM,uBAAuB,CAAC,CAC9B,QAAQ,MAAM,EAAE,SAAS,CAAC;EAC7B,IAAI,WAAW,WAAW,GAAG,OAAO,CAAC;EAErC,MAAM,UAAU,WAAW,KAAK,KAAK;EAErC,MAAM,EACJ,eACA,WACA,QAAQ,cACN,qBAAqB,UAAU,OAAO,MAAM,CAAC,OAAO,CAAC;EAGzD,MAAM,MAAM,+EAA+E,OAAO,wBAAwB,UAAU,UAAU,UAAU,+DAA+D,OAAO,QAF3M,gBAAgB,QAAQ,kBAAkB,GAEoL,6BAA6B;EAS9Q,QAAO,MAPY,GAAG,MAKnB,KAAK,SAAS,EAAA,CAEL,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,SAAS,IAAI;GACb,OAAO,IAAI;EACb,EAAE;CACJ;AACF;;;;;AAMA,MAAa,sBAAwC;CACnD,eAAe,EAAE,WAAW,WAAW,QAAQ;EAE7C,OAAO,CACL,8BAA8B,UAAU,sEAF9B,KAAK,eAEmG,8CAClH,8BAA8B,UAAU,MAAM,UAAU,0CAC1D;CACF;CAMA,eAAe;CACf,sBAAsB,QAAQ,YAAY,IAAI;CAE9C,MAAM,OAAO,EACX,IACA,WACA,WACA,gBACA,OACA,MACA,MACA,eACA,uBACC;EACD,MAAM,aAAa,cAAc,cAAc;EAC/C,MAAM,MAAM,KAAK;EAEjB,MAAM,EACJ,eACA,WACA,QAAQ,cACN,qBAAqB,UAAU,OAAO,MAAM,CAAC,UAAU,CAAC;EAC5D,MAAM,aAAa,gBAAgB,SAAS,cAAc,KAAK;EAO/D,QAAO,MALY,GAAG,MACpB,iDAAiD,oBAAoB,GAAG,EAAE,iBAAiB,UAAU,UAAU,UAAU,4BAA4B,WAAW,0BAA0B,aAC1L,SACF,EAAA,CAEY,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,OAAO,IAAI,IAAI;EACjB,EAAE;CACJ;CAEA,gBAAgB,KAAK;EAEnB,OAAO,IAAI,aAAa,KAAK,MAAM,GAAa,CAAa;CAC/D;AACF;;;;;;;AAQA,MAAa,qBAAsC;CACjD,OAAO;CACP,eAAe,EAAE,aAAa;EAC5B,OAAO,CACL,8BAA8B,UAAU,kJACxC,8BAA8B,UAAU,gBAAgB,UAAU,mBACpE;CACF;CACA,aAAa,SAAS;EACpB,OAAO,EAAE,OAAO,cAAc,OAAO,EAAE;CACzC;CACA,UAAU,UAAU,YAAY;EAC9B,MAAM,QAAQ,CAAC,GAAG,IAAI,IAAI,SAAS,SAAS,YAAY,cAAc,OAAO,CAAC,CAAC,CAAC;EAChF,OAAO;GAAE,KAAK,eAAe,WAAW;GAAW,QAAQ,CAAC,KAAK;EAAE;CACrE;AACF;AAEA,MAAa,gBAAkC;CAC7C,eAAe,CAAC,uCAAuC;CACvD,aAAa,QAAQ;EACnB,OAAO,CACL,8BAA8B,iBAAiB,MAAM,EAAE,wDACzD;CACF;CACA,kBAAkB;CAClB,0BAA0B;CAC1B,KAAK;CACL,KAAK;CACL,OAAO;AACT"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/indexer-pglite",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "PGlite-backed indexer implementation: full-text + vector search over a @statewalker/db-api Db.",
|
|
@@ -32,10 +32,10 @@
|
|
|
32
32
|
],
|
|
33
33
|
"dependencies": {
|
|
34
34
|
"@statewalker/db-api": "^0.1.0",
|
|
35
|
-
"@statewalker/indexer-
|
|
35
|
+
"@statewalker/indexer-api": "^0.1.5",
|
|
36
36
|
"@statewalker/indexer-vector": "^0.1.6",
|
|
37
|
-
"@statewalker/indexer-
|
|
38
|
-
"@statewalker/indexer-
|
|
37
|
+
"@statewalker/indexer-core": "^0.3.0",
|
|
38
|
+
"@statewalker/indexer-fulltext": "^0.1.6"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
41
|
"@electric-sql/pglite": "^0.5.15",
|
package/src/dialect.ts
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type {
|
|
2
|
+
SqlBackedDialect,
|
|
3
|
+
SqlFtsDialect,
|
|
4
|
+
SqlNgramDialect,
|
|
5
|
+
SqlVectorDialect,
|
|
6
|
+
} from "@statewalker/indexer-core";
|
|
2
7
|
import {
|
|
3
8
|
buildSearchFilterSql,
|
|
4
9
|
embeddingToLiteral,
|
|
5
10
|
ISO_639_1_LANGUAGE_NAMES,
|
|
11
|
+
ngramTrigrams,
|
|
6
12
|
sqlDocsTableName,
|
|
7
13
|
} from "@statewalker/indexer-core";
|
|
8
14
|
|
|
@@ -137,6 +143,28 @@ export const pgliteVectorDialect: SqlVectorDialect = {
|
|
|
137
143
|
};
|
|
138
144
|
|
|
139
145
|
/** Aggregate PGlite dialect for `createSqlBackedIndexer`. */
|
|
146
|
+
/**
|
|
147
|
+
* PGlite n-gram store (`gin`): each row keeps `grams`, the distinct trigrams of its case-folded
|
|
148
|
+
* content, under a GIN index (core Postgres, no extension). A branch's candidates are the rows
|
|
149
|
+
* whose `grams` contain every trigram of its literals.
|
|
150
|
+
*/
|
|
151
|
+
export const pgliteNgramDialect: SqlNgramDialect = {
|
|
152
|
+
store: "gin",
|
|
153
|
+
createTableDdl({ tableName }) {
|
|
154
|
+
return [
|
|
155
|
+
`CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, metadata TEXT, grams TEXT[] NOT NULL, PRIMARY KEY (doc_id, block_id))`,
|
|
156
|
+
`CREATE INDEX IF NOT EXISTS ${tableName}_grams_idx ON ${tableName} USING GIN (grams)`,
|
|
157
|
+
];
|
|
158
|
+
},
|
|
159
|
+
storeColumns(content) {
|
|
160
|
+
return { grams: ngramTrigrams(content) };
|
|
161
|
+
},
|
|
162
|
+
branchSql(literals, startParam) {
|
|
163
|
+
const grams = [...new Set(literals.flatMap((literal) => ngramTrigrams(literal)))];
|
|
164
|
+
return { sql: `b.grams @> $${startParam}::text[]`, params: [grams] };
|
|
165
|
+
},
|
|
166
|
+
};
|
|
167
|
+
|
|
140
168
|
export const pgliteDialect: SqlBackedDialect = {
|
|
141
169
|
extensionInit: ["CREATE EXTENSION IF NOT EXISTS vector"],
|
|
142
170
|
docsTableDdl(prefix) {
|
|
@@ -148,4 +176,5 @@ export const pgliteDialect: SqlBackedDialect = {
|
|
|
148
176
|
supportsDDLInTransaction: true,
|
|
149
177
|
fts: pgliteFtsDialect,
|
|
150
178
|
vec: pgliteVectorDialect,
|
|
179
|
+
ngram: pgliteNgramDialect,
|
|
151
180
|
};
|