@statewalker/indexer-pglite 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -29
- package/dist/dialect.js +1 -11
- package/dist/dialect.js.map +1 -1
- package/dist/pglite-indexer.d.ts +7 -6
- package/dist/pglite-indexer.d.ts.map +1 -1
- package/dist/pglite-indexer.js +7 -14
- package/dist/pglite-indexer.js.map +1 -1
- package/package.json +15 -10
- package/src/dialect.ts +1 -18
- package/src/pglite-indexer.ts +9 -20
package/README.md
CHANGED
|
@@ -1,53 +1,99 @@
|
|
|
1
1
|
# @statewalker/indexer-pglite
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
## What it is
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
An `Indexer` stored in [PGlite](https://github.com/electric-sql/pglite) (PostgreSQL compiled to WASM). Full-text sub-indexes use PostgreSQL `tsvector` columns with a GIN index and `ts_rank_cd` scoring; vector sub-indexes use pgvector with an HNSW cosine index. It runs in the browser, in Node and in workers.
|
|
6
|
+
|
|
7
|
+
## Why it exists
|
|
8
|
+
|
|
9
|
+
It gives real PostgreSQL text search (stemming per language) and an approximate nearest-neighbour vector index without a server. With an IndexedDB or file `path`, data persists without a separate save step.
|
|
10
|
+
|
|
11
|
+
## How to use
|
|
6
12
|
|
|
7
13
|
```sh
|
|
8
|
-
pnpm add @statewalker/indexer-pglite @electric-sql/pglite
|
|
14
|
+
pnpm add @statewalker/indexer-pglite @statewalker/db-pglite @electric-sql/pglite-pgvector
|
|
9
15
|
```
|
|
10
16
|
|
|
11
|
-
|
|
17
|
+
The indexer runs over a `Db` from `@statewalker/db-pglite`. Open it with the pgvector extension — the indexer issues `CREATE EXTENSION IF NOT EXISTS vector` and fails with `extension "vector" is not available` otherwise. The indexer never closes the `Db`; close it yourself.
|
|
18
|
+
|
|
19
|
+
The `Db` must come from `newPgliteDb` (PostgreSQL dialect). A SQLite `Db` typechecks, because both are `Db`, but fails at the first `CREATE EXTENSION` with `near "EXTENSION": syntax error`.
|
|
20
|
+
|
|
21
|
+
- `createPGLiteIndexer({ db }): Promise<Indexer>` — `db: Db`.
|
|
22
|
+
- `PGLiteIndexerOptions` — the options type.
|
|
23
|
+
|
|
24
|
+
## Examples
|
|
25
|
+
|
|
26
|
+
In-memory database:
|
|
12
27
|
|
|
13
28
|
```ts
|
|
29
|
+
import { vector } from "@electric-sql/pglite-pgvector";
|
|
30
|
+
import { newPgliteDb } from "@statewalker/db-pglite";
|
|
14
31
|
import { createPGLiteIndexer } from "@statewalker/indexer-pglite";
|
|
32
|
+
import { newFullTextAccess, setFullTextConfig } from "@statewalker/indexer-fulltext";
|
|
33
|
+
|
|
34
|
+
const db = await newPgliteDb({ extensions: { vector } });
|
|
35
|
+
const indexer = await createPGLiteIndexer({ db });
|
|
15
36
|
|
|
16
|
-
|
|
17
|
-
|
|
37
|
+
const params = { name: "docs" };
|
|
38
|
+
setFullTextConfig(params, "q", { language: "en" });
|
|
39
|
+
const index = await indexer.createIndex(params);
|
|
18
40
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
// import { vector } from "@electric-sql/pglite/vector";
|
|
22
|
-
// const db = await PGlite.create({ dataDir: "idb://myapp", extensions: { vector } });
|
|
23
|
-
// const indexer = await createPGLiteIndexer({ db });
|
|
41
|
+
const fts = newFullTextAccess("q");
|
|
42
|
+
await fts.get(index).addDocument([{ path: "/docs/a", blockId: "1", content: "running dogs" }]);
|
|
24
43
|
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
}
|
|
44
|
+
const request = { topK: 10 };
|
|
45
|
+
fts.setQuery(request, { queries: ["run"] }); // stemmed: matches "running"
|
|
46
|
+
for await (const r of index.search(request)) {
|
|
47
|
+
console.log(r.path, r.blockId, fts.getResult(r)?.score); // ts_rank_cd
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
await indexer.close();
|
|
51
|
+
await db.close();
|
|
30
52
|
```
|
|
31
53
|
|
|
32
|
-
|
|
54
|
+
Persistent database:
|
|
33
55
|
|
|
34
|
-
|
|
56
|
+
```ts
|
|
57
|
+
import { vector } from "@electric-sql/pglite-pgvector";
|
|
58
|
+
import { newPgliteDb } from "@statewalker/db-pglite";
|
|
59
|
+
import { createPGLiteIndexer } from "@statewalker/indexer-pglite";
|
|
60
|
+
|
|
61
|
+
// a directory on Node.js ("./pgdata"), IndexedDB in the browser ("idb://my-app")
|
|
62
|
+
const db = await newPgliteDb({ path: "idb://my-app", extensions: { vector } });
|
|
63
|
+
const indexer = await createPGLiteIndexer({ db });
|
|
64
|
+
// ...
|
|
65
|
+
await indexer.close(); // does not close db
|
|
66
|
+
await db.close();
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Internals
|
|
70
|
+
|
|
71
|
+
### The vector extension must be loaded
|
|
35
72
|
|
|
36
|
-
|
|
73
|
+
At startup the indexer runs `CREATE EXTENSION IF NOT EXISTS vector`. The `Db` you pass in must be opened with `extensions: { vector }`, otherwise this statement fails with `extension "vector" is not available` and the indexer is not created.
|
|
74
|
+
|
|
75
|
+
### Tables
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
__indexer_manifest(name, config)
|
|
79
|
+
idx_<index>_docs(doc_id SERIAL, path UNIQUE) path -> doc_id
|
|
80
|
+
idx_<index>_<sub>_fts(doc_id, block_id, content, content_tsv TSVECTOR GENERATED, metadata) + GIN
|
|
81
|
+
idx_<index>_<sub>_vec(doc_id, block_id, embedding vector(dim)) + HNSW vector_cosine_ops
|
|
82
|
+
```
|
|
37
83
|
|
|
38
|
-
|
|
84
|
+
`content_tsv` is a generated column, so the full-text index is always current; there is no rebuild step.
|
|
39
85
|
|
|
40
|
-
|
|
41
|
-
- `idx_<prefix>_fts(doc_id, block_id, content, content_tsv TSVECTOR GENERATED ..., metadata)` + `USING GIN (content_tsv)`.
|
|
42
|
-
- `idx_<prefix>_vec(doc_id, block_id, embedding vector(dim))` + `USING hnsw (embedding vector_cosine_ops)`.
|
|
86
|
+
### Queries, scores and languages
|
|
43
87
|
|
|
44
|
-
|
|
88
|
+
- A full-text query is lowercased, split on whitespace, stripped of tsquery operators (`& | ! ( ) : ' " \`) and joined with `|` (OR). Score is `ts_rank_cd`. With several `queries` in one sub-query, each block keeps its best score across them (scores are not summed).
|
|
89
|
+
- Vector `score` is `1 - (embedding <=> query)` (cosine). Embeddings are sent as text literals cast to `vector(dim)`.
|
|
90
|
+
- `language` accepts ISO-639-1 codes (`en`, `fr`, `de`, `es`, `it`, `pt`, `nl`, `ru`, `sv`, `no`, `da`, `fi`, `hu`, `ro`, `tr`), `simple`, or the matching Postgres config names (`english`, ...). Anything else throws `pglite FTS: unsupported language "<x>" (must be an ISO-639-1 code or a Postgres text-search config name)`. The value is put into SQL, which is why it is checked against this list.
|
|
91
|
+
- Default retrieval depth is `topK = 100` per sub-query.
|
|
45
92
|
|
|
46
|
-
|
|
93
|
+
### Dependencies
|
|
47
94
|
|
|
48
|
-
-
|
|
49
|
-
- `PGLiteIndexerOptions` — option type.
|
|
95
|
+
`@statewalker/db-api` (the `Db` it runs on), `@statewalker/indexer-core` (`createSqlIndexer` and the SQL retrievers), `@statewalker/indexer-api`, `@statewalker/indexer-fulltext`, `@statewalker/indexer-vector`.
|
|
50
96
|
|
|
51
|
-
##
|
|
97
|
+
## License
|
|
52
98
|
|
|
53
|
-
|
|
99
|
+
MIT
|
package/dist/dialect.js
CHANGED
|
@@ -1,15 +1,5 @@
|
|
|
1
1
|
import { buildPathPrefixesSql } from "@statewalker/indexer-core";
|
|
2
2
|
//#region src/dialect.ts
|
|
3
|
-
/** Adapt `@electric-sql/pglite`'s `PGlite` to `@statewalker/indexer-core`'s minimal `SqlDb`. */
|
|
4
|
-
function wrapDbAsSqlDb(db) {
|
|
5
|
-
return {
|
|
6
|
-
exec: (sql) => db.exec(sql).then(() => void 0),
|
|
7
|
-
async query(sql, params) {
|
|
8
|
-
const { rows } = await db.query(sql, params ?? []);
|
|
9
|
-
return rows;
|
|
10
|
-
}
|
|
11
|
-
};
|
|
12
|
-
}
|
|
13
3
|
const LANGUAGE_MAP = {
|
|
14
4
|
en: "english",
|
|
15
5
|
fr: "french",
|
|
@@ -119,6 +109,6 @@ const pgliteDialect = {
|
|
|
119
109
|
vec: pgliteVectorDialect
|
|
120
110
|
};
|
|
121
111
|
//#endregion
|
|
122
|
-
export { pgliteDialect, pgliteFtsDialect, pgliteVectorDialect
|
|
112
|
+
export { pgliteDialect, pgliteFtsDialect, pgliteVectorDialect };
|
|
123
113
|
|
|
124
114
|
//# sourceMappingURL=dialect.js.map
|
package/dist/dialect.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dialect.js","names":[],"sources":["../src/dialect.ts"],"sourcesContent":["import type {
|
|
1
|
+
{"version":3,"file":"dialect.js","names":[],"sources":["../src/dialect.ts"],"sourcesContent":["import type { SqlBackedDialect, SqlFtsDialect, SqlVectorDialect } from \"@statewalker/indexer-core\";\nimport { buildPathPrefixesSql } from \"@statewalker/indexer-core\";\n\nconst LANGUAGE_MAP: Record<string, string> = {\n en: \"english\",\n fr: \"french\",\n de: \"german\",\n es: \"spanish\",\n it: \"italian\",\n pt: \"portuguese\",\n nl: \"dutch\",\n ru: \"russian\",\n sv: \"swedish\",\n no: \"norwegian\",\n da: \"danish\",\n fi: \"finnish\",\n hu: \"hungarian\",\n ro: \"romanian\",\n tr: \"turkish\",\n simple: \"simple\",\n};\n\n// Postgres text-search configuration names that have always-on availability;\n// any value outside this set is interpolated into DDL/queries and must be a\n// whitelisted identifier — never user-controlled free-form text.\nconst KNOWN_PG_CONFIGS: ReadonlySet<string> = new Set(Object.values(LANGUAGE_MAP));\n\nfunction resolvePgLanguage(lang: string): string {\n const mapped = LANGUAGE_MAP[lang];\n if (mapped) return mapped;\n if (KNOWN_PG_CONFIGS.has(lang)) return lang;\n throw new Error(\n `pglite FTS: unsupported language \"${lang}\" (must be an ISO-639-1 code or a Postgres text-search config name)`,\n );\n}\n\n/**\n * pgvector's `vector(dim)` column accepts either a JSON-array text literal (`\"[1,2,3]\"`) cast with\n * `::vector(dim)`, or a JS array bound by the driver. The text-literal path is currently used\n * because it works uniformly across the pglite vector extension versions available in the catalog;\n * swapping to native array binding is a follow-up when driver support is confirmed.\n */\nfunction embeddingToLiteral(embedding: Float32Array): string {\n return `[${Array.from(embedding).join(\",\")}]`;\n}\n\n/**\n * PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring,\n * and `to_tsquery` with a language-specific stemmer.\n */\nexport const pgliteFtsDialect: SqlFtsDialect = {\n createTableDdl({ tableName, info }) {\n const pgLang = resolvePgLanguage(info.language);\n return [\n `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('${pgLang}', content)) STORED, metadata TEXT, PRIMARY KEY (doc_id, block_id))`,\n `CREATE INDEX IF NOT EXISTS ${tableName}_tsv_idx ON ${tableName} USING GIN (content_tsv)`,\n ];\n },\n\n async search({ db, tableName, docsTable, info, query, paths, topK }) {\n const pgLang = resolvePgLanguage(info.language);\n // Strip only tsquery-significant punctuation; keep Unicode letters/digits so\n // non-ASCII languages (Russian, French, CJK, …) reach the FTS engine intact.\n const validWords = query\n .toLowerCase()\n .split(/\\s+/)\n .map((w) => w.replace(/[&|!():'\"\\\\]/g, \"\"))\n .filter((w) => w.length > 0);\n if (validWords.length === 0) return [];\n\n const orTerms = validWords.join(\" | \");\n\n const allParams: unknown[] = [orTerms];\n let pathClause = \"\";\n if (paths && paths.length > 0) {\n const cond = buildPathPrefixesSql(\"d.path\", paths as string[], allParams.length + 1);\n pathClause = ` AND ${cond.sql}`;\n allParams.push(...cond.params);\n }\n\n const topKParam = `$${allParams.length + 1}`;\n allParams.push(topK);\n\n const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`;\n\n const rows = await db.query<{\n path: string;\n block_id: string;\n content: string;\n score: number;\n }>(sql, allParams);\n\n return rows.map((row) => ({\n path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n blockId: row.block_id,\n content: row.content,\n score: row.score,\n }));\n },\n};\n\n/**\n * PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.\n * Embeddings are currently bound as stringified array literals — Phase 9 switches to parameter-bound\n * driver-native arrays.\n */\nexport const pgliteVectorDialect: SqlVectorDialect = {\n createTableDdl({ tableName, indexName, info }) {\n const dim = info.dimensionality;\n return [\n `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding vector(${dim}) NOT NULL, PRIMARY KEY (doc_id, block_id))`,\n `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING hnsw (embedding vector_cosine_ops)`,\n ];\n },\n\n bindEmbedding: embeddingToLiteral,\n embeddingCastSuffix: (dim) => `::vector(${dim})`,\n\n async search({\n db,\n tableName,\n docsTable,\n queryEmbedding,\n paths,\n topK,\n info,\n bindEmbedding,\n embeddingCastSuffix,\n }) {\n const vecLiteral = bindEmbedding(queryEmbedding);\n const dim = info.dimensionality;\n\n const allParams: unknown[] = [vecLiteral];\n let pathClause = \"\";\n if (paths && paths.length > 0) {\n const cond = buildPathPrefixesSql(\"d.path\", paths as string[], allParams.length + 1);\n pathClause = `WHERE ${cond.sql} `;\n allParams.push(...cond.params);\n }\n\n const topKParam = `$${allParams.length + 1}`;\n allParams.push(topK);\n\n const rows = await db.query<{ path: string; block_id: string; dist: number }>(\n `SELECT d.path, b.block_id, (b.embedding <=> $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`,\n allParams,\n );\n\n return rows.map((row) => ({\n path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n blockId: row.block_id,\n score: 1 - row.dist,\n }));\n },\n\n decodeEmbedding(raw) {\n // PGlite returns vector as a string \"[1,2,3]\".\n return new Float32Array(JSON.parse(raw as string) as number[]);\n },\n};\n\n/** Aggregate PGlite dialect for `createSqlBackedIndexer`. */\nexport const pgliteDialect: SqlBackedDialect = {\n extensionInit: [\"CREATE EXTENSION IF NOT EXISTS vector\"],\n docsTableDdl(prefix) {\n return [\n `CREATE TABLE IF NOT EXISTS idx_${prefix}_docs (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`,\n ];\n },\n unionAliasSuffix: \" AS combined\",\n supportsDDLInTransaction: true,\n fts: pgliteFtsDialect,\n vec: pgliteVectorDialect,\n};\n"],"mappings":";;AAGA,MAAM,eAAuC;CAC3C,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,QAAQ;AACV;AAKA,MAAM,mBAAwC,IAAI,IAAI,OAAO,OAAO,YAAY,CAAC;AAEjF,SAAS,kBAAkB,MAAsB;CAC/C,MAAM,SAAS,aAAa;CAC5B,IAAI,QAAQ,OAAO;CACnB,IAAI,iBAAiB,IAAI,IAAI,GAAG,OAAO;CACvC,MAAM,IAAI,MACR,qCAAqC,KAAK,oEAC5C;AACF;;;;;;;AAQA,SAAS,mBAAmB,WAAiC;CAC3D,OAAO,IAAI,MAAM,KAAK,SAAS,CAAC,CAAC,KAAK,GAAG,EAAE;AAC7C;;;;;AAMA,MAAa,mBAAkC;CAC7C,eAAe,EAAE,WAAW,QAAQ;EAElC,OAAO,CACL,8BAA8B,UAAU,mIAF3B,kBAAkB,KAAK,QAE4I,EAAE,sEAClL,8BAA8B,UAAU,cAAc,UAAU,yBAClE;CACF;CAEA,MAAM,OAAO,EAAE,IAAI,WAAW,WAAW,MAAM,OAAO,OAAO,QAAQ;EACnE,MAAM,SAAS,kBAAkB,KAAK,QAAQ;EAG9C,MAAM,aAAa,MAChB,YAAY,CAAC,CACb,MAAM,KAAK,CAAC,CACZ,KAAK,MAAM,EAAE,QAAQ,iBAAiB,EAAE,CAAC,CAAC,CAC1C,QAAQ,MAAM,EAAE,SAAS,CAAC;EAC7B,IAAI,WAAW,WAAW,GAAG,OAAO,CAAC;EAIrC,MAAM,YAAuB,CAFb,WAAW,KAAK,KAEI,CAAC;EACrC,IAAI,aAAa;EACjB,IAAI,SAAS,MAAM,SAAS,GAAG;GAC7B,MAAM,OAAO,qBAAqB,UAAU,OAAmB,UAAU,SAAS,CAAC;GACnF,aAAa,QAAQ,KAAK;GAC1B,UAAU,KAAK,GAAG,KAAK,MAAM;EAC/B;EAEA,MAAM,YAAY,IAAI,UAAU,SAAS;EACzC,UAAU,KAAK,IAAI;EAEnB,MAAM,MAAM,+EAA+E,OAAO,wBAAwB,UAAU,UAAU,UAAU,+DAA+D,OAAO,QAAQ,WAAW,6BAA6B;EAS9Q,QAAO,MAPY,GAAG,MAKnB,KAAK,SAAS,EAAA,CAEL,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,SAAS,IAAI;GACb,OAAO,IAAI;EACb,EAAE;CACJ;AACF;;;;;;AAOA,MAAa,sBAAwC;CACnD,eAAe,EAAE,WAAW,WAAW,QAAQ;EAE7C,OAAO,CACL,8BAA8B,UAAU,sEAF9B,KAAK,eAEmG,8CAClH,8BAA8B,UAAU,MAAM,UAAU,0CAC1D;CACF;CAEA,eAAe;CACf,sBAAsB,QAAQ,YAAY,IAAI;CAE9C,MAAM,OAAO,EACX,IACA,WACA,WACA,gBACA,OACA,MACA,MACA,eACA,uBACC;EACD,MAAM,aAAa,cAAc,cAAc;EAC/C,MAAM,MAAM,KAAK;EAEjB,MAAM,YAAuB,CAAC,UAAU;EACxC,IAAI,aAAa;EACjB,IAAI,SAAS,MAAM,SAAS,GAAG;GAC7B,MAAM,OAAO,qBAAqB,UAAU,OAAmB,UAAU,SAAS,CAAC;GACnF,aAAa,SAAS,KAAK,IAAI;GAC/B,UAAU,KAAK,GAAG,KAAK,MAAM;EAC/B;EAEA,MAAM,YAAY,IAAI,UAAU,SAAS;EACzC,UAAU,KAAK,IAAI;EAOnB,QAAO,MALY,GAAG,MACpB,iDAAiD,oBAAoB,GAAG,EAAE,iBAAiB,UAAU,UAAU,UAAU,4BAA4B,WAAW,0BAA0B,aAC1L,SACF,EAAA,CAEY,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,OAAO,IAAI,IAAI;EACjB,EAAE;CACJ;CAEA,gBAAgB,KAAK;EAEnB,OAAO,IAAI,aAAa,KAAK,MAAM,GAAa,CAAa;CAC/D;AACF;;AAGA,MAAa,gBAAkC;CAC7C,eAAe,CAAC,uCAAuC;CACvD,aAAa,QAAQ;EACnB,OAAO,CACL,kCAAkC,OAAO,6DAC3C;CACF;CACA,kBAAkB;CAClB,0BAA0B;CAC1B,KAAK;CACL,KAAK;AACP"}
|
package/dist/pglite-indexer.d.ts
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Db } from "@statewalker/db-api";
|
|
2
2
|
import { Indexer } from "@statewalker/indexer-api";
|
|
3
3
|
//#region src/pglite-indexer.d.ts
|
|
4
4
|
export interface PGLiteIndexerOptions {
|
|
5
|
-
|
|
5
|
+
/** A PGlite-backed `Db` with the pgvector extension loaded. The indexer never closes it. */
|
|
6
|
+
db: Db;
|
|
6
7
|
}
|
|
7
8
|
/**
|
|
8
9
|
* PGlite-backed `Indexer` over the open-registry contract.
|
|
9
10
|
*
|
|
10
11
|
* Wires the per-Index docs table, per-sub-index FTS / vector tables, and the
|
|
11
|
-
* `__indexer_manifest` table via `createSqlIndexer`.
|
|
12
|
-
*
|
|
13
|
-
* with
|
|
12
|
+
* `__indexer_manifest` table via `createSqlIndexer`. The FTS modality uses
|
|
13
|
+
* PostgreSQL `tsvector` with GIN; the vector modality uses pgvector's
|
|
14
|
+
* `vector(dim)` with HNSW, so `db` must be opened with the vector extension.
|
|
14
15
|
*/
|
|
15
|
-
export declare function createPGLiteIndexer(options
|
|
16
|
+
export declare function createPGLiteIndexer(options: PGLiteIndexerOptions): Promise<Indexer>;
|
|
16
17
|
//#endregion
|
|
17
18
|
//# sourceMappingURL=pglite-indexer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pglite-indexer.d.ts","names":[],"sources":["../src/pglite-indexer.ts"],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"pglite-indexer.d.ts","names":[],"sources":["../src/pglite-indexer.ts"],"mappings":";;;iBAKiB;;EAEf,IAAI;;;;;;;;;;wBAWU,oBAAoB,SAAS,uBAAuB,QAAQ"}
|
package/dist/pglite-indexer.js
CHANGED
|
@@ -1,25 +1,18 @@
|
|
|
1
|
-
import { pgliteDialect
|
|
2
|
-
import { PGlite } from "@electric-sql/pglite";
|
|
3
|
-
import { vector } from "@electric-sql/pglite-pgvector";
|
|
1
|
+
import { pgliteDialect } from "./dialect.js";
|
|
4
2
|
import { createSqlIndexer } from "@statewalker/indexer-core";
|
|
5
3
|
//#region src/pglite-indexer.ts
|
|
6
4
|
/**
|
|
7
5
|
* PGlite-backed `Indexer` over the open-registry contract.
|
|
8
6
|
*
|
|
9
7
|
* Wires the per-Index docs table, per-sub-index FTS / vector tables, and the
|
|
10
|
-
* `__indexer_manifest` table via `createSqlIndexer`.
|
|
11
|
-
*
|
|
12
|
-
* with
|
|
8
|
+
* `__indexer_manifest` table via `createSqlIndexer`. The FTS modality uses
|
|
9
|
+
* PostgreSQL `tsvector` with GIN; the vector modality uses pgvector's
|
|
10
|
+
* `vector(dim)` with HNSW, so `db` must be opened with the vector extension.
|
|
13
11
|
*/
|
|
14
|
-
|
|
15
|
-
const ownsDb = !options?.db;
|
|
16
|
-
const db = options?.db ?? await PGlite.create({ extensions: { vector } });
|
|
12
|
+
function createPGLiteIndexer(options) {
|
|
17
13
|
return createSqlIndexer({
|
|
18
|
-
db:
|
|
19
|
-
dialect: pgliteDialect
|
|
20
|
-
onClose: ownsDb ? async () => {
|
|
21
|
-
await db.close();
|
|
22
|
-
} : void 0
|
|
14
|
+
db: options.db,
|
|
15
|
+
dialect: pgliteDialect
|
|
23
16
|
});
|
|
24
17
|
}
|
|
25
18
|
//#endregion
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pglite-indexer.js","names":[],"sources":["../src/pglite-indexer.ts"],"sourcesContent":["import
|
|
1
|
+
{"version":3,"file":"pglite-indexer.js","names":[],"sources":["../src/pglite-indexer.ts"],"sourcesContent":["import type { Db } from \"@statewalker/db-api\";\nimport type { Indexer } from \"@statewalker/indexer-api\";\nimport { createSqlIndexer } from \"@statewalker/indexer-core\";\nimport { pgliteDialect } from \"./dialect.js\";\n\nexport interface PGLiteIndexerOptions {\n /** A PGlite-backed `Db` with the pgvector extension loaded. The indexer never closes it. */\n db: Db;\n}\n\n/**\n * PGlite-backed `Indexer` over the open-registry contract.\n *\n * Wires the per-Index docs table, per-sub-index FTS / vector tables, and the\n * `__indexer_manifest` table via `createSqlIndexer`. The FTS modality uses\n * PostgreSQL `tsvector` with GIN; the vector modality uses pgvector's\n * `vector(dim)` with HNSW, so `db` must be opened with the vector extension.\n */\nexport function createPGLiteIndexer(options: PGLiteIndexerOptions): Promise<Indexer> {\n return createSqlIndexer({ db: options.db, dialect: pgliteDialect });\n}\n"],"mappings":";;;;;;;;;;;AAkBA,SAAgB,oBAAoB,SAAiD;CACnF,OAAO,iBAAiB;EAAE,IAAI,QAAQ;EAAI,SAAS;CAAc,CAAC;AACpE"}
|
package/package.json
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/indexer-pglite",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
|
-
"description": "PGlite-backed indexer implementation: full-text + vector search
|
|
7
|
-
"
|
|
6
|
+
"description": "PGlite-backed indexer implementation: full-text + vector search over a @statewalker/db-api Db.",
|
|
7
|
+
"bugs": {
|
|
8
|
+
"url": "https://github.com/statewalker/statewalker-search/issues"
|
|
9
|
+
},
|
|
10
|
+
"homepage": "https://github.com/statewalker/statewalker-search/tree/main/packages/indexer-pglite#readme",
|
|
8
11
|
"author": {
|
|
9
12
|
"name": "Mikhail Kotelnikov",
|
|
10
13
|
"email": "mikhail.kotelnikov@gmail.com"
|
|
@@ -12,7 +15,8 @@
|
|
|
12
15
|
"license": "MIT",
|
|
13
16
|
"repository": {
|
|
14
17
|
"type": "git",
|
|
15
|
-
"url": "git+
|
|
18
|
+
"url": "git+https://github.com/statewalker/statewalker-search.git",
|
|
19
|
+
"directory": "packages/indexer-pglite"
|
|
16
20
|
},
|
|
17
21
|
"exports": {
|
|
18
22
|
".": {
|
|
@@ -27,14 +31,15 @@
|
|
|
27
31
|
"src"
|
|
28
32
|
],
|
|
29
33
|
"dependencies": {
|
|
30
|
-
"@
|
|
31
|
-
"@
|
|
32
|
-
"@statewalker/indexer-
|
|
33
|
-
"@statewalker/indexer-
|
|
34
|
-
"@statewalker/indexer-
|
|
35
|
-
"@statewalker/indexer-vector": "^0.1.1"
|
|
34
|
+
"@statewalker/db-api": "^0.1.0",
|
|
35
|
+
"@statewalker/indexer-api": "^0.1.3",
|
|
36
|
+
"@statewalker/indexer-core": "^0.1.2",
|
|
37
|
+
"@statewalker/indexer-fulltext": "^0.1.3",
|
|
38
|
+
"@statewalker/indexer-vector": "^0.1.3"
|
|
36
39
|
},
|
|
37
40
|
"devDependencies": {
|
|
41
|
+
"@electric-sql/pglite-pgvector": "^0.0.9",
|
|
42
|
+
"@statewalker/db-pglite": "^0.1.0",
|
|
38
43
|
"rimraf": "^6.1.3",
|
|
39
44
|
"tsdown": "^0.23.0",
|
|
40
45
|
"typescript": "^7.0.2",
|
package/src/dialect.ts
CHANGED
|
@@ -1,23 +1,6 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
3
|
-
SqlBackedDialect,
|
|
4
|
-
SqlDb,
|
|
5
|
-
SqlFtsDialect,
|
|
6
|
-
SqlVectorDialect,
|
|
7
|
-
} from "@statewalker/indexer-core";
|
|
1
|
+
import type { SqlBackedDialect, SqlFtsDialect, SqlVectorDialect } from "@statewalker/indexer-core";
|
|
8
2
|
import { buildPathPrefixesSql } from "@statewalker/indexer-core";
|
|
9
3
|
|
|
10
|
-
/** Adapt `@electric-sql/pglite`'s `PGlite` to `@statewalker/indexer-core`'s minimal `SqlDb`. */
|
|
11
|
-
export function wrapDbAsSqlDb(db: PGlite): SqlDb {
|
|
12
|
-
return {
|
|
13
|
-
exec: (sql) => db.exec(sql).then(() => undefined),
|
|
14
|
-
async query<T>(sql: string, params?: unknown[]): Promise<T[]> {
|
|
15
|
-
const { rows } = await db.query<T>(sql, params ?? []);
|
|
16
|
-
return rows;
|
|
17
|
-
},
|
|
18
|
-
};
|
|
19
|
-
}
|
|
20
|
-
|
|
21
4
|
const LANGUAGE_MAP: Record<string, string> = {
|
|
22
5
|
en: "english",
|
|
23
6
|
fr: "french",
|
package/src/pglite-indexer.ts
CHANGED
|
@@ -1,32 +1,21 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { vector } from "@electric-sql/pglite-pgvector";
|
|
1
|
+
import type { Db } from "@statewalker/db-api";
|
|
3
2
|
import type { Indexer } from "@statewalker/indexer-api";
|
|
4
3
|
import { createSqlIndexer } from "@statewalker/indexer-core";
|
|
5
|
-
import { pgliteDialect
|
|
4
|
+
import { pgliteDialect } from "./dialect.js";
|
|
6
5
|
|
|
7
6
|
export interface PGLiteIndexerOptions {
|
|
8
|
-
|
|
7
|
+
/** A PGlite-backed `Db` with the pgvector extension loaded. The indexer never closes it. */
|
|
8
|
+
db: Db;
|
|
9
9
|
}
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
12
|
* PGlite-backed `Indexer` over the open-registry contract.
|
|
13
13
|
*
|
|
14
14
|
* Wires the per-Index docs table, per-sub-index FTS / vector tables, and the
|
|
15
|
-
* `__indexer_manifest` table via `createSqlIndexer`.
|
|
16
|
-
*
|
|
17
|
-
* with
|
|
15
|
+
* `__indexer_manifest` table via `createSqlIndexer`. The FTS modality uses
|
|
16
|
+
* PostgreSQL `tsvector` with GIN; the vector modality uses pgvector's
|
|
17
|
+
* `vector(dim)` with HNSW, so `db` must be opened with the vector extension.
|
|
18
18
|
*/
|
|
19
|
-
export
|
|
20
|
-
|
|
21
|
-
const db = options?.db ?? (await PGlite.create({ extensions: { vector } }));
|
|
22
|
-
|
|
23
|
-
return createSqlIndexer({
|
|
24
|
-
db: wrapDbAsSqlDb(db),
|
|
25
|
-
dialect: pgliteDialect,
|
|
26
|
-
onClose: ownsDb
|
|
27
|
-
? async () => {
|
|
28
|
-
await db.close();
|
|
29
|
-
}
|
|
30
|
-
: undefined,
|
|
31
|
-
});
|
|
19
|
+
export function createPGLiteIndexer(options: PGLiteIndexerOptions): Promise<Indexer> {
|
|
20
|
+
return createSqlIndexer({ db: options.db, dialect: pgliteDialect });
|
|
32
21
|
}
|