@statewalker/indexer-pglite 0.2.2 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -14
- package/dist/dialect.js +2 -13
- package/dist/dialect.js.map +1 -1
- package/dist/pglite-indexer.d.ts +7 -6
- package/dist/pglite-indexer.d.ts.map +1 -1
- package/dist/pglite-indexer.js +7 -14
- package/dist/pglite-indexer.js.map +1 -1
- package/package.json +9 -8
- package/src/dialect.ts +2 -20
- package/src/pglite-indexer.ts +9 -20
package/README.md
CHANGED
|
@@ -6,30 +6,33 @@ An `Indexer` stored in [PGlite](https://github.com/electric-sql/pglite) (Postgre
|
|
|
6
6
|
|
|
7
7
|
## Why it exists
|
|
8
8
|
|
|
9
|
-
It gives real PostgreSQL text search (stemming per language) and an approximate nearest-neighbour vector index without a server. With an IndexedDB or file `
|
|
9
|
+
It gives real PostgreSQL text search (stemming per language) and an approximate nearest-neighbour vector index without a server. With an IndexedDB or file `path`, data persists without a separate save step.
|
|
10
10
|
|
|
11
11
|
## How to use
|
|
12
12
|
|
|
13
13
|
```sh
|
|
14
|
-
pnpm add @statewalker/indexer-pglite
|
|
14
|
+
pnpm add @statewalker/indexer-pglite @statewalker/db-pglite @electric-sql/pglite-pgvector
|
|
15
15
|
```
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
The indexer runs over a `Db` from `@statewalker/db-pglite`. Open it with the pgvector extension — the indexer issues `CREATE EXTENSION IF NOT EXISTS vector` and fails with `extension "vector" is not available` otherwise. The indexer never closes the `Db`; close it yourself.
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
The `Db` must come from `newPgliteDb` (PostgreSQL dialect). A SQLite `Db` typechecks, because both are `Db`, but fails at the first `CREATE EXTENSION` with `near "EXTENSION": syntax error`.
|
|
20
20
|
|
|
21
|
-
- `createPGLiteIndexer(
|
|
21
|
+
- `createPGLiteIndexer({ db }): Promise<Indexer>` — `db: Db`.
|
|
22
22
|
- `PGLiteIndexerOptions` — the options type.
|
|
23
23
|
|
|
24
24
|
## Examples
|
|
25
25
|
|
|
26
|
-
In-memory database
|
|
26
|
+
In-memory database:
|
|
27
27
|
|
|
28
28
|
```ts
|
|
29
|
+
import { vector } from "@electric-sql/pglite-pgvector";
|
|
30
|
+
import { newPgliteDb } from "@statewalker/db-pglite";
|
|
29
31
|
import { createPGLiteIndexer } from "@statewalker/indexer-pglite";
|
|
30
32
|
import { newFullTextAccess, setFullTextConfig } from "@statewalker/indexer-fulltext";
|
|
31
33
|
|
|
32
|
-
const
|
|
34
|
+
const db = await newPgliteDb({ extensions: { vector } });
|
|
35
|
+
const indexer = await createPGLiteIndexer({ db });
|
|
33
36
|
|
|
34
37
|
const params = { name: "docs" };
|
|
35
38
|
setFullTextConfig(params, "q", { language: "en" });
|
|
@@ -44,26 +47,30 @@ for await (const r of index.search(request)) {
|
|
|
44
47
|
console.log(r.path, r.blockId, fts.getResult(r)?.score); // ts_rank_cd
|
|
45
48
|
}
|
|
46
49
|
|
|
47
|
-
await indexer.close();
|
|
50
|
+
await indexer.close();
|
|
51
|
+
await db.close();
|
|
48
52
|
```
|
|
49
53
|
|
|
50
|
-
|
|
54
|
+
Persistent database:
|
|
51
55
|
|
|
52
56
|
```ts
|
|
53
|
-
import { PGlite } from "@electric-sql/pglite";
|
|
54
57
|
import { vector } from "@electric-sql/pglite-pgvector";
|
|
58
|
+
import { newPgliteDb } from "@statewalker/db-pglite";
|
|
55
59
|
import { createPGLiteIndexer } from "@statewalker/indexer-pglite";
|
|
56
60
|
|
|
57
|
-
|
|
61
|
+
// a directory on Node.js ("./pgdata"), IndexedDB in the browser ("idb://my-app")
|
|
62
|
+
const db = await newPgliteDb({ path: "idb://my-app", extensions: { vector } });
|
|
58
63
|
const indexer = await createPGLiteIndexer({ db });
|
|
59
|
-
//
|
|
64
|
+
// ...
|
|
65
|
+
await indexer.close(); // does not close db
|
|
66
|
+
await db.close();
|
|
60
67
|
```
|
|
61
68
|
|
|
62
69
|
## Internals
|
|
63
70
|
|
|
64
71
|
### The vector extension must be loaded
|
|
65
72
|
|
|
66
|
-
At startup the indexer runs `CREATE EXTENSION IF NOT EXISTS vector`.
|
|
73
|
+
At startup the indexer runs `CREATE EXTENSION IF NOT EXISTS vector`. The `Db` you pass in must be opened with `extensions: { vector }`, otherwise this statement fails with `extension "vector" is not available` and the indexer is not created.
|
|
67
74
|
|
|
68
75
|
### Tables
|
|
69
76
|
|
|
@@ -85,7 +92,7 @@ idx_<index>_<sub>_vec(doc_id, block_id, embedding vector(dim))
|
|
|
85
92
|
|
|
86
93
|
### Dependencies
|
|
87
94
|
|
|
88
|
-
`@
|
|
95
|
+
`@statewalker/db-api` (the `Db` it runs on), `@statewalker/indexer-core` (`createSqlIndexer` and the SQL retrievers), `@statewalker/indexer-api`, `@statewalker/indexer-fulltext`, `@statewalker/indexer-vector`.
|
|
89
96
|
|
|
90
97
|
## License
|
|
91
98
|
|
package/dist/dialect.js
CHANGED
|
@@ -1,15 +1,5 @@
|
|
|
1
1
|
import { buildPathPrefixesSql } from "@statewalker/indexer-core";
|
|
2
2
|
//#region src/dialect.ts
|
|
3
|
-
/** Adapt `@electric-sql/pglite`'s `PGlite` to `@statewalker/indexer-core`'s minimal `SqlDb`. */
|
|
4
|
-
function wrapDbAsSqlDb(db) {
|
|
5
|
-
return {
|
|
6
|
-
exec: (sql) => db.exec(sql).then(() => void 0),
|
|
7
|
-
async query(sql, params) {
|
|
8
|
-
const { rows } = await db.query(sql, params ?? []);
|
|
9
|
-
return rows;
|
|
10
|
-
}
|
|
11
|
-
};
|
|
12
|
-
}
|
|
13
3
|
const LANGUAGE_MAP = {
|
|
14
4
|
en: "english",
|
|
15
5
|
fr: "french",
|
|
@@ -76,8 +66,7 @@ const pgliteFtsDialect = {
|
|
|
76
66
|
};
|
|
77
67
|
/**
|
|
78
68
|
* PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.
|
|
79
|
-
* Embeddings are
|
|
80
|
-
* driver-native arrays.
|
|
69
|
+
* Embeddings are bound as stringified array literals.
|
|
81
70
|
*/
|
|
82
71
|
const pgliteVectorDialect = {
|
|
83
72
|
createTableDdl({ tableName, indexName, info }) {
|
|
@@ -119,6 +108,6 @@ const pgliteDialect = {
|
|
|
119
108
|
vec: pgliteVectorDialect
|
|
120
109
|
};
|
|
121
110
|
//#endregion
|
|
122
|
-
export { pgliteDialect, pgliteFtsDialect, pgliteVectorDialect
|
|
111
|
+
export { pgliteDialect, pgliteFtsDialect, pgliteVectorDialect };
|
|
123
112
|
|
|
124
113
|
//# sourceMappingURL=dialect.js.map
|
package/dist/dialect.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dialect.js","names":[],"sources":["../src/dialect.ts"],"sourcesContent":["import type {
|
|
1
|
+
{"version":3,"file":"dialect.js","names":[],"sources":["../src/dialect.ts"],"sourcesContent":["import type { SqlBackedDialect, SqlFtsDialect, SqlVectorDialect } from \"@statewalker/indexer-core\";\nimport { buildPathPrefixesSql } from \"@statewalker/indexer-core\";\n\nconst LANGUAGE_MAP: Record<string, string> = {\n en: \"english\",\n fr: \"french\",\n de: \"german\",\n es: \"spanish\",\n it: \"italian\",\n pt: \"portuguese\",\n nl: \"dutch\",\n ru: \"russian\",\n sv: \"swedish\",\n no: \"norwegian\",\n da: \"danish\",\n fi: \"finnish\",\n hu: \"hungarian\",\n ro: \"romanian\",\n tr: \"turkish\",\n simple: \"simple\",\n};\n\n// Postgres text-search configuration names that have always-on availability;\n// any value outside this set is interpolated into DDL/queries and must be a\n// whitelisted identifier — never user-controlled free-form text.\nconst KNOWN_PG_CONFIGS: ReadonlySet<string> = new Set(Object.values(LANGUAGE_MAP));\n\nfunction resolvePgLanguage(lang: string): string {\n const mapped = LANGUAGE_MAP[lang];\n if (mapped) return mapped;\n if (KNOWN_PG_CONFIGS.has(lang)) return lang;\n throw new Error(\n `pglite FTS: unsupported language \"${lang}\" (must be an ISO-639-1 code or a Postgres text-search config name)`,\n );\n}\n\n/**\n * pgvector's `vector(dim)` column accepts either a JSON-array text literal (`\"[1,2,3]\"`) cast with\n * `::vector(dim)`, or a JS array bound by the driver. The text-literal path is currently used\n * because it works uniformly across the pglite vector extension versions available in the catalog;\n * swapping to native array binding is a follow-up when driver support is confirmed.\n */\nfunction embeddingToLiteral(embedding: Float32Array): string {\n return `[${Array.from(embedding).join(\",\")}]`;\n}\n\n/**\n * PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring,\n * and `to_tsquery` with a language-specific stemmer.\n */\nexport const pgliteFtsDialect: SqlFtsDialect = {\n createTableDdl({ tableName, info }) {\n const pgLang = resolvePgLanguage(info.language);\n return [\n `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('${pgLang}', content)) STORED, metadata TEXT, PRIMARY KEY (doc_id, block_id))`,\n `CREATE INDEX IF NOT EXISTS ${tableName}_tsv_idx ON ${tableName} USING GIN (content_tsv)`,\n ];\n },\n\n async search({ db, tableName, docsTable, info, query, paths, topK }) {\n const pgLang = resolvePgLanguage(info.language);\n // Strip only tsquery-significant punctuation; keep Unicode letters/digits so\n // non-ASCII languages (Russian, French, CJK, …) reach the FTS engine intact.\n const validWords = query\n .toLowerCase()\n .split(/\\s+/)\n .map((w) => w.replace(/[&|!():'\"\\\\]/g, \"\"))\n .filter((w) => w.length > 0);\n if (validWords.length === 0) return [];\n\n const orTerms = validWords.join(\" | \");\n\n const allParams: unknown[] = [orTerms];\n let pathClause = \"\";\n if (paths && paths.length > 0) {\n const cond = buildPathPrefixesSql(\"d.path\", paths as string[], allParams.length + 1);\n pathClause = ` AND ${cond.sql}`;\n allParams.push(...cond.params);\n }\n\n const topKParam = `$${allParams.length + 1}`;\n allParams.push(topK);\n\n const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`;\n\n const rows = await db.query<{\n path: string;\n block_id: string;\n content: string;\n score: number;\n }>(sql, allParams);\n\n return rows.map((row) => ({\n path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n blockId: row.block_id,\n content: row.content,\n score: row.score,\n }));\n },\n};\n\n/**\n * PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.\n * Embeddings are bound as stringified array literals.\n */\nexport const pgliteVectorDialect: SqlVectorDialect = {\n createTableDdl({ tableName, indexName, info }) {\n const dim = info.dimensionality;\n return [\n `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding vector(${dim}) NOT NULL, PRIMARY KEY (doc_id, block_id))`,\n `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING hnsw (embedding vector_cosine_ops)`,\n ];\n },\n\n bindEmbedding: embeddingToLiteral,\n embeddingCastSuffix: (dim) => `::vector(${dim})`,\n\n async search({\n db,\n tableName,\n docsTable,\n queryEmbedding,\n paths,\n topK,\n info,\n bindEmbedding,\n embeddingCastSuffix,\n }) {\n const vecLiteral = bindEmbedding(queryEmbedding);\n const dim = info.dimensionality;\n\n const allParams: unknown[] = [vecLiteral];\n let pathClause = \"\";\n if (paths && paths.length > 0) {\n const cond = buildPathPrefixesSql(\"d.path\", paths as string[], allParams.length + 1);\n pathClause = `WHERE ${cond.sql} `;\n allParams.push(...cond.params);\n }\n\n const topKParam = `$${allParams.length + 1}`;\n allParams.push(topK);\n\n const rows = await db.query<{ path: string; block_id: string; dist: number }>(\n `SELECT d.path, b.block_id, (b.embedding <=> $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`,\n allParams,\n );\n\n return rows.map((row) => ({\n path: row.path as import(\"@statewalker/indexer-api\").DocumentPath,\n blockId: row.block_id,\n score: 1 - row.dist,\n }));\n },\n\n decodeEmbedding(raw) {\n // PGlite returns vector as a string \"[1,2,3]\".\n return new Float32Array(JSON.parse(raw as string) as number[]);\n },\n};\n\n/** Aggregate PGlite dialect for `createSqlBackedIndexer`. */\nexport const pgliteDialect: SqlBackedDialect = {\n extensionInit: [\"CREATE EXTENSION IF NOT EXISTS vector\"],\n docsTableDdl(prefix) {\n return [\n `CREATE TABLE IF NOT EXISTS idx_${prefix}_docs (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`,\n ];\n },\n unionAliasSuffix: \" AS combined\",\n supportsDDLInTransaction: true,\n fts: pgliteFtsDialect,\n vec: pgliteVectorDialect,\n};\n"],"mappings":";;AAGA,MAAM,eAAuC;CAC3C,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,QAAQ;AACV;AAKA,MAAM,mBAAwC,IAAI,IAAI,OAAO,OAAO,YAAY,CAAC;AAEjF,SAAS,kBAAkB,MAAsB;CAC/C,MAAM,SAAS,aAAa;CAC5B,IAAI,QAAQ,OAAO;CACnB,IAAI,iBAAiB,IAAI,IAAI,GAAG,OAAO;CACvC,MAAM,IAAI,MACR,qCAAqC,KAAK,oEAC5C;AACF;;;;;;;AAQA,SAAS,mBAAmB,WAAiC;CAC3D,OAAO,IAAI,MAAM,KAAK,SAAS,CAAC,CAAC,KAAK,GAAG,EAAE;AAC7C;;;;;AAMA,MAAa,mBAAkC;CAC7C,eAAe,EAAE,WAAW,QAAQ;EAElC,OAAO,CACL,8BAA8B,UAAU,mIAF3B,kBAAkB,KAAK,QAE4I,EAAE,sEAClL,8BAA8B,UAAU,cAAc,UAAU,yBAClE;CACF;CAEA,MAAM,OAAO,EAAE,IAAI,WAAW,WAAW,MAAM,OAAO,OAAO,QAAQ;EACnE,MAAM,SAAS,kBAAkB,KAAK,QAAQ;EAG9C,MAAM,aAAa,MAChB,YAAY,CAAC,CACb,MAAM,KAAK,CAAC,CACZ,KAAK,MAAM,EAAE,QAAQ,iBAAiB,EAAE,CAAC,CAAC,CAC1C,QAAQ,MAAM,EAAE,SAAS,CAAC;EAC7B,IAAI,WAAW,WAAW,GAAG,OAAO,CAAC;EAIrC,MAAM,YAAuB,CAFb,WAAW,KAAK,KAEI,CAAC;EACrC,IAAI,aAAa;EACjB,IAAI,SAAS,MAAM,SAAS,GAAG;GAC7B,MAAM,OAAO,qBAAqB,UAAU,OAAmB,UAAU,SAAS,CAAC;GACnF,aAAa,QAAQ,KAAK;GAC1B,UAAU,KAAK,GAAG,KAAK,MAAM;EAC/B;EAEA,MAAM,YAAY,IAAI,UAAU,SAAS;EACzC,UAAU,KAAK,IAAI;EAEnB,MAAM,MAAM,+EAA+E,OAAO,wBAAwB,UAAU,UAAU,UAAU,+DAA+D,OAAO,QAAQ,WAAW,6BAA6B;EAS9Q,QAAO,MAPY,GAAG,MAKnB,KAAK,SAAS,EAAA,CAEL,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,SAAS,IAAI;GACb,OAAO,IAAI;EACb,EAAE;CACJ;AACF;;;;;AAMA,MAAa,sBAAwC;CACnD,eAAe,EAAE,WAAW,WAAW,QAAQ;EAE7C,OAAO,CACL,8BAA8B,UAAU,sEAF9B,KAAK,eAEmG,8CAClH,8BAA8B,UAAU,MAAM,UAAU,0CAC1D;CACF;CAEA,eAAe;CACf,sBAAsB,QAAQ,YAAY,IAAI;CAE9C,MAAM,OAAO,EACX,IACA,WACA,WACA,gBACA,OACA,MACA,MACA,eACA,uBACC;EACD,MAAM,aAAa,cAAc,cAAc;EAC/C,MAAM,MAAM,KAAK;EAEjB,MAAM,YAAuB,CAAC,UAAU;EACxC,IAAI,aAAa;EACjB,IAAI,SAAS,MAAM,SAAS,GAAG;GAC7B,MAAM,OAAO,qBAAqB,UAAU,OAAmB,UAAU,SAAS,CAAC;GACnF,aAAa,SAAS,KAAK,IAAI;GAC/B,UAAU,KAAK,GAAG,KAAK,MAAM;EAC/B;EAEA,MAAM,YAAY,IAAI,UAAU,SAAS;EACzC,UAAU,KAAK,IAAI;EAOnB,QAAO,MALY,GAAG,MACpB,iDAAiD,oBAAoB,GAAG,EAAE,iBAAiB,UAAU,UAAU,UAAU,4BAA4B,WAAW,0BAA0B,aAC1L,SACF,EAAA,CAEY,KAAK,SAAS;GACxB,MAAM,IAAI;GACV,SAAS,IAAI;GACb,OAAO,IAAI,IAAI;EACjB,EAAE;CACJ;CAEA,gBAAgB,KAAK;EAEnB,OAAO,IAAI,aAAa,KAAK,MAAM,GAAa,CAAa;CAC/D;AACF;;AAGA,MAAa,gBAAkC;CAC7C,eAAe,CAAC,uCAAuC;CACvD,aAAa,QAAQ;EACnB,OAAO,CACL,kCAAkC,OAAO,6DAC3C;CACF;CACA,kBAAkB;CAClB,0BAA0B;CAC1B,KAAK;CACL,KAAK;AACP"}
|
package/dist/pglite-indexer.d.ts
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Db } from "@statewalker/db-api";
|
|
2
2
|
import { Indexer } from "@statewalker/indexer-api";
|
|
3
3
|
//#region src/pglite-indexer.d.ts
|
|
4
4
|
export interface PGLiteIndexerOptions {
|
|
5
|
-
|
|
5
|
+
/** A PGlite-backed `Db` with the pgvector extension loaded. The indexer never closes it. */
|
|
6
|
+
db: Db;
|
|
6
7
|
}
|
|
7
8
|
/**
|
|
8
9
|
* PGlite-backed `Indexer` over the open-registry contract.
|
|
9
10
|
*
|
|
10
11
|
* Wires the per-Index docs table, per-sub-index FTS / vector tables, and the
|
|
11
|
-
* `__indexer_manifest` table via `createSqlIndexer`.
|
|
12
|
-
*
|
|
13
|
-
* with
|
|
12
|
+
* `__indexer_manifest` table via `createSqlIndexer`. The FTS modality uses
|
|
13
|
+
* PostgreSQL `tsvector` with GIN; the vector modality uses pgvector's
|
|
14
|
+
* `vector(dim)` with HNSW, so `db` must be opened with the vector extension.
|
|
14
15
|
*/
|
|
15
|
-
export declare function createPGLiteIndexer(options
|
|
16
|
+
export declare function createPGLiteIndexer(options: PGLiteIndexerOptions): Promise<Indexer>;
|
|
16
17
|
//#endregion
|
|
17
18
|
//# sourceMappingURL=pglite-indexer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pglite-indexer.d.ts","names":[],"sources":["../src/pglite-indexer.ts"],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"pglite-indexer.d.ts","names":[],"sources":["../src/pglite-indexer.ts"],"mappings":";;;iBAKiB;;EAEf,IAAI;;;;;;;;;;wBAWU,oBAAoB,SAAS,uBAAuB,QAAQ"}
|
package/dist/pglite-indexer.js
CHANGED
|
@@ -1,25 +1,18 @@
|
|
|
1
|
-
import { pgliteDialect
|
|
2
|
-
import { PGlite } from "@electric-sql/pglite";
|
|
3
|
-
import { vector } from "@electric-sql/pglite-pgvector";
|
|
1
|
+
import { pgliteDialect } from "./dialect.js";
|
|
4
2
|
import { createSqlIndexer } from "@statewalker/indexer-core";
|
|
5
3
|
//#region src/pglite-indexer.ts
|
|
6
4
|
/**
|
|
7
5
|
* PGlite-backed `Indexer` over the open-registry contract.
|
|
8
6
|
*
|
|
9
7
|
* Wires the per-Index docs table, per-sub-index FTS / vector tables, and the
|
|
10
|
-
* `__indexer_manifest` table via `createSqlIndexer`.
|
|
11
|
-
*
|
|
12
|
-
* with
|
|
8
|
+
* `__indexer_manifest` table via `createSqlIndexer`. The FTS modality uses
|
|
9
|
+
* PostgreSQL `tsvector` with GIN; the vector modality uses pgvector's
|
|
10
|
+
* `vector(dim)` with HNSW, so `db` must be opened with the vector extension.
|
|
13
11
|
*/
|
|
14
|
-
|
|
15
|
-
const ownsDb = !options?.db;
|
|
16
|
-
const db = options?.db ?? await PGlite.create({ extensions: { vector } });
|
|
12
|
+
function createPGLiteIndexer(options) {
|
|
17
13
|
return createSqlIndexer({
|
|
18
|
-
db:
|
|
19
|
-
dialect: pgliteDialect
|
|
20
|
-
onClose: ownsDb ? async () => {
|
|
21
|
-
await db.close();
|
|
22
|
-
} : void 0
|
|
14
|
+
db: options.db,
|
|
15
|
+
dialect: pgliteDialect
|
|
23
16
|
});
|
|
24
17
|
}
|
|
25
18
|
//#endregion
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pglite-indexer.js","names":[],"sources":["../src/pglite-indexer.ts"],"sourcesContent":["import
|
|
1
|
+
{"version":3,"file":"pglite-indexer.js","names":[],"sources":["../src/pglite-indexer.ts"],"sourcesContent":["import type { Db } from \"@statewalker/db-api\";\nimport type { Indexer } from \"@statewalker/indexer-api\";\nimport { createSqlIndexer } from \"@statewalker/indexer-core\";\nimport { pgliteDialect } from \"./dialect.js\";\n\nexport interface PGLiteIndexerOptions {\n /** A PGlite-backed `Db` with the pgvector extension loaded. The indexer never closes it. */\n db: Db;\n}\n\n/**\n * PGlite-backed `Indexer` over the open-registry contract.\n *\n * Wires the per-Index docs table, per-sub-index FTS / vector tables, and the\n * `__indexer_manifest` table via `createSqlIndexer`. The FTS modality uses\n * PostgreSQL `tsvector` with GIN; the vector modality uses pgvector's\n * `vector(dim)` with HNSW, so `db` must be opened with the vector extension.\n */\nexport function createPGLiteIndexer(options: PGLiteIndexerOptions): Promise<Indexer> {\n return createSqlIndexer({ db: options.db, dialect: pgliteDialect });\n}\n"],"mappings":";;;;;;;;;;;AAkBA,SAAgB,oBAAoB,SAAiD;CACnF,OAAO,iBAAiB;EAAE,IAAI,QAAQ;EAAI,SAAS;CAAc,CAAC;AACpE"}
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/indexer-pglite",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
|
-
"description": "PGlite-backed indexer implementation: full-text + vector search
|
|
6
|
+
"description": "PGlite-backed indexer implementation: full-text + vector search over a @statewalker/db-api Db.",
|
|
7
7
|
"bugs": {
|
|
8
8
|
"url": "https://github.com/statewalker/statewalker-search/issues"
|
|
9
9
|
},
|
|
@@ -31,14 +31,15 @@
|
|
|
31
31
|
"src"
|
|
32
32
|
],
|
|
33
33
|
"dependencies": {
|
|
34
|
-
"@
|
|
35
|
-
"@
|
|
36
|
-
"@statewalker/indexer-
|
|
37
|
-
"@statewalker/indexer-core": "^0.1.
|
|
38
|
-
"@statewalker/indexer-
|
|
39
|
-
"@statewalker/indexer-vector": "^0.1.2"
|
|
34
|
+
"@statewalker/db-api": "^0.1.0",
|
|
35
|
+
"@statewalker/indexer-api": "^0.1.3",
|
|
36
|
+
"@statewalker/indexer-fulltext": "^0.1.4",
|
|
37
|
+
"@statewalker/indexer-core": "^0.1.3",
|
|
38
|
+
"@statewalker/indexer-vector": "^0.1.4"
|
|
40
39
|
},
|
|
41
40
|
"devDependencies": {
|
|
41
|
+
"@electric-sql/pglite-pgvector": "^0.0.9",
|
|
42
|
+
"@statewalker/db-pglite": "^0.1.0",
|
|
42
43
|
"rimraf": "^6.1.3",
|
|
43
44
|
"tsdown": "^0.23.0",
|
|
44
45
|
"typescript": "^7.0.2",
|
package/src/dialect.ts
CHANGED
|
@@ -1,23 +1,6 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
3
|
-
SqlBackedDialect,
|
|
4
|
-
SqlDb,
|
|
5
|
-
SqlFtsDialect,
|
|
6
|
-
SqlVectorDialect,
|
|
7
|
-
} from "@statewalker/indexer-core";
|
|
1
|
+
import type { SqlBackedDialect, SqlFtsDialect, SqlVectorDialect } from "@statewalker/indexer-core";
|
|
8
2
|
import { buildPathPrefixesSql } from "@statewalker/indexer-core";
|
|
9
3
|
|
|
10
|
-
/** Adapt `@electric-sql/pglite`'s `PGlite` to `@statewalker/indexer-core`'s minimal `SqlDb`. */
|
|
11
|
-
export function wrapDbAsSqlDb(db: PGlite): SqlDb {
|
|
12
|
-
return {
|
|
13
|
-
exec: (sql) => db.exec(sql).then(() => undefined),
|
|
14
|
-
async query<T>(sql: string, params?: unknown[]): Promise<T[]> {
|
|
15
|
-
const { rows } = await db.query<T>(sql, params ?? []);
|
|
16
|
-
return rows;
|
|
17
|
-
},
|
|
18
|
-
};
|
|
19
|
-
}
|
|
20
|
-
|
|
21
4
|
const LANGUAGE_MAP: Record<string, string> = {
|
|
22
5
|
en: "english",
|
|
23
6
|
fr: "french",
|
|
@@ -118,8 +101,7 @@ export const pgliteFtsDialect: SqlFtsDialect = {
|
|
|
118
101
|
|
|
119
102
|
/**
|
|
120
103
|
* PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator.
|
|
121
|
-
* Embeddings are
|
|
122
|
-
* driver-native arrays.
|
|
104
|
+
* Embeddings are bound as stringified array literals.
|
|
123
105
|
*/
|
|
124
106
|
export const pgliteVectorDialect: SqlVectorDialect = {
|
|
125
107
|
createTableDdl({ tableName, indexName, info }) {
|
package/src/pglite-indexer.ts
CHANGED
|
@@ -1,32 +1,21 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { vector } from "@electric-sql/pglite-pgvector";
|
|
1
|
+
import type { Db } from "@statewalker/db-api";
|
|
3
2
|
import type { Indexer } from "@statewalker/indexer-api";
|
|
4
3
|
import { createSqlIndexer } from "@statewalker/indexer-core";
|
|
5
|
-
import { pgliteDialect
|
|
4
|
+
import { pgliteDialect } from "./dialect.js";
|
|
6
5
|
|
|
7
6
|
export interface PGLiteIndexerOptions {
|
|
8
|
-
|
|
7
|
+
/** A PGlite-backed `Db` with the pgvector extension loaded. The indexer never closes it. */
|
|
8
|
+
db: Db;
|
|
9
9
|
}
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
12
|
* PGlite-backed `Indexer` over the open-registry contract.
|
|
13
13
|
*
|
|
14
14
|
* Wires the per-Index docs table, per-sub-index FTS / vector tables, and the
|
|
15
|
-
* `__indexer_manifest` table via `createSqlIndexer`.
|
|
16
|
-
*
|
|
17
|
-
* with
|
|
15
|
+
* `__indexer_manifest` table via `createSqlIndexer`. The FTS modality uses
|
|
16
|
+
* PostgreSQL `tsvector` with GIN; the vector modality uses pgvector's
|
|
17
|
+
* `vector(dim)` with HNSW, so `db` must be opened with the vector extension.
|
|
18
18
|
*/
|
|
19
|
-
export
|
|
20
|
-
|
|
21
|
-
const db = options?.db ?? (await PGlite.create({ extensions: { vector } }));
|
|
22
|
-
|
|
23
|
-
return createSqlIndexer({
|
|
24
|
-
db: wrapDbAsSqlDb(db),
|
|
25
|
-
dialect: pgliteDialect,
|
|
26
|
-
onClose: ownsDb
|
|
27
|
-
? async () => {
|
|
28
|
-
await db.close();
|
|
29
|
-
}
|
|
30
|
-
: undefined,
|
|
31
|
-
});
|
|
19
|
+
export function createPGLiteIndexer(options: PGLiteIndexerOptions): Promise<Indexer> {
|
|
20
|
+
return createSqlIndexer({ db: options.db, dialect: pgliteDialect });
|
|
32
21
|
}
|