@typicalday/firegraph 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -17
- package/dist/{backend-CvImIwTY.d.cts → backend-CE3pM9-T.d.ts} +32 -2
- package/dist/{backend-BpYLdwCW.d.cts → backend-DNzv8KSR.d.cts} +33 -19
- package/dist/{backend-BpYLdwCW.d.ts → backend-DNzv8KSR.d.ts} +33 -19
- package/dist/{backend-YH5HtawN.d.ts → backend-EjFfw9yO.d.cts} +32 -2
- package/dist/backend.cjs.map +1 -1
- package/dist/backend.d.cts +2 -2
- package/dist/backend.d.ts +2 -2
- package/dist/backend.js +1 -1
- package/dist/{chunk-FODIMIWY.js → chunk-5JBNLH5W.js} +17 -6
- package/dist/chunk-5JBNLH5W.js.map +1 -0
- package/dist/{chunk-5HIRYV2S.js → chunk-6IO74NKD.js} +12 -10
- package/dist/{chunk-5HIRYV2S.js.map → chunk-6IO74NKD.js.map} +1 -1
- package/dist/{chunk-ULRDQ6HZ.js → chunk-NZVSLWNY.js} +6 -1
- package/dist/chunk-NZVSLWNY.js.map +1 -0
- package/dist/{chunk-N5HFDWQX.js → chunk-PWIO46RT.js} +1 -1
- package/dist/{chunk-N5HFDWQX.js.map → chunk-PWIO46RT.js.map} +1 -1
- package/dist/{client-B5o39X79.d.ts → client-CNAwJayO.d.ts} +1 -1
- package/dist/{client-BGHwxwPg.d.cts → client-CaXH5D5C.d.cts} +1 -1
- package/dist/cloudflare/index.cjs +11 -9
- package/dist/cloudflare/index.cjs.map +1 -1
- package/dist/cloudflare/index.d.cts +3 -3
- package/dist/cloudflare/index.d.ts +3 -3
- package/dist/cloudflare/index.js +3 -3
- package/dist/codegen/index.d.cts +1 -1
- package/dist/codegen/index.d.ts +1 -1
- package/dist/firestore-enterprise/index.cjs +11 -9
- package/dist/firestore-enterprise/index.cjs.map +1 -1
- package/dist/firestore-enterprise/index.d.cts +3 -3
- package/dist/firestore-enterprise/index.d.ts +3 -3
- package/dist/firestore-enterprise/index.js +2 -2
- package/dist/firestore-standard/index.cjs +11 -9
- package/dist/firestore-standard/index.cjs.map +1 -1
- package/dist/firestore-standard/index.d.cts +3 -3
- package/dist/firestore-standard/index.d.ts +3 -3
- package/dist/firestore-standard/index.js +2 -2
- package/dist/index.cjs +11 -9
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1 -1
- package/dist/{registry-tKTb5Kx1.d.ts → registry-By1i-zge.d.ts} +1 -1
- package/dist/{registry-BGh7Jqpb.d.cts → registry-CNToyEra.d.cts} +1 -1
- package/dist/sqlite/index.cjs +24 -12
- package/dist/sqlite/index.cjs.map +1 -1
- package/dist/sqlite/index.d.cts +4 -4
- package/dist/sqlite/index.d.ts +4 -4
- package/dist/sqlite/index.js +4 -4
- package/dist/sqlite/local.cjs +474 -47
- package/dist/sqlite/local.cjs.map +1 -1
- package/dist/sqlite/local.d.cts +31 -5
- package/dist/sqlite/local.d.ts +31 -5
- package/dist/sqlite/local.js +429 -4
- package/dist/sqlite/local.js.map +1 -1
- package/package.json +1 -1
- package/dist/chunk-FODIMIWY.js.map +0 -1
- package/dist/chunk-ULRDQ6HZ.js.map +0 -1
package/dist/sqlite/local.d.cts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Database } from 'better-sqlite3';
|
|
2
|
-
import { S as StorageBackend } from '../backend-
|
|
3
|
-
import { a as SqliteCapability, S as SqliteBackendOptions, b as SqliteExecutor } from '../backend-
|
|
2
|
+
import { S as StorageBackend } from '../backend-DNzv8KSR.cjs';
|
|
3
|
+
import { a as SqliteCapability, S as SqliteBackendOptions, b as SqliteExecutor } from '../backend-EjFfw9yO.cjs';
|
|
4
4
|
import '@google-cloud/firestore';
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -17,8 +17,34 @@ import '@google-cloud/firestore';
|
|
|
17
17
|
* `journal_mode = WAL` and a `busy_timeout` applied; caller-provided
|
|
18
18
|
* databases are used as-is (only `busy_timeout` is set) since the caller
|
|
19
19
|
* owns their pragma configuration.
|
|
20
|
+
*
|
|
21
|
+
* ## Search capabilities
|
|
22
|
+
*
|
|
23
|
+
* On top of the shared SQLite capability set, the local backend declares
|
|
24
|
+
* `search.fullText` and `search.vector` (see `src/internal/sqlite-search.ts`
|
|
25
|
+
* for the mechanics):
|
|
26
|
+
*
|
|
27
|
+
* - **Full-text search** is backed by one FTS5 table per graph table,
|
|
28
|
+
* kept in sync by pure-SQL triggers installed with the table's DDL.
|
|
29
|
+
* Because the triggers live in the database file, writes from ANY
|
|
30
|
+
* process or connection stay indexed. The trade-off is a per-write
|
|
31
|
+
* overhead (text extraction via `json_tree` + an FTS index update) on
|
|
32
|
+
* every insert/update/delete.
|
|
33
|
+
* - **Vector search** is a brute-force scan scored by a deterministic
|
|
34
|
+
* scalar UDF registered on this connection. UDFs are connection-local:
|
|
35
|
+
* `findNearest` only works through a backend created by this factory
|
|
36
|
+
* (other connections to the same file can read/write normally — only
|
|
37
|
+
* vector *search* needs the UDF).
|
|
20
38
|
*/
|
|
21
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Capability union for the local better-sqlite3 backend: everything the
|
|
42
|
+
* shared SQLite edition declares, plus native FTS5 full-text search and
|
|
43
|
+
* brute-force vector search. `search.geo` stays out — there is no geo
|
|
44
|
+
* index in stock SQLite, and a UDF-scored scan without a haversine
|
|
45
|
+
* contract pinned by Firestore parity tests would be guesswork.
|
|
46
|
+
*/
|
|
47
|
+
type LocalSqliteCapability = SqliteCapability | 'search.fullText' | 'search.vector';
|
|
22
48
|
interface LocalSqliteBackendOptions extends SqliteBackendOptions {
|
|
23
49
|
/** Root graph table name. Defaults to `'firegraph'`. */
|
|
24
50
|
tableName?: string;
|
|
@@ -41,7 +67,7 @@ interface LocalSqliteBackendOptions extends SqliteBackendOptions {
|
|
|
41
67
|
}
|
|
42
68
|
interface LocalSqliteBackend {
|
|
43
69
|
/** The graph storage backend — pass to `createGraphClient`. */
|
|
44
|
-
backend: StorageBackend<
|
|
70
|
+
backend: StorageBackend<LocalSqliteCapability>;
|
|
45
71
|
/** The underlying better-sqlite3 database, for raw access. */
|
|
46
72
|
db: Database;
|
|
47
73
|
/**
|
|
@@ -70,7 +96,7 @@ declare function createBetterSqliteExecutor(db: Database): SqliteExecutor;
|
|
|
70
96
|
*
|
|
71
97
|
* const { backend, close } = await createLocalSqliteBackend('./graph.db');
|
|
72
98
|
* const client = createGraphClient(backend);
|
|
73
|
-
* // ... use the client ...
|
|
99
|
+
* // ... use the client — including fullTextSearch() and findNearest() ...
|
|
74
100
|
* close();
|
|
75
101
|
* ```
|
|
76
102
|
*
|
|
@@ -80,4 +106,4 @@ declare function createBetterSqliteExecutor(db: Database): SqliteExecutor;
|
|
|
80
106
|
*/
|
|
81
107
|
declare function createLocalSqliteBackend(pathOrDb: string | Database, options?: LocalSqliteBackendOptions): Promise<LocalSqliteBackend>;
|
|
82
108
|
|
|
83
|
-
export { type LocalSqliteBackend, type LocalSqliteBackendOptions, createBetterSqliteExecutor, createLocalSqliteBackend };
|
|
109
|
+
export { type LocalSqliteBackend, type LocalSqliteBackendOptions, type LocalSqliteCapability, createBetterSqliteExecutor, createLocalSqliteBackend };
|
package/dist/sqlite/local.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Database } from 'better-sqlite3';
|
|
2
|
-
import { S as StorageBackend } from '../backend-
|
|
3
|
-
import { a as SqliteCapability, S as SqliteBackendOptions, b as SqliteExecutor } from '../backend-
|
|
2
|
+
import { S as StorageBackend } from '../backend-DNzv8KSR.js';
|
|
3
|
+
import { a as SqliteCapability, S as SqliteBackendOptions, b as SqliteExecutor } from '../backend-CE3pM9-T.js';
|
|
4
4
|
import '@google-cloud/firestore';
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -17,8 +17,34 @@ import '@google-cloud/firestore';
|
|
|
17
17
|
* `journal_mode = WAL` and a `busy_timeout` applied; caller-provided
|
|
18
18
|
* databases are used as-is (only `busy_timeout` is set) since the caller
|
|
19
19
|
* owns their pragma configuration.
|
|
20
|
+
*
|
|
21
|
+
* ## Search capabilities
|
|
22
|
+
*
|
|
23
|
+
* On top of the shared SQLite capability set, the local backend declares
|
|
24
|
+
* `search.fullText` and `search.vector` (see `src/internal/sqlite-search.ts`
|
|
25
|
+
* for the mechanics):
|
|
26
|
+
*
|
|
27
|
+
* - **Full-text search** is backed by one FTS5 table per graph table,
|
|
28
|
+
* kept in sync by pure-SQL triggers installed with the table's DDL.
|
|
29
|
+
* Because the triggers live in the database file, writes from ANY
|
|
30
|
+
* process or connection stay indexed. The trade-off is a per-write
|
|
31
|
+
* overhead (text extraction via `json_tree` + an FTS index update) on
|
|
32
|
+
* every insert/update/delete.
|
|
33
|
+
* - **Vector search** is a brute-force scan scored by a deterministic
|
|
34
|
+
* scalar UDF registered on this connection. UDFs are connection-local:
|
|
35
|
+
* `findNearest` only works through a backend created by this factory
|
|
36
|
+
* (other connections to the same file can read/write normally — only
|
|
37
|
+
* vector *search* needs the UDF).
|
|
20
38
|
*/
|
|
21
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Capability union for the local better-sqlite3 backend: everything the
|
|
42
|
+
* shared SQLite edition declares, plus native FTS5 full-text search and
|
|
43
|
+
* brute-force vector search. `search.geo` stays out — there is no geo
|
|
44
|
+
* index in stock SQLite, and a UDF-scored scan without a haversine
|
|
45
|
+
* contract pinned by Firestore parity tests would be guesswork.
|
|
46
|
+
*/
|
|
47
|
+
type LocalSqliteCapability = SqliteCapability | 'search.fullText' | 'search.vector';
|
|
22
48
|
interface LocalSqliteBackendOptions extends SqliteBackendOptions {
|
|
23
49
|
/** Root graph table name. Defaults to `'firegraph'`. */
|
|
24
50
|
tableName?: string;
|
|
@@ -41,7 +67,7 @@ interface LocalSqliteBackendOptions extends SqliteBackendOptions {
|
|
|
41
67
|
}
|
|
42
68
|
interface LocalSqliteBackend {
|
|
43
69
|
/** The graph storage backend — pass to `createGraphClient`. */
|
|
44
|
-
backend: StorageBackend<
|
|
70
|
+
backend: StorageBackend<LocalSqliteCapability>;
|
|
45
71
|
/** The underlying better-sqlite3 database, for raw access. */
|
|
46
72
|
db: Database;
|
|
47
73
|
/**
|
|
@@ -70,7 +96,7 @@ declare function createBetterSqliteExecutor(db: Database): SqliteExecutor;
|
|
|
70
96
|
*
|
|
71
97
|
* const { backend, close } = await createLocalSqliteBackend('./graph.db');
|
|
72
98
|
* const client = createGraphClient(backend);
|
|
73
|
-
* // ... use the client ...
|
|
99
|
+
* // ... use the client — including fullTextSearch() and findNearest() ...
|
|
74
100
|
* close();
|
|
75
101
|
* ```
|
|
76
102
|
*
|
|
@@ -80,4 +106,4 @@ declare function createBetterSqliteExecutor(db: Database): SqliteExecutor;
|
|
|
80
106
|
*/
|
|
81
107
|
declare function createLocalSqliteBackend(pathOrDb: string | Database, options?: LocalSqliteBackendOptions): Promise<LocalSqliteBackend>;
|
|
82
108
|
|
|
83
|
-
export { type LocalSqliteBackend, type LocalSqliteBackendOptions, createBetterSqliteExecutor, createLocalSqliteBackend };
|
|
109
|
+
export { type LocalSqliteBackend, type LocalSqliteBackendOptions, type LocalSqliteCapability, createBetterSqliteExecutor, createLocalSqliteBackend };
|
package/dist/sqlite/local.js
CHANGED
|
@@ -1,15 +1,317 @@
|
|
|
1
1
|
import {
|
|
2
|
+
catalogTableName,
|
|
2
3
|
createSqliteBackend
|
|
3
|
-
} from "../chunk-
|
|
4
|
-
import
|
|
4
|
+
} from "../chunk-5JBNLH5W.js";
|
|
5
|
+
import {
|
|
6
|
+
compileFilterConditions,
|
|
7
|
+
quoteIdent,
|
|
8
|
+
rowToRecord,
|
|
9
|
+
validateJsonPathKey,
|
|
10
|
+
validateTableName
|
|
11
|
+
} from "../chunk-NZVSLWNY.js";
|
|
5
12
|
import "../chunk-2DHMNTV6.js";
|
|
6
|
-
import
|
|
13
|
+
import {
|
|
14
|
+
createCapabilities
|
|
15
|
+
} from "../chunk-PWIO46RT.js";
|
|
7
16
|
import "../chunk-NGAJCALM.js";
|
|
8
17
|
import {
|
|
9
18
|
FiregraphError
|
|
10
19
|
} from "../chunk-SIHE4UY4.js";
|
|
11
20
|
import "../chunk-EQJUUVFG.js";
|
|
12
21
|
|
|
22
|
+
// src/internal/sqlite-search.ts
|
|
23
|
+
var VECTOR_DISTANCE_UDF = "firegraph_vector_distance";
|
|
24
|
+
var DISTANCE_ALIAS = "__fg_distance";
|
|
25
|
+
var BACKEND_ERR_LABEL = "SQLite backend";
|
|
26
|
+
var ENVELOPE_FIELDS = /* @__PURE__ */ new Set([
|
|
27
|
+
"aType",
|
|
28
|
+
"aUid",
|
|
29
|
+
"axbType",
|
|
30
|
+
"bType",
|
|
31
|
+
"bUid",
|
|
32
|
+
"createdAt",
|
|
33
|
+
"updatedAt",
|
|
34
|
+
"v"
|
|
35
|
+
]);
|
|
36
|
+
function ftsTableName(table) {
|
|
37
|
+
return `${table}_fts`;
|
|
38
|
+
}
|
|
39
|
+
function ftsMapTableName(table) {
|
|
40
|
+
return `${table}_fts_map`;
|
|
41
|
+
}
|
|
42
|
+
function textExtractionExpr(dataRef) {
|
|
43
|
+
return `(SELECT coalesce(group_concat("value", ' '), '') FROM json_tree(coalesce(${dataRef}, '{}')) WHERE "type" = 'text')`;
|
|
44
|
+
}
|
|
45
|
+
function buildFtsDDL(table) {
|
|
46
|
+
const t = quoteIdent(table);
|
|
47
|
+
const fts = quoteIdent(ftsTableName(table));
|
|
48
|
+
const map = quoteIdent(ftsMapTableName(table));
|
|
49
|
+
const mappedId = `(SELECT "id" FROM ${map} WHERE "doc_id" = new."doc_id")`;
|
|
50
|
+
const reindexBody = ` INSERT INTO ${map} ("doc_id") SELECT new."doc_id" WHERE NOT EXISTS (SELECT 1 FROM ${map} WHERE "doc_id" = new."doc_id");
|
|
51
|
+
DELETE FROM ${fts} WHERE rowid = ${mappedId};
|
|
52
|
+
INSERT INTO ${fts} (rowid, "text") VALUES (${mappedId}, ${textExtractionExpr('new."data"')});
|
|
53
|
+
`;
|
|
54
|
+
return [
|
|
55
|
+
`CREATE TABLE IF NOT EXISTS ${map} (
|
|
56
|
+
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
57
|
+
"doc_id" TEXT NOT NULL UNIQUE
|
|
58
|
+
)`,
|
|
59
|
+
`CREATE VIRTUAL TABLE IF NOT EXISTS ${fts} USING fts5("text")`,
|
|
60
|
+
`CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${table}_fts_ai`)} AFTER INSERT ON ${t} BEGIN
|
|
61
|
+
${reindexBody}END`,
|
|
62
|
+
`CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${table}_fts_au`)} AFTER UPDATE ON ${t} BEGIN
|
|
63
|
+
${reindexBody}END`,
|
|
64
|
+
`CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${table}_fts_ad`)} AFTER DELETE ON ${t} BEGIN
|
|
65
|
+
DELETE FROM ${fts} WHERE rowid = (SELECT "id" FROM ${map} WHERE "doc_id" = old."doc_id");
|
|
66
|
+
DELETE FROM ${map} WHERE "doc_id" = old."doc_id";
|
|
67
|
+
END`
|
|
68
|
+
];
|
|
69
|
+
}
|
|
70
|
+
function buildFtsSyncStatements(table) {
|
|
71
|
+
const t = quoteIdent(table);
|
|
72
|
+
const fts = quoteIdent(ftsTableName(table));
|
|
73
|
+
const map = quoteIdent(ftsMapTableName(table));
|
|
74
|
+
return [
|
|
75
|
+
`DELETE FROM ${fts} WHERE rowid IN (
|
|
76
|
+
SELECT m."id" FROM ${map} m LEFT JOIN ${t} t ON t."doc_id" = m."doc_id"
|
|
77
|
+
WHERE t."doc_id" IS NULL
|
|
78
|
+
)`,
|
|
79
|
+
`DELETE FROM ${map} WHERE "doc_id" NOT IN (SELECT "doc_id" FROM ${t})`,
|
|
80
|
+
`INSERT OR IGNORE INTO ${map} ("doc_id") SELECT "doc_id" FROM ${t}`,
|
|
81
|
+
`INSERT INTO ${fts} (rowid, "text")
|
|
82
|
+
SELECT m."id", ${textExtractionExpr('t."data"')}
|
|
83
|
+
FROM ${t} t JOIN ${map} m ON m."doc_id" = t."doc_id"
|
|
84
|
+
WHERE m."id" NOT IN (SELECT rowid FROM ${fts})`
|
|
85
|
+
];
|
|
86
|
+
}
|
|
87
|
+
function buildLocalSearchDDL(table) {
|
|
88
|
+
return [...buildFtsDDL(table), ...buildFtsSyncStatements(table)];
|
|
89
|
+
}
|
|
90
|
+
function normalizeVectorFieldPath(label, field) {
|
|
91
|
+
if (ENVELOPE_FIELDS.has(field)) {
|
|
92
|
+
throw new FiregraphError(
|
|
93
|
+
`findNearest(): ${label} '${field}' is a built-in envelope field \u2014 vectors must live under \`data.*\`. Use a path like 'data.${field}' if you really meant a nested data field.`,
|
|
94
|
+
"INVALID_QUERY"
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
if (field === "data" || field.startsWith("data.")) return field;
|
|
98
|
+
return `data.${field}`;
|
|
99
|
+
}
|
|
100
|
+
function normalizeFullTextFieldPath(field) {
|
|
101
|
+
if (ENVELOPE_FIELDS.has(field)) {
|
|
102
|
+
throw new FiregraphError(
|
|
103
|
+
`fullTextSearch(): field '${field}' is a built-in envelope field \u2014 text-indexed fields must live under \`data.*\`. Use a path like 'data.${field}' if you really meant a nested data field.`,
|
|
104
|
+
"INVALID_QUERY"
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
if (field === "data" || field.startsWith("data.")) return field;
|
|
108
|
+
return `data.${field}`;
|
|
109
|
+
}
|
|
110
|
+
function buildSearchFilters(params) {
|
|
111
|
+
const filters = [];
|
|
112
|
+
if (params.aType) filters.push({ field: "aType", op: "==", value: params.aType });
|
|
113
|
+
if (params.axbType) filters.push({ field: "axbType", op: "==", value: params.axbType });
|
|
114
|
+
if (params.bType) filters.push({ field: "bType", op: "==", value: params.bType });
|
|
115
|
+
for (const clause of params.where ?? []) {
|
|
116
|
+
const field = ENVELOPE_FIELDS.has(clause.field) || clause.field.startsWith("data.") ? clause.field : `data.${clause.field}`;
|
|
117
|
+
filters.push({ field, op: clause.op, value: clause.value });
|
|
118
|
+
}
|
|
119
|
+
return filters;
|
|
120
|
+
}
|
|
121
|
+
function compileFullTextSearch(table, params) {
|
|
122
|
+
if (typeof params.query !== "string" || params.query.length === 0) {
|
|
123
|
+
throw new FiregraphError(
|
|
124
|
+
"fullTextSearch(): query must be a non-empty string.",
|
|
125
|
+
"INVALID_QUERY"
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
if (!Number.isInteger(params.limit) || params.limit <= 0) {
|
|
129
|
+
throw new FiregraphError(
|
|
130
|
+
`fullTextSearch(): limit must be a positive integer (got ${params.limit}).`,
|
|
131
|
+
"INVALID_QUERY"
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
const normalizedFields = params.fields?.map((f) => normalizeFullTextFieldPath(f));
|
|
135
|
+
if (normalizedFields !== void 0 && normalizedFields.length > 0) {
|
|
136
|
+
throw new FiregraphError(
|
|
137
|
+
"fullTextSearch(): the `fields` option is not yet supported \u2014 the local SQLite FTS index stores one combined text column per record. Omit `fields` to search all string values.",
|
|
138
|
+
"INVALID_QUERY"
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
const t = quoteIdent(table);
|
|
142
|
+
const fts = quoteIdent(ftsTableName(table));
|
|
143
|
+
const map = quoteIdent(ftsMapTableName(table));
|
|
144
|
+
const sqlParams = [params.query];
|
|
145
|
+
const conditions = [`${fts} MATCH ?`];
|
|
146
|
+
conditions.push(...compileFilterConditions(buildSearchFilters(params), sqlParams));
|
|
147
|
+
sqlParams.push(params.limit);
|
|
148
|
+
const sql = `SELECT ${t}.* FROM ${fts} JOIN ${map} ON ${map}."id" = ${fts}.rowid JOIN ${t} ON ${t}."doc_id" = ${map}."doc_id" WHERE ${conditions.join(" AND ")} ORDER BY bm25(${fts}) ASC, ${t}."doc_id" ASC LIMIT ?`;
|
|
149
|
+
return { sql, params: sqlParams };
|
|
150
|
+
}
|
|
151
|
+
var DISTANCE_MEASURES = /* @__PURE__ */ new Set(["EUCLIDEAN", "COSINE", "DOT_PRODUCT"]);
|
|
152
|
+
function toNumberArray(qv) {
|
|
153
|
+
if (Array.isArray(qv)) return qv;
|
|
154
|
+
if (typeof qv.toArray === "function") {
|
|
155
|
+
return qv.toArray();
|
|
156
|
+
}
|
|
157
|
+
throw new FiregraphError(
|
|
158
|
+
"findNearest(): queryVector must be a number[] or a Firestore VectorValue.",
|
|
159
|
+
"INVALID_QUERY"
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
function compileFindNearest(table, params) {
|
|
163
|
+
const vec = toNumberArray(params.queryVector);
|
|
164
|
+
if (vec.length === 0) {
|
|
165
|
+
throw new FiregraphError(
|
|
166
|
+
"findNearest(): queryVector is empty \u2014 at least one dimension is required.",
|
|
167
|
+
"INVALID_QUERY"
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
if (!Number.isInteger(params.limit) || params.limit <= 0 || params.limit > 1e3) {
|
|
171
|
+
throw new FiregraphError(
|
|
172
|
+
`findNearest(): limit must be a positive integer \u2264 1000 (got ${params.limit}).`,
|
|
173
|
+
"INVALID_QUERY"
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
if (!DISTANCE_MEASURES.has(params.distanceMeasure)) {
|
|
177
|
+
throw new FiregraphError(
|
|
178
|
+
`findNearest(): unknown distanceMeasure '${String(params.distanceMeasure)}' \u2014 expected EUCLIDEAN, COSINE, or DOT_PRODUCT.`,
|
|
179
|
+
"INVALID_QUERY"
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
const vectorField = normalizeVectorFieldPath("vectorField", params.vectorField);
|
|
183
|
+
let vectorExpr;
|
|
184
|
+
if (vectorField === "data") {
|
|
185
|
+
vectorExpr = '"data"';
|
|
186
|
+
} else {
|
|
187
|
+
const suffix = vectorField.slice("data.".length);
|
|
188
|
+
for (const part of suffix.split(".")) {
|
|
189
|
+
validateJsonPathKey(part, BACKEND_ERR_LABEL);
|
|
190
|
+
}
|
|
191
|
+
vectorExpr = `json_extract("data", '$.${suffix}')`;
|
|
192
|
+
}
|
|
193
|
+
let distancePath = null;
|
|
194
|
+
if (params.distanceResultField !== void 0) {
|
|
195
|
+
const normalized = normalizeVectorFieldPath("distanceResultField", params.distanceResultField);
|
|
196
|
+
if (normalized === "data") {
|
|
197
|
+
throw new FiregraphError(
|
|
198
|
+
`findNearest(): distanceResultField 'data' would replace the entire data payload \u2014 use a nested path like 'data.distance'.`,
|
|
199
|
+
"INVALID_QUERY"
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
distancePath = normalized.slice("data.".length).split(".");
|
|
203
|
+
for (const part of distancePath) {
|
|
204
|
+
validateJsonPathKey(part, BACKEND_ERR_LABEL);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
const sqlParams = [JSON.stringify(vec), params.distanceMeasure];
|
|
208
|
+
const conditions = compileFilterConditions(buildSearchFilters(params), sqlParams);
|
|
209
|
+
const innerWhere = conditions.length > 0 ? ` WHERE ${conditions.join(" AND ")}` : "";
|
|
210
|
+
const dist = quoteIdent(DISTANCE_ALIAS);
|
|
211
|
+
const descending = params.distanceMeasure === "DOT_PRODUCT";
|
|
212
|
+
let sql = `SELECT * FROM (SELECT *, ${VECTOR_DISTANCE_UDF}(${vectorExpr}, ?, ?) AS ${dist} FROM ${quoteIdent(table)}${innerWhere}) WHERE ${dist} IS NOT NULL`;
|
|
213
|
+
if (params.distanceThreshold !== void 0) {
|
|
214
|
+
sql += ` AND ${dist} ${descending ? ">=" : "<="} ?`;
|
|
215
|
+
sqlParams.push(params.distanceThreshold);
|
|
216
|
+
}
|
|
217
|
+
sql += ` ORDER BY ${dist} ${descending ? "DESC" : "ASC"}, "doc_id" ASC LIMIT ?`;
|
|
218
|
+
sqlParams.push(params.limit);
|
|
219
|
+
return { stmt: { sql, params: sqlParams }, distancePath };
|
|
220
|
+
}
|
|
221
|
+
var memoQueryJson = null;
|
|
222
|
+
var memoQueryVec = null;
|
|
223
|
+
function computeVectorDistance(storedJson, queryJson, measure) {
|
|
224
|
+
if (typeof storedJson !== "string" || typeof queryJson !== "string" || typeof measure !== "string") {
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
let query;
|
|
228
|
+
if (memoQueryJson === queryJson && memoQueryVec !== null) {
|
|
229
|
+
query = memoQueryVec;
|
|
230
|
+
} else {
|
|
231
|
+
let parsed;
|
|
232
|
+
try {
|
|
233
|
+
parsed = JSON.parse(queryJson);
|
|
234
|
+
} catch {
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
237
|
+
if (!Array.isArray(parsed)) return null;
|
|
238
|
+
query = parsed;
|
|
239
|
+
memoQueryJson = queryJson;
|
|
240
|
+
memoQueryVec = query;
|
|
241
|
+
}
|
|
242
|
+
let stored;
|
|
243
|
+
try {
|
|
244
|
+
stored = JSON.parse(storedJson);
|
|
245
|
+
} catch {
|
|
246
|
+
return null;
|
|
247
|
+
}
|
|
248
|
+
if (!Array.isArray(stored) || stored.length !== query.length) return null;
|
|
249
|
+
let dot = 0;
|
|
250
|
+
let sumSq = 0;
|
|
251
|
+
let normStored = 0;
|
|
252
|
+
let normQuery = 0;
|
|
253
|
+
for (let i = 0; i < query.length; i++) {
|
|
254
|
+
const a = stored[i];
|
|
255
|
+
const b = query[i];
|
|
256
|
+
if (typeof a !== "number" || !Number.isFinite(a)) return null;
|
|
257
|
+
if (typeof b !== "number" || !Number.isFinite(b)) return null;
|
|
258
|
+
dot += a * b;
|
|
259
|
+
const diff = a - b;
|
|
260
|
+
sumSq += diff * diff;
|
|
261
|
+
normStored += a * a;
|
|
262
|
+
normQuery += b * b;
|
|
263
|
+
}
|
|
264
|
+
let result;
|
|
265
|
+
switch (measure) {
|
|
266
|
+
case "EUCLIDEAN":
|
|
267
|
+
result = Math.sqrt(sumSq);
|
|
268
|
+
break;
|
|
269
|
+
case "COSINE": {
|
|
270
|
+
const denom = Math.sqrt(normStored) * Math.sqrt(normQuery);
|
|
271
|
+
if (denom === 0) return null;
|
|
272
|
+
result = 1 - dot / denom;
|
|
273
|
+
break;
|
|
274
|
+
}
|
|
275
|
+
case "DOT_PRODUCT":
|
|
276
|
+
result = dot;
|
|
277
|
+
break;
|
|
278
|
+
default:
|
|
279
|
+
return null;
|
|
280
|
+
}
|
|
281
|
+
return Number.isFinite(result) ? result : null;
|
|
282
|
+
}
|
|
283
|
+
function setDataPath(data, path, value) {
|
|
284
|
+
let cursor = data;
|
|
285
|
+
for (let i = 0; i < path.length - 1; i++) {
|
|
286
|
+
const key = path[i];
|
|
287
|
+
const next = cursor[key];
|
|
288
|
+
if (typeof next !== "object" || next === null || Array.isArray(next)) {
|
|
289
|
+
const created = {};
|
|
290
|
+
cursor[key] = created;
|
|
291
|
+
cursor = created;
|
|
292
|
+
} else {
|
|
293
|
+
cursor = next;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
cursor[path[path.length - 1]] = value;
|
|
297
|
+
}
|
|
298
|
+
function findOrphanedFtsTables(allTables, catalogTables, rootTable) {
|
|
299
|
+
const names = new Set(allTables);
|
|
300
|
+
const liveGraphTables = new Set(catalogTables);
|
|
301
|
+
const subgraphPrefix = `${rootTable}_g_`;
|
|
302
|
+
const orphans = [];
|
|
303
|
+
for (const name of names) {
|
|
304
|
+
let base = null;
|
|
305
|
+
if (name.endsWith("_fts_map")) base = name.slice(0, -"_fts_map".length);
|
|
306
|
+
else if (name.endsWith("_fts")) base = name.slice(0, -"_fts".length);
|
|
307
|
+
if (base === null || !base.startsWith(subgraphPrefix)) continue;
|
|
308
|
+
if (liveGraphTables.has(name)) continue;
|
|
309
|
+
if (names.has(base)) continue;
|
|
310
|
+
orphans.push(name);
|
|
311
|
+
}
|
|
312
|
+
return orphans.sort();
|
|
313
|
+
}
|
|
314
|
+
|
|
13
315
|
// src/sqlite/local.ts
|
|
14
316
|
function createBetterSqliteExecutor(db) {
|
|
15
317
|
return {
|
|
@@ -66,6 +368,118 @@ function applyPragmas(db, pragmas) {
|
|
|
66
368
|
db.pragma(`${key} = ${value}`);
|
|
67
369
|
}
|
|
68
370
|
}
|
|
371
|
+
function registerVectorUdf(db) {
|
|
372
|
+
try {
|
|
373
|
+
db.function(
|
|
374
|
+
VECTOR_DISTANCE_UDF,
|
|
375
|
+
{ deterministic: true },
|
|
376
|
+
(stored, query, measure) => computeVectorDistance(stored, query, measure)
|
|
377
|
+
);
|
|
378
|
+
} catch {
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
async function sweepOrphanedFtsArtifacts(executor, rootTable) {
|
|
382
|
+
const tableRows = await executor.all(
|
|
383
|
+
`SELECT "name" FROM sqlite_master WHERE "type" = 'table'`,
|
|
384
|
+
[]
|
|
385
|
+
);
|
|
386
|
+
const allTables = tableRows.map((r) => String(r.name));
|
|
387
|
+
const catalogRows = await executor.all(
|
|
388
|
+
`SELECT "table_name" FROM ${quoteIdent(catalogTableName(rootTable))}`,
|
|
389
|
+
[]
|
|
390
|
+
);
|
|
391
|
+
const catalogTables = catalogRows.map((r) => String(r.table_name));
|
|
392
|
+
for (const name of findOrphanedFtsTables(allTables, catalogTables, rootTable)) {
|
|
393
|
+
validateTableName(name);
|
|
394
|
+
await executor.run(`DROP TABLE IF EXISTS ${quoteIdent(name)}`, []);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
function wrapLocalSearchBackend(inner, executor, rootTable) {
|
|
398
|
+
const caps = /* @__PURE__ */ new Set([
|
|
399
|
+
...inner.capabilities.values(),
|
|
400
|
+
"search.fullText",
|
|
401
|
+
"search.vector"
|
|
402
|
+
]);
|
|
403
|
+
const healableTables = /* @__PURE__ */ new Set([
|
|
404
|
+
inner.collectionPath,
|
|
405
|
+
ftsTableName(inner.collectionPath),
|
|
406
|
+
ftsMapTableName(inner.collectionPath)
|
|
407
|
+
]);
|
|
408
|
+
const runWithSchema = async (op) => {
|
|
409
|
+
await inner.ensureReady();
|
|
410
|
+
try {
|
|
411
|
+
return await op();
|
|
412
|
+
} catch (err) {
|
|
413
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
414
|
+
const missing = /no such table: (\S+)/.exec(message)?.[1];
|
|
415
|
+
if (missing === void 0 || !healableTables.has(missing)) throw err;
|
|
416
|
+
await inner.ensureReady(true);
|
|
417
|
+
return op();
|
|
418
|
+
}
|
|
419
|
+
};
|
|
420
|
+
const wrapper = {
|
|
421
|
+
capabilities: createCapabilities(caps),
|
|
422
|
+
collectionPath: inner.collectionPath,
|
|
423
|
+
scopePath: inner.scopePath,
|
|
424
|
+
getDoc: (docId) => inner.getDoc(docId),
|
|
425
|
+
query: (filters, options) => inner.query(filters, options),
|
|
426
|
+
setDoc: (docId, record, mode) => inner.setDoc(docId, record, mode),
|
|
427
|
+
updateDoc: (docId, update) => inner.updateDoc(docId, update),
|
|
428
|
+
deleteDoc: (docId) => inner.deleteDoc(docId),
|
|
429
|
+
runTransaction: (fn) => inner.runTransaction(fn),
|
|
430
|
+
createBatch: () => inner.createBatch(),
|
|
431
|
+
subgraph: (parentNodeUid, name) => wrapLocalSearchBackend(inner.subgraph(parentNodeUid, name), executor, rootTable),
|
|
432
|
+
removeNodeCascade: async (uid, reader, options) => {
|
|
433
|
+
const result = await inner.removeNodeCascade(uid, reader, options);
|
|
434
|
+
if (result.errors.length === 0) {
|
|
435
|
+
await sweepOrphanedFtsArtifacts(executor, rootTable);
|
|
436
|
+
}
|
|
437
|
+
return result;
|
|
438
|
+
},
|
|
439
|
+
bulkRemoveEdges: (params, reader, options) => inner.bulkRemoveEdges(params, reader, options),
|
|
440
|
+
aggregate: (spec, filters) => inner.aggregate(spec, filters),
|
|
441
|
+
bulkDelete: (filters, options) => inner.bulkDelete(filters, options),
|
|
442
|
+
bulkUpdate: (filters, patch, options) => inner.bulkUpdate(filters, patch, options),
|
|
443
|
+
expand: (params) => inner.expand(params),
|
|
444
|
+
findEdgesProjected: (select, filters, options) => inner.findEdgesProjected(select, filters, options),
|
|
445
|
+
// `findEdgesGlobal` stays absent, same as the inner backend — each graph
|
|
446
|
+
// is its own table; there is no cross-table index.
|
|
447
|
+
async findNearest(params) {
|
|
448
|
+
const { stmt, distancePath } = compileFindNearest(inner.collectionPath, params);
|
|
449
|
+
const rows = await runWithSchema(() => executor.all(stmt.sql, stmt.params));
|
|
450
|
+
return rows.map((row) => {
|
|
451
|
+
const record = rowToRecord(row);
|
|
452
|
+
if (distancePath) {
|
|
453
|
+
const distance = row[DISTANCE_ALIAS];
|
|
454
|
+
setDataPath(
|
|
455
|
+
record.data,
|
|
456
|
+
distancePath,
|
|
457
|
+
typeof distance === "number" ? distance : Number(distance)
|
|
458
|
+
);
|
|
459
|
+
}
|
|
460
|
+
return record;
|
|
461
|
+
});
|
|
462
|
+
},
|
|
463
|
+
async fullTextSearch(params) {
|
|
464
|
+
const stmt = compileFullTextSearch(inner.collectionPath, params);
|
|
465
|
+
let rows;
|
|
466
|
+
try {
|
|
467
|
+
rows = await runWithSchema(() => executor.all(stmt.sql, stmt.params));
|
|
468
|
+
} catch (err) {
|
|
469
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
470
|
+
if (message.includes("fts5") || message.includes("unknown special query")) {
|
|
471
|
+
throw new FiregraphError(
|
|
472
|
+
`fullTextSearch(): invalid FTS5 query syntax \u2014 ${message}`,
|
|
473
|
+
"INVALID_QUERY"
|
|
474
|
+
);
|
|
475
|
+
}
|
|
476
|
+
throw err;
|
|
477
|
+
}
|
|
478
|
+
return rows.map(rowToRecord);
|
|
479
|
+
}
|
|
480
|
+
};
|
|
481
|
+
return wrapper;
|
|
482
|
+
}
|
|
69
483
|
async function createLocalSqliteBackend(pathOrDb, options = {}) {
|
|
70
484
|
const {
|
|
71
485
|
tableName = "firegraph",
|
|
@@ -102,7 +516,18 @@ async function createLocalSqliteBackend(pathOrDb, options = {}) {
|
|
|
102
516
|
if (pragmas) {
|
|
103
517
|
applyPragmas(db, pragmas);
|
|
104
518
|
}
|
|
105
|
-
|
|
519
|
+
registerVectorUdf(db);
|
|
520
|
+
const userExtraDDL = backendOptions.extraTableDDL;
|
|
521
|
+
const optionsWithSearch = {
|
|
522
|
+
...backendOptions,
|
|
523
|
+
extraTableDDL: (table) => [
|
|
524
|
+
...userExtraDDL ? userExtraDDL(table) : [],
|
|
525
|
+
...buildLocalSearchDDL(table)
|
|
526
|
+
]
|
|
527
|
+
};
|
|
528
|
+
const executor = createBetterSqliteExecutor(db);
|
|
529
|
+
const inner = createSqliteBackend(executor, tableName, optionsWithSearch);
|
|
530
|
+
const backend = wrapLocalSearchBackend(inner, executor, tableName);
|
|
106
531
|
let closed = false;
|
|
107
532
|
return {
|
|
108
533
|
backend,
|