chloedb 0.0.1 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +37 -5
- package/client.d.ts +103 -0
- package/client.js +619 -0
- package/common.d.ts +176 -0
- package/common.js +359 -0
- package/index.d.ts +235 -0
- package/index.js +594 -4
- package/nearby.js +279 -0
- package/package.json +45 -5
package/nearby.js
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
// Search near a device's user on the device, with a chloedb-server as the
|
|
2
|
+
// fallback: `openNearby(url, table, dir)`.
|
|
3
|
+
//
|
|
4
|
+
// The server builds a table's slice (`POST /v1/admin/slices`, or the
|
|
5
|
+
// command center's Slices view): a manifest and the rows of each map tile,
|
|
6
|
+
// or of each vector cluster. `Nearby` fetches the manifest, then the tiles
|
|
7
|
+
// around its user as the user moves (`moveTo`), or the clusters nearest
|
|
8
|
+
// the user's taste (`focus`), into a local database (`dir`). A search runs
|
|
9
|
+
// there and scores every row held exactly as the server would. When the
|
|
10
|
+
// k-th local score beats the most a row not held could score, the local
|
|
11
|
+
// answer is the server's and no request is made; otherwise the same query
|
|
12
|
+
// goes to the server (`POST /v1/query`). Each answer says which, and why.
|
|
13
|
+
//
|
|
14
|
+
// Before it trusts its slice it asks the server whether the slice is the
|
|
15
|
+
// current one (`GET /v1/slices/<table>/status`, every `statusEvery` ms): it
|
|
16
|
+
// fetches a newer one, and while the table has changed since the build it
|
|
17
|
+
// asks the server for every search (unless the slice is under `maxLag` ms
|
|
18
|
+
// old). It reports its counts to the server every `reportEvery` searches
|
|
19
|
+
// (`POST /v1/slices/<table>/usage`), which the command center shows.
|
|
20
|
+
//
|
|
21
|
+
// import { openNearby } from "chloedb";
|
|
22
|
+
//
|
|
23
|
+
// const near = await openNearby("http://127.0.0.1:7700", "places", "./slice");
|
|
24
|
+
// await near.moveTo(40.7484, -73.9857);
|
|
25
|
+
// const r = await near.search("pizza", { k: 10 });
|
|
26
|
+
// r.where, r.why; // "device" or "server", and why
|
|
27
|
+
// for (const row of r) console.log(row.name);
|
|
28
|
+
// await near.close();
|
|
29
|
+
//
|
|
30
|
+
// const tracks = await openNearby("http://127.0.0.1:7700", "tracks", "./tracks");
|
|
31
|
+
// await tracks.focus(taste, { rows: 20000 }); // the user's taste, a vector
|
|
32
|
+
// const s = await tracks.similar(taste, { k: 10 });
|
|
33
|
+
//
|
|
34
|
+
// A vector slice's device also fetches every row's codes (`codes.bin.gz`,
|
|
35
|
+
// one byte a dimension), which bound the rows it lacks tightly; when they
|
|
36
|
+
// leave its answer in doubt it fetches the few clusters they name, up to
|
|
37
|
+
// `refine` bytes, and proves it on the device.
|
|
38
|
+
|
|
39
|
+
import { Buffer } from "node:buffer";
|
|
40
|
+
import process from "node:process";
|
|
41
|
+
import { randomUUID } from "node:crypto";
|
|
42
|
+
|
|
43
|
+
import { Result, ChloeDBError, decode } from "./common.js";
|
|
44
|
+
|
|
45
|
+
const REASONS = ["proved", "no_match", "short", "bound", "stale"];
|
|
46
|
+
|
|
47
|
+
function empty() {
|
|
48
|
+
return { searches: 0, why: Object.fromEntries(REASONS.map((w) => [w, 0])), device_ms: 0, cloud_ms: 0 };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** A search's answer, the device's or the server's, and where it was
|
|
52
|
+
* answered: `where` is "device" or "server"; `why` is "proved" or
|
|
53
|
+
* "no_match" (answered on the device), or "short" (the device held fewer
|
|
54
|
+
* than k matching rows), "bound" (a row beyond the tiles or clusters held
|
|
55
|
+
* could have scored higher) or "stale" (the table changed since the slice
|
|
56
|
+
* was built). `ask` is the body of the request (`POST /v1/query`) that
|
|
57
|
+
* asks the server for it. */
|
|
58
|
+
export class NearbyResult extends Result {
|
|
59
|
+
get onDevice() {
|
|
60
|
+
return this.where === "device";
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** A device's slice of a table on a chloedb-server; see the module. Make
|
|
65
|
+
* one with `openNearby`. */
|
|
66
|
+
export class Nearby {
|
|
67
|
+
#native;
|
|
68
|
+
#dev = null;
|
|
69
|
+
#pending = empty();
|
|
70
|
+
#statusAt = 0;
|
|
71
|
+
|
|
72
|
+
constructor(native, url, table, dir, { token, reach = 2, timeout = 30_000, statusEvery = 30_000, maxLag = 0, reportEvery = 20, device, codes = true, fetch: f } = {}) {
|
|
73
|
+
this.#native = native;
|
|
74
|
+
this.url = url.replace(/\/+$/, "");
|
|
75
|
+
this.table = table;
|
|
76
|
+
this.dir = dir;
|
|
77
|
+
this.token = token ?? process.env.CHLOEDB_TOKEN ?? process.env.STRATA_TOKEN;
|
|
78
|
+
this.reach = reach;
|
|
79
|
+
this.timeout = timeout;
|
|
80
|
+
this.statusEvery = statusEvery;
|
|
81
|
+
this.maxLag = maxLag;
|
|
82
|
+
this.reportEvery = reportEvery;
|
|
83
|
+
this.codes = codes;
|
|
84
|
+
this.device = device ?? `js-${randomUUID().slice(0, 12)}`;
|
|
85
|
+
this.fetch = f ?? globalThis.fetch;
|
|
86
|
+
this.position = null;
|
|
87
|
+
this.taste = null;
|
|
88
|
+
this.info = null;
|
|
89
|
+
this.stats = { searches: 0, onDevice: 0, fetchedTiles: 0, fetchedBytes: 0 };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// -------------------------------------------------------------- HTTP
|
|
93
|
+
|
|
94
|
+
async #request(method, path, body) {
|
|
95
|
+
const headers = body === undefined ? {} : { "content-type": "application/json" };
|
|
96
|
+
if (this.token) headers.authorization = `Bearer ${this.token}`;
|
|
97
|
+
const r = await this.fetch(this.url + path, { method, headers, body, signal: AbortSignal.timeout(this.timeout) });
|
|
98
|
+
const bytes = Buffer.from(await r.arrayBuffer());
|
|
99
|
+
if (!r.ok) {
|
|
100
|
+
let text = bytes.toString();
|
|
101
|
+
try {
|
|
102
|
+
text = JSON.parse(text).error ?? text;
|
|
103
|
+
} catch {}
|
|
104
|
+
throw new ChloeDBError(`${method} ${path}: ${r.status}: ${text}`, { status: r.status });
|
|
105
|
+
}
|
|
106
|
+
return bytes;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
get #d() {
|
|
110
|
+
if (!this.#dev) throw new ChloeDBError("the slice is closed");
|
|
111
|
+
return this.#dev;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
#slice(rel) {
|
|
115
|
+
return this.#request("GET", `/v1/slices/${encodeURIComponent(this.table)}/${rel}`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// -------------------------------------------------------------- the slice
|
|
119
|
+
|
|
120
|
+
/** Fetches the manifest and opens the device's database (`openNearby`
|
|
121
|
+
* calls it). */
|
|
122
|
+
async load() {
|
|
123
|
+
const text = (await this.#slice("manifest.json")).toString();
|
|
124
|
+
this.stats.fetchedBytes += text.length;
|
|
125
|
+
this.#dev?.close();
|
|
126
|
+
this.#dev = await this.#native.NativeSlice.create(this.dir, text);
|
|
127
|
+
this.info = JSON.parse(this.#dev.info());
|
|
128
|
+
if (this.codes && this.info.codes_bytes) {
|
|
129
|
+
const data = await this.#slice("codes.bin.gz");
|
|
130
|
+
await this.#dev.addCodes(data);
|
|
131
|
+
this.stats.fetchedBytes += data.length;
|
|
132
|
+
}
|
|
133
|
+
this.#statusAt = Date.now();
|
|
134
|
+
if (this.position) await this.moveTo(...this.position);
|
|
135
|
+
if (this.taste) await this.focus(...this.taste);
|
|
136
|
+
return this;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Asks the server whether the slice is the current one: fetches a newer
|
|
140
|
+
* one (and the tiles around the user again), and marks the device stale
|
|
141
|
+
* while the table has changed since the build. Answers the server's
|
|
142
|
+
* status. */
|
|
143
|
+
async check() {
|
|
144
|
+
let st = JSON.parse((await this.#slice("status")).toString());
|
|
145
|
+
this.#statusAt = Date.now();
|
|
146
|
+
if (st.version !== this.info.version) {
|
|
147
|
+
await this.load();
|
|
148
|
+
st = JSON.parse((await this.#slice("status")).toString());
|
|
149
|
+
}
|
|
150
|
+
const lag = Date.now() - (this.info.built_ms ?? 0);
|
|
151
|
+
this.#d.setStale(!st.fresh && lag > this.maxLag);
|
|
152
|
+
return st;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Moves the user: fetches the tiles of the square around it the device
|
|
156
|
+
* lacks (nearest first) and drops those far behind. Answers what it did:
|
|
157
|
+
* {fetched, dropped, bytes}. */
|
|
158
|
+
async moveTo(lat, lon, { reach } = {}) {
|
|
159
|
+
if (reach !== undefined) this.reach = reach;
|
|
160
|
+
this.position = [lat, lon];
|
|
161
|
+
return this.#apply(JSON.parse(this.#d.plan(lat, lon, this.reach)));
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** For a vector slice: holds the clusters whose rows could score highest
|
|
165
|
+
* for `taste` (a vector, such as the user's history summed), as many as
|
|
166
|
+
* fit in `rows` rows, fetching those it lacks (best first) and dropping
|
|
167
|
+
* those that fell well behind. Answers what it did: {fetched, dropped,
|
|
168
|
+
* bytes}. */
|
|
169
|
+
async focus(taste, { rows = 20_000 } = {}) {
|
|
170
|
+
taste = Array.from(taste, Number);
|
|
171
|
+
this.taste = [taste, { rows }];
|
|
172
|
+
return this.#apply(JSON.parse(this.#d.planVector(taste, rows)));
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
async #apply(plan) {
|
|
176
|
+
for (const [x, y] of plan.drop) await this.#d.dropTile(x, y);
|
|
177
|
+
let bytes = 0;
|
|
178
|
+
for (const [x, y, path] of plan.fetch) {
|
|
179
|
+
const data = await this.#slice(path);
|
|
180
|
+
await this.#d.addTile(x, y, data);
|
|
181
|
+
bytes += data.length;
|
|
182
|
+
}
|
|
183
|
+
this.stats.fetchedTiles += plan.fetch.length;
|
|
184
|
+
this.stats.fetchedBytes += bytes;
|
|
185
|
+
return { fetched: plan.fetch.length, dropped: plan.drop.length, bytes };
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** The tiles held, as [x, y, rows]. */
|
|
189
|
+
held() {
|
|
190
|
+
return JSON.parse(this.#d.held());
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// -------------------------------------------------------------- searching
|
|
194
|
+
|
|
195
|
+
/** The `k` best places for `q` near the user (or near `lat`, `lon`),
|
|
196
|
+
* ranked by BM25 faded by distance (a place `d0` metres away counts
|
|
197
|
+
* half): the device's answer when it can prove it is the server's, else
|
|
198
|
+
* the server's. */
|
|
199
|
+
async search(q, { k = 10, d0 = 1000, lat, lon } = {}) {
|
|
200
|
+
if (lat === undefined || lon === undefined) {
|
|
201
|
+
if (!this.position) throw new ChloeDBError("moveTo(lat, lon) first, or pass lat and lon");
|
|
202
|
+
[lat, lon] = this.position;
|
|
203
|
+
}
|
|
204
|
+
return this.#answer(() => this.#d.search(q, lat, lon, k, d0));
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** For a vector slice: the `k` rows most similar to `vector` (by dot
|
|
208
|
+
* product, as `similarity` ranks them): the device's answer when it can
|
|
209
|
+
* prove it is the server's, else the server's, which re-scores every row
|
|
210
|
+
* (an exact search). When the device's codes name clusters that could
|
|
211
|
+
* still beat its k-th, it fetches them if they come to at most `refine`
|
|
212
|
+
* bytes (0: never), and then proves its answer. */
|
|
213
|
+
async similar(vector, { k = 10, refine = 2_000_000 } = {}) {
|
|
214
|
+
vector = Array.from(vector, Number);
|
|
215
|
+
return this.#answer(() => this.#d.searchVector(vector, k), refine);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
async #answer(local, refine = 0) {
|
|
219
|
+
if (this.statusEvery && Date.now() - this.#statusAt >= this.statusEvery) await this.check();
|
|
220
|
+
let t0 = performance.now();
|
|
221
|
+
let a = decode(await local());
|
|
222
|
+
let deviceMs = performance.now() - t0;
|
|
223
|
+
const need = a.need ?? [];
|
|
224
|
+
let fetched = 0;
|
|
225
|
+
if (!a.on_device && a.why === "bound" && need.length && need.reduce((n, t) => n + t[2], 0) <= refine) {
|
|
226
|
+
await this.#apply({ fetch: need.map(([x, y, b]) => [x, y, `tiles/${x}_${y}.jsonl.gz`, b]), drop: [] });
|
|
227
|
+
fetched = need.length;
|
|
228
|
+
t0 = performance.now();
|
|
229
|
+
a = decode(await local());
|
|
230
|
+
deviceMs += performance.now() - t0;
|
|
231
|
+
}
|
|
232
|
+
let r;
|
|
233
|
+
let serverMs = null;
|
|
234
|
+
if (a.on_device) {
|
|
235
|
+
r = new NearbyResult(a);
|
|
236
|
+
r.where = "device";
|
|
237
|
+
} else {
|
|
238
|
+
t0 = performance.now();
|
|
239
|
+
// The body as the device wrote it: its floats stay floats.
|
|
240
|
+
r = new NearbyResult(decode((await this.#request("POST", "/v1/query", a.ask)).toString()));
|
|
241
|
+
serverMs = performance.now() - t0;
|
|
242
|
+
r.where = "server";
|
|
243
|
+
}
|
|
244
|
+
Object.assign(r, { why: a.why, ask: a.ask, proved: a.proved, stale: a.stale, bound: a.bound, edgeM: a.edge_m, deviceMs, serverMs, fetched });
|
|
245
|
+
this.#note(r);
|
|
246
|
+
return r;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// -------------------------------------------------------------- reporting
|
|
250
|
+
|
|
251
|
+
#note(r) {
|
|
252
|
+
this.stats.searches += 1;
|
|
253
|
+
if (r.where === "device") this.stats.onDevice += 1;
|
|
254
|
+
const p = this.#pending;
|
|
255
|
+
p.searches += 1;
|
|
256
|
+
p.why[r.why] += 1;
|
|
257
|
+
p.device_ms += r.deviceMs;
|
|
258
|
+
p.cloud_ms += r.serverMs ?? 0;
|
|
259
|
+
if (this.reportEvery && p.searches >= this.reportEvery) this.report().catch(() => {});
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** Sends the counts since the last report to the server; they are best
|
|
263
|
+
* effort, so a failure is ignored. */
|
|
264
|
+
async report() {
|
|
265
|
+
const p = this.#pending;
|
|
266
|
+
this.#pending = empty();
|
|
267
|
+
if (!p.searches) return;
|
|
268
|
+
try {
|
|
269
|
+
await this.#request("POST", `/v1/slices/${encodeURIComponent(this.table)}/usage`, JSON.stringify({ device: this.device, ...p }));
|
|
270
|
+
} catch {}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** Reports what is left and closes the device's database. */
|
|
274
|
+
async close() {
|
|
275
|
+
await this.report();
|
|
276
|
+
this.#dev?.close();
|
|
277
|
+
this.#dev = null;
|
|
278
|
+
}
|
|
279
|
+
}
|
package/package.json
CHANGED
|
@@ -1,11 +1,51 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "chloedb",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.3.5",
|
|
4
|
+
"description": "ChloeDB from JavaScript: a search database with SQL, opened in-process from its file (Node, Bun, Deno) or over HTTP from any runtime with fetch",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
|
-
"repository": { "type": "git", "url": "git+https://github.com/chloetech-ai/chloedb.git", "directory": "packaging/chloedb/npm" },
|
|
7
|
-
"homepage": "https://github.com/chloetech-ai/strata",
|
|
8
6
|
"type": "module",
|
|
9
7
|
"main": "./index.js",
|
|
10
|
-
"
|
|
8
|
+
"types": "./index.d.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"types": "./index.d.ts",
|
|
12
|
+
"browser": "./client.js",
|
|
13
|
+
"workerd": "./client.js",
|
|
14
|
+
"default": "./index.js"
|
|
15
|
+
},
|
|
16
|
+
"./client": {
|
|
17
|
+
"types": "./client.d.ts",
|
|
18
|
+
"default": "./client.js"
|
|
19
|
+
},
|
|
20
|
+
"./package.json": "./package.json"
|
|
21
|
+
},
|
|
22
|
+
"files": [
|
|
23
|
+
"index.js",
|
|
24
|
+
"index.d.ts",
|
|
25
|
+
"client.js",
|
|
26
|
+
"client.d.ts",
|
|
27
|
+
"common.js",
|
|
28
|
+
"common.d.ts",
|
|
29
|
+
"nearby.js",
|
|
30
|
+
"README.md",
|
|
31
|
+
"LICENSE"
|
|
32
|
+
],
|
|
33
|
+
"engines": {
|
|
34
|
+
"node": ">=18"
|
|
35
|
+
},
|
|
36
|
+
"repository": {
|
|
37
|
+
"type": "git",
|
|
38
|
+
"url": "https://github.com/chloetech-ai/chloedb",
|
|
39
|
+
"directory": "chloedb-js"
|
|
40
|
+
},
|
|
41
|
+
"scripts": {},
|
|
42
|
+
"optionalDependencies": {
|
|
43
|
+
"@chloedb/linux-x64-gnu": "0.3.5",
|
|
44
|
+
"@chloedb/linux-arm64-gnu": "0.3.5",
|
|
45
|
+
"@chloedb/darwin-arm64": "0.3.5",
|
|
46
|
+
"@chloedb/darwin-x64": "0.3.5"
|
|
47
|
+
},
|
|
48
|
+
"publishConfig": {
|
|
49
|
+
"access": "public"
|
|
50
|
+
}
|
|
11
51
|
}
|