chloedb 0.0.1 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/nearby.js ADDED
@@ -0,0 +1,279 @@
1
+ // Search near a device's user on the device, with a chloedb-server as the
2
+ // fallback: `openNearby(url, table, dir)`.
3
+ //
4
+ // The server builds a table's slice (`POST /v1/admin/slices`, or the
5
+ // command center's Slices view): a manifest and the rows of each map tile,
6
+ // or of each vector cluster. `Nearby` fetches the manifest, then the tiles
7
+ // around its user as the user moves (`moveTo`), or the clusters nearest
8
+ // the user's taste (`focus`), into a local database (`dir`). A search runs
9
+ // there and scores every row held exactly as the server would. When the
10
+ // k-th local score beats the most a row not held could score, the local
11
+ // answer is the server's and no request is made; otherwise the same query
12
+ // goes to the server (`POST /v1/query`). Each answer says which, and why.
13
+ //
14
+ // Before it trusts its slice it asks the server whether the slice is the
15
+ // current one (`GET /v1/slices/<table>/status`, every `statusEvery` ms): it
16
+ // fetches a newer one, and while the table has changed since the build it
17
+ // asks the server for every search (unless the slice is under `maxLag` ms
18
+ // old). It reports its counts to the server every `reportEvery` searches
19
+ // (`POST /v1/slices/<table>/usage`), which the command center shows.
20
+ //
21
+ // import { openNearby } from "chloedb";
22
+ //
23
+ // const near = await openNearby("http://127.0.0.1:7700", "places", "./slice");
24
+ // await near.moveTo(40.7484, -73.9857);
25
+ // const r = await near.search("pizza", { k: 10 });
26
+ // r.where, r.why; // "device" or "server", and why
27
+ // for (const row of r) console.log(row.name);
28
+ // await near.close();
29
+ //
30
+ // const tracks = await openNearby("http://127.0.0.1:7700", "tracks", "./tracks");
31
+ // await tracks.focus(taste, { rows: 20000 }); // the user's taste, a vector
32
+ // const s = await tracks.similar(taste, { k: 10 });
33
+ //
34
+ // A vector slice's device also fetches every row's codes (`codes.bin.gz`,
35
+ // one byte a dimension), which bound the rows it lacks tightly; when they
36
+ // leave its answer in doubt it fetches the few clusters they name, up to
37
+ // `refine` bytes, and proves it on the device.
38
+
39
+ import { Buffer } from "node:buffer";
40
+ import process from "node:process";
41
+ import { randomUUID } from "node:crypto";
42
+
43
+ import { Result, ChloeDBError, decode } from "./common.js";
44
+
45
+ const REASONS = ["proved", "no_match", "short", "bound", "stale"];
46
+
47
+ function empty() {
48
+ return { searches: 0, why: Object.fromEntries(REASONS.map((w) => [w, 0])), device_ms: 0, cloud_ms: 0 };
49
+ }
50
+
51
+ /** A search's answer, the device's or the server's, and where it was
52
+ * answered: `where` is "device" or "server"; `why` is "proved" or
53
+ * "no_match" (answered on the device), or "short" (the device held fewer
54
+ * than k matching rows), "bound" (a row beyond the tiles or clusters held
55
+ * could have scored higher) or "stale" (the table changed since the slice
56
+ * was built). `ask` is the body of the request (`POST /v1/query`) that
57
+ * asks the server for it. */
58
+ export class NearbyResult extends Result {
59
+ get onDevice() {
60
+ return this.where === "device";
61
+ }
62
+ }
63
+
64
+ /** A device's slice of a table on a chloedb-server; see the module. Make
65
+ * one with `openNearby`. */
66
+ export class Nearby {
67
+ #native;
68
+ #dev = null;
69
+ #pending = empty();
70
+ #statusAt = 0;
71
+
72
+ constructor(native, url, table, dir, { token, reach = 2, timeout = 30_000, statusEvery = 30_000, maxLag = 0, reportEvery = 20, device, codes = true, fetch: f } = {}) {
73
+ this.#native = native;
74
+ this.url = url.replace(/\/+$/, "");
75
+ this.table = table;
76
+ this.dir = dir;
77
+ this.token = token ?? process.env.CHLOEDB_TOKEN ?? process.env.STRATA_TOKEN;
78
+ this.reach = reach;
79
+ this.timeout = timeout;
80
+ this.statusEvery = statusEvery;
81
+ this.maxLag = maxLag;
82
+ this.reportEvery = reportEvery;
83
+ this.codes = codes;
84
+ this.device = device ?? `js-${randomUUID().slice(0, 12)}`;
85
+ this.fetch = f ?? globalThis.fetch;
86
+ this.position = null;
87
+ this.taste = null;
88
+ this.info = null;
89
+ this.stats = { searches: 0, onDevice: 0, fetchedTiles: 0, fetchedBytes: 0 };
90
+ }
91
+
92
+ // -------------------------------------------------------------- HTTP
93
+
94
+ async #request(method, path, body) {
95
+ const headers = body === undefined ? {} : { "content-type": "application/json" };
96
+ if (this.token) headers.authorization = `Bearer ${this.token}`;
97
+ const r = await this.fetch(this.url + path, { method, headers, body, signal: AbortSignal.timeout(this.timeout) });
98
+ const bytes = Buffer.from(await r.arrayBuffer());
99
+ if (!r.ok) {
100
+ let text = bytes.toString();
101
+ try {
102
+ text = JSON.parse(text).error ?? text;
103
+ } catch {}
104
+ throw new ChloeDBError(`${method} ${path}: ${r.status}: ${text}`, { status: r.status });
105
+ }
106
+ return bytes;
107
+ }
108
+
109
+ get #d() {
110
+ if (!this.#dev) throw new ChloeDBError("the slice is closed");
111
+ return this.#dev;
112
+ }
113
+
114
+ #slice(rel) {
115
+ return this.#request("GET", `/v1/slices/${encodeURIComponent(this.table)}/${rel}`);
116
+ }
117
+
118
+ // -------------------------------------------------------------- the slice
119
+
120
+ /** Fetches the manifest and opens the device's database (`openNearby`
121
+ * calls it). */
122
+ async load() {
123
+ const text = (await this.#slice("manifest.json")).toString();
124
+ this.stats.fetchedBytes += text.length;
125
+ this.#dev?.close();
126
+ this.#dev = await this.#native.NativeSlice.create(this.dir, text);
127
+ this.info = JSON.parse(this.#dev.info());
128
+ if (this.codes && this.info.codes_bytes) {
129
+ const data = await this.#slice("codes.bin.gz");
130
+ await this.#dev.addCodes(data);
131
+ this.stats.fetchedBytes += data.length;
132
+ }
133
+ this.#statusAt = Date.now();
134
+ if (this.position) await this.moveTo(...this.position);
135
+ if (this.taste) await this.focus(...this.taste);
136
+ return this;
137
+ }
138
+
139
+ /** Asks the server whether the slice is the current one: fetches a newer
140
+ * one (and the tiles around the user again), and marks the device stale
141
+ * while the table has changed since the build. Answers the server's
142
+ * status. */
143
+ async check() {
144
+ let st = JSON.parse((await this.#slice("status")).toString());
145
+ this.#statusAt = Date.now();
146
+ if (st.version !== this.info.version) {
147
+ await this.load();
148
+ st = JSON.parse((await this.#slice("status")).toString());
149
+ }
150
+ const lag = Date.now() - (this.info.built_ms ?? 0);
151
+ this.#d.setStale(!st.fresh && lag > this.maxLag);
152
+ return st;
153
+ }
154
+
155
+ /** Moves the user: fetches the tiles of the square around it the device
156
+ * lacks (nearest first) and drops those far behind. Answers what it did:
157
+ * {fetched, dropped, bytes}. */
158
+ async moveTo(lat, lon, { reach } = {}) {
159
+ if (reach !== undefined) this.reach = reach;
160
+ this.position = [lat, lon];
161
+ return this.#apply(JSON.parse(this.#d.plan(lat, lon, this.reach)));
162
+ }
163
+
164
+ /** For a vector slice: holds the clusters whose rows could score highest
165
+ * for `taste` (a vector, such as the user's history summed), as many as
166
+ * fit in `rows` rows, fetching those it lacks (best first) and dropping
167
+ * those that fell well behind. Answers what it did: {fetched, dropped,
168
+ * bytes}. */
169
+ async focus(taste, { rows = 20_000 } = {}) {
170
+ taste = Array.from(taste, Number);
171
+ this.taste = [taste, { rows }];
172
+ return this.#apply(JSON.parse(this.#d.planVector(taste, rows)));
173
+ }
174
+
175
+ async #apply(plan) {
176
+ for (const [x, y] of plan.drop) await this.#d.dropTile(x, y);
177
+ let bytes = 0;
178
+ for (const [x, y, path] of plan.fetch) {
179
+ const data = await this.#slice(path);
180
+ await this.#d.addTile(x, y, data);
181
+ bytes += data.length;
182
+ }
183
+ this.stats.fetchedTiles += plan.fetch.length;
184
+ this.stats.fetchedBytes += bytes;
185
+ return { fetched: plan.fetch.length, dropped: plan.drop.length, bytes };
186
+ }
187
+
188
+ /** The tiles held, as [x, y, rows]. */
189
+ held() {
190
+ return JSON.parse(this.#d.held());
191
+ }
192
+
193
+ // -------------------------------------------------------------- searching
194
+
195
+ /** The `k` best places for `q` near the user (or near `lat`, `lon`),
196
+ * ranked by BM25 faded by distance (a place `d0` metres away counts
197
+ * half): the device's answer when it can prove it is the server's, else
198
+ * the server's. */
199
+ async search(q, { k = 10, d0 = 1000, lat, lon } = {}) {
200
+ if (lat === undefined || lon === undefined) {
201
+ if (!this.position) throw new ChloeDBError("moveTo(lat, lon) first, or pass lat and lon");
202
+ [lat, lon] = this.position;
203
+ }
204
+ return this.#answer(() => this.#d.search(q, lat, lon, k, d0));
205
+ }
206
+
207
+ /** For a vector slice: the `k` rows most similar to `vector` (by dot
208
+ * product, as `similarity` ranks them): the device's answer when it can
209
+ * prove it is the server's, else the server's, which re-scores every row
210
+ * (an exact search). When the device's codes name clusters that could
211
+ * still beat its k-th, it fetches them if they come to at most `refine`
212
+ * bytes (0: never), and then proves its answer. */
213
+ async similar(vector, { k = 10, refine = 2_000_000 } = {}) {
214
+ vector = Array.from(vector, Number);
215
+ return this.#answer(() => this.#d.searchVector(vector, k), refine);
216
+ }
217
+
218
+ async #answer(local, refine = 0) {
219
+ if (this.statusEvery && Date.now() - this.#statusAt >= this.statusEvery) await this.check();
220
+ let t0 = performance.now();
221
+ let a = decode(await local());
222
+ let deviceMs = performance.now() - t0;
223
+ const need = a.need ?? [];
224
+ let fetched = 0;
225
+ if (!a.on_device && a.why === "bound" && need.length && need.reduce((n, t) => n + t[2], 0) <= refine) {
226
+ await this.#apply({ fetch: need.map(([x, y, b]) => [x, y, `tiles/${x}_${y}.jsonl.gz`, b]), drop: [] });
227
+ fetched = need.length;
228
+ t0 = performance.now();
229
+ a = decode(await local());
230
+ deviceMs += performance.now() - t0;
231
+ }
232
+ let r;
233
+ let serverMs = null;
234
+ if (a.on_device) {
235
+ r = new NearbyResult(a);
236
+ r.where = "device";
237
+ } else {
238
+ t0 = performance.now();
239
+ // The body as the device wrote it: its floats stay floats.
240
+ r = new NearbyResult(decode((await this.#request("POST", "/v1/query", a.ask)).toString()));
241
+ serverMs = performance.now() - t0;
242
+ r.where = "server";
243
+ }
244
+ Object.assign(r, { why: a.why, ask: a.ask, proved: a.proved, stale: a.stale, bound: a.bound, edgeM: a.edge_m, deviceMs, serverMs, fetched });
245
+ this.#note(r);
246
+ return r;
247
+ }
248
+
249
+ // -------------------------------------------------------------- reporting
250
+
251
+ #note(r) {
252
+ this.stats.searches += 1;
253
+ if (r.where === "device") this.stats.onDevice += 1;
254
+ const p = this.#pending;
255
+ p.searches += 1;
256
+ p.why[r.why] += 1;
257
+ p.device_ms += r.deviceMs;
258
+ p.cloud_ms += r.serverMs ?? 0;
259
+ if (this.reportEvery && p.searches >= this.reportEvery) this.report().catch(() => {});
260
+ }
261
+
262
+ /** Sends the counts since the last report to the server; they are best
263
+ * effort, so a failure is ignored. */
264
+ async report() {
265
+ const p = this.#pending;
266
+ this.#pending = empty();
267
+ if (!p.searches) return;
268
+ try {
269
+ await this.#request("POST", `/v1/slices/${encodeURIComponent(this.table)}/usage`, JSON.stringify({ device: this.device, ...p }));
270
+ } catch {}
271
+ }
272
+
273
+ /** Reports what is left and closes the device's database. */
274
+ async close() {
275
+ await this.report();
276
+ this.#dev?.close();
277
+ this.#dev = null;
278
+ }
279
+ }
package/package.json CHANGED
@@ -1,11 +1,51 @@
1
1
  {
2
2
  "name": "chloedb",
3
- "version": "0.0.1",
4
- "description": "Placeholder: holds the name for Strata. Install releases from https://github.com/chloetech-ai/strata/releases",
3
+ "version": "0.3.5",
4
+ "description": "ChloeDB from JavaScript: a search database with SQL, opened in-process from its file (Node, Bun, Deno) or over HTTP from any runtime with fetch",
5
5
  "license": "Apache-2.0",
6
- "repository": { "type": "git", "url": "git+https://github.com/chloetech-ai/chloedb.git", "directory": "packaging/chloedb/npm" },
7
- "homepage": "https://github.com/chloetech-ai/strata",
8
6
  "type": "module",
9
7
  "main": "./index.js",
10
- "files": ["index.js", "README.md"]
8
+ "types": "./index.d.ts",
9
+ "exports": {
10
+ ".": {
11
+ "types": "./index.d.ts",
12
+ "browser": "./client.js",
13
+ "workerd": "./client.js",
14
+ "default": "./index.js"
15
+ },
16
+ "./client": {
17
+ "types": "./client.d.ts",
18
+ "default": "./client.js"
19
+ },
20
+ "./package.json": "./package.json"
21
+ },
22
+ "files": [
23
+ "index.js",
24
+ "index.d.ts",
25
+ "client.js",
26
+ "client.d.ts",
27
+ "common.js",
28
+ "common.d.ts",
29
+ "nearby.js",
30
+ "README.md",
31
+ "LICENSE"
32
+ ],
33
+ "engines": {
34
+ "node": ">=18"
35
+ },
36
+ "repository": {
37
+ "type": "git",
38
+ "url": "https://github.com/chloetech-ai/chloedb",
39
+ "directory": "chloedb-js"
40
+ },
41
+ "scripts": {},
42
+ "optionalDependencies": {
43
+ "@chloedb/linux-x64-gnu": "0.3.5",
44
+ "@chloedb/linux-arm64-gnu": "0.3.5",
45
+ "@chloedb/darwin-arm64": "0.3.5",
46
+ "@chloedb/darwin-x64": "0.3.5"
47
+ },
48
+ "publishConfig": {
49
+ "access": "public"
50
+ }
11
51
  }