@botejs/core 0.1.2 → 0.1.4

package/README.md

@@ -0,0 +1,69 @@
+# bote
+
+a minimal, ergonomic and low-memory approach to navigating a big JSON:
+
+```sh
+npm install @botejs/core
+```
+
+```ts
+import { open, fromFile } from '@botejs/core'
+
+import * as z from 'zod' // or bring your own Standard Schema validator
+
+const User = z.object({
+  id: z.string(),
+  email: z.string(),
+})
+
+type User = z.infer<typeof User>
+
+await using cursor = await open(fromFile('./your-big.json'))
+
+// if you want one value
+const user0: unknown = await cursor.get('/1234/users/0')
+
+// for .get and .iter, you can supply a validator
+const user1: User = await cursor.get('/1234/users/1', User)
+
+// if you want to iterate a list of values
+for await (const user of cursor.iter('/1234/users')) {
+  console.log(user)
+}
+
+// if you want to iterate but not fully resolve values
+for await (const userCursor of cursor.walk('/1234/users')) {
+  const id = await userCursor.get('/id')
+  console.log({ id })
+}
+
+// 'await using' would normally clean up resources for you
+// when it goes out of lexical scope. if you hate that,
+// you can do it explicitly as well.
+await cursor.close()
+```
+
+given a **seekable** source (e.g. a file, an HTTP range) and a JSON pointer, it can retrieve values in a JSON quickly, without loading the whole thing in-memory.
+
+here's a run (Apple M1 Pro 2021, 500MB JSON array file, cold-cache, default settings):
+
+| operation    | approach   |      time | js heap peak Δ | rust heap peak |
+| ------------ | ---------- | --------: | -------------: | -------------: |
+| items[0]     | JSON.parse |    1.75 s |        1.21 GB |            n/a |
+| items[len/2] | JSON.parse |    1.82 s |        1.21 GB |            n/a |
+| items[len-1] | JSON.parse |    1.76 s |        1.21 GB |            n/a |
+| items[0]     | bote       |   1.43 ms |        25.9 KB |        94.9 KB |
+| items[len/2] | bote       | 328.81 ms |         1.3 MB |        56.6 MB |
+| items[len-1] | bote       | 636.78 ms |         1.3 MB |        56.6 MB |
+
+## sources
+
+bote currently only has `fromFile` and `fromHttpRange` as pre-built sources. create your own by implementing the `Source` interface. see [./packages/core/src/sources.ts](./packages/core/src/sources.ts) on how it works.
+
+## status
+
+pre-1.0 so still in development and APIs may change based on feedback, bugs and holy divinations from the coding gods.
+
+## license
+
+MIT.

package/dist/sources.js

@@ -66,7 +66,9 @@ function fromHttpRange(url, options) {
                     userSignal.addEventListener('abort', () => controller.abort(userSignal.reason), { once: true });
                 }
             }
-            const head = await fetch(url, { ...init, method: 'HEAD', signal: controller.signal });
+            const headHeaders = new Headers(init?.headers);
+            headHeaders.set('Accept-Encoding', 'identity');
+            const head = await fetch(url, { ...init, headers: headHeaders, method: 'HEAD', signal: controller.signal });
             if (!head.ok) {
                 throw new Error(`HEAD ${url} failed: ${head.status} ${head.statusText}`);
             }
@@ -88,6 +90,7 @@ function fromHttpRange(url, options) {
                     const end = Math.min(offset + dst.byteLength, size) - 1;
                     const headers = new Headers(init?.headers);
                     headers.set('Range', `bytes=${offset}-${end}`);
+                    headers.set('Accept-Encoding', 'identity');
                     const res = await fetch(url, { ...init, headers, method: 'GET', signal: controller.signal });
                     if (res.status === 206) {
                         const body = new Uint8Array(await res.arrayBuffer());

package/package.json

@@ -1,16 +1,17 @@
 {
   "name": "@botejs/core",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "license": "MIT",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/jankdc/bote.git",
-    "directory": "packages/bote"
+    "directory": "packages/core"
   },
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "files": [
-    "dist"
+    "dist",
+    "README.md"
   ],
   "engines": {
     "node": ">= 18.17.0 < 19 || >= 20.3.0 < 21 || >= 21.1.0"
@@ -24,7 +25,7 @@
     "build:debug": "tsc --sourceMap",
     "test": "node --test --experimental-strip-types --no-warnings=ExperimentalWarning __test__/*.spec.ts",
     "lint": "oxlint src",
-    "prepublishOnly": "tsc"
+    "prepublishOnly": "cp ../../README.md ./README.md && tsc"
   },
   "dependencies": {
     "@botejs/native": "workspace:*"