@botejs/core 0.1.3 → 0.2.0

package/README.md

@@ -0,0 +1,106 @@
+# bote
+
+a minimal, ergonomic and low-memory approach to navigating a big JSON:
+
+```sh
+npm install @botejs/core
+```
+
+```ts
+import { open, fromFile } from '@botejs/core'
+
+import * as z from 'zod' // or bring your own Standard Schema validator
+
+const User = z.object({
+  id: z.string(),
+  name: z.string(),
+  email: z.string(),
+  details: z.object({
+    lastLoggedIn: z.number(),
+  }),
+})
+
+type User = z.infer<typeof User>
+
+await using cursor = await open(fromFile('./your-big.json'))
+
+// users[1000].name
+const desc0: unknown = await cursor.get('users', 1000, 'name')
+// for .get and .iter, you can supply a validator as the last argument
+const desc1: string = await cursor.get('users', 1000, 'name', User.shape.name)
+
+// iterate an array in batches
+for await (const batch of cursor.iter('users', User)) {
+  // batch: User[]
+  for (const user of batch) {
+    console.log(user)
+  }
+}
+
+// pick several fields into a named object to avoid resolving big items
+for await (const batch of cursor.iter('users', {
+  select: {
+    id: 'id',
+    logged: ['details', 'lastLoggedIn'],
+  },
+  schema: z.object({
+    id: User.shape.id,
+    logged: User.shape.details.lastLoggedIn,
+  }),
+})) {
+  // batch: { id: string, logged: number }[]
+  for (const userLog of batch) {
+    console.log(userLog)
+  }
+}
+
+// or pick a single field
+for await (const batch of cursor.iter('users', {
+  select: 'name',
+  schema: User.shape.name,
+})) {
+  // batch: string[]
+  for (const name of batch) {
+    console.log({ name })
+  }
+}
+
+// for open-ended per-child work (e.g. conditional reads, recursive descent, nested
+// iters), `walk` yields a subcursor positioned at each child:
+for await (const metaCursor of cursor.walk('meta')) {
+  if (metaCursor.key === 'details') {
+    const detailsValue = await metaCursor.get()
+    console.log(detailsValue)
+  }
+}
+
+// 'await using' would normally clean up resources for you
+// when it goes out of lexical scope. if you hate that,
+// you can do it explicitly as well.
+await cursor.close()
+```
+
+given a **seekable** source (e.g. a file, an HTTP range) and a path, it can retrieve values in a JSON quickly, without loading the whole thing in-memory.
+
+here's a run (Apple M1 Pro 2021, ~500MB JSON array file, cold-cache, default settings):
+
+| operation      | approach   |      time | js heap peak Δ | rust heap peak |
+| -------------- | ---------- | --------: | -------------: | -------------: |
+| items[0]       | JSON.parse | 616.02 ms |        1.03 GB |            n/a |
+| items[535399]  | JSON.parse | 604.63 ms |        1.03 GB |            n/a |
+| items[1070797] | JSON.parse | 600.68 ms |        1.03 GB |            n/a |
+| items[0]       | bote       | 527.80 µs |       291.6 KB |       130.4 KB |
+| items[535399]  | bote       | 187.24 ms |       742.3 KB |        36.7 MB |
+| items[1070797] | bote       | 371.61 ms |       828.7 KB |        37.1 MB |
+
+## sources
+
+bote currently only has `fromFile` and `fromHttpRange` as pre-built sources. create your own by implementing the `Source` interface. see [./packages/core/src/sources.ts](./packages/core/src/sources.ts) on how it works.
+
+## status
+
+pre-1.0 so still in development and APIs may change based on feedback, bugs and holy divinations from the coding gods.
+
+## license
+
+MIT.

package/dist/args.d.ts

@@ -0,0 +1,21 @@
+import type { Path, Segment, StandardSchemaV1 } from './validate.ts';
+export interface IterOptions {
+    select?: Segment | Path | Record<string, Segment | Path>;
+    /** How many items are yielded per batch. Higher is faster, but takes more memory to materialise those items. */
+    batch?: number;
+    /** Validate each yielded item against this schema (after `select`). */
+    schema?: StandardSchemaV1;
+    /** Policy for items failing `schema`. Default `'throw'`; `'skip'` drops them. */
+    onInvalid?: 'throw' | 'skip';
+    /** Yield `[index, value]` tuples instead of bare values, where `index` is
+     *  the zero-based position of the element in the source array. */
+    withIndex?: boolean;
+}
+export type VariadicPathArgs<TTail> = [...Segment[]] | [...Segment[], TTail];
+export declare function splitArgs<TTail>(args: VariadicPathArgs<TTail>): {
+    path: Segment[];
+    tail: TTail | undefined;
+};
+export declare function isSchema(value: unknown): value is StandardSchemaV1;
+export declare function normalizeIterTail(tail: StandardSchemaV1 | IterOptions | undefined): IterOptions;
+export declare function serializeSelect(select: Segment | Path | Record<string, Segment | Path>): string;

package/dist/args.js

@@ -0,0 +1,64 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.splitArgs = splitArgs;
+exports.isSchema = isSchema;
+exports.normalizeIterTail = normalizeIterTail;
+exports.serializeSelect = serializeSelect;
+const path_ts_1 = require("./path.js");
+function splitArgs(args) {
+    let pathArgs;
+    let tail;
+    if (args.length === 0) {
+        pathArgs = [];
+        tail = undefined;
+    }
+    else {
+        const last = args[args.length - 1];
+        if (last !== null && typeof last === 'object' && !Array.isArray(last)) {
+            pathArgs = args.slice(0, -1);
+            tail = last;
+        }
+        else {
+            pathArgs = args;
+            tail = undefined;
+        }
+    }
+    (0, path_ts_1.validatePath)(pathArgs);
+    return { path: pathArgs, tail };
+}
+function isSchema(value) {
+    return typeof value === 'object' && value !== null && '~standard' in value;
+}
+function normalizeIterTail(tail) {
+    if (!tail)
+        return {};
+    if (isSchema(tail))
+        return { schema: tail };
+    return tail;
+}
+function serializeSelect(select) {
+    if (typeof select === 'string' || typeof select === 'number') {
+        const one = [select];
+        (0, path_ts_1.validatePath)(one);
+        return JSON.stringify({ one });
+    }
+    if (Array.isArray(select)) {
+        (0, path_ts_1.validatePath)(select);
+        if (select.length === 0) {
+            throw new RangeError('iter: select sub-path must have at least one segment');
+        }
+        return JSON.stringify({ one: select });
+    }
+    const entries = Object.entries(select).map(([k, sub]) => {
+        const path = typeof sub === 'string' || typeof sub === 'number' ? [sub] : sub;
+        (0, path_ts_1.validatePath)(path);
+        if (path.length === 0) {
+            throw new RangeError(`iter: select field ${JSON.stringify(k)} sub-path must have at least one segment`);
+        }
+        return [k, path];
+    });
+    if (entries.length === 0) {
+        throw new RangeError('iter: select must have at least one field');
+    }
+    return JSON.stringify({ map: entries });
+}

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-export type { JsonPointer } from './pointer.ts';
-export { open, type Cursor, type RootCursor, type SessionOptions } from './open.ts';
+export { type IterOptions } from './args.ts';
+export { ValidationError, formatPath, type Path, type Segment, type StandardSchemaV1 } from './validate.ts';
+export { open, DEFAULT_ITER_BATCH, type Cursor, type RootCursor, type OpenOptions, type IterIndex as IterKey, } from './open.ts';
 export { fromBuffer, fromFile, fromHttpRange, type FactoryOptions, type Source, type SourceReader, type HttpRangeOptions, } from './sources.ts';
-export { ValidationError, type StandardSchemaV1 } from './validate.ts';

package/dist/index.js

@@ -1,17 +1,19 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ValidationError = exports.fromHttpRange = exports.fromFile = exports.fromBuffer = void 0;
+exports.fromHttpRange = exports.fromFile = exports.fromBuffer = exports.DEFAULT_ITER_BATCH = exports.formatPath = exports.ValidationError = void 0;
 // Node 18 and Node 20.3 predate `Symbol.asyncDispose`; mirror what TS emits for
 // `await using` so the well-known symbol is available across our engine range.
 if (!Symbol.asyncDispose) {
     ;
     Symbol.asyncDispose = Symbol.for('Symbol.asyncDispose');
 }
+var validate_ts_1 = require("./validate.js");
+Object.defineProperty(exports, "ValidationError", { enumerable: true, get: function () { return validate_ts_1.ValidationError; } });
+Object.defineProperty(exports, "formatPath", { enumerable: true, get: function () { return validate_ts_1.formatPath; } });
 var open_ts_1 = require("./open.js");
 Object.defineProperty(exports, "open", { enumerable: true, get: function () { return open_ts_1.open; } });
+Object.defineProperty(exports, "DEFAULT_ITER_BATCH", { enumerable: true, get: function () { return open_ts_1.DEFAULT_ITER_BATCH; } });
 var sources_ts_1 = require("./sources.js");
 Object.defineProperty(exports, "fromBuffer", { enumerable: true, get: function () { return sources_ts_1.fromBuffer; } });
 Object.defineProperty(exports, "fromFile", { enumerable: true, get: function () { return sources_ts_1.fromFile; } });
 Object.defineProperty(exports, "fromHttpRange", { enumerable: true, get: function () { return sources_ts_1.fromHttpRange; } });
-var validate_ts_1 = require("./validate.js");
-Object.defineProperty(exports, "ValidationError", { enumerable: true, get: function () { return validate_ts_1.ValidationError; } });

package/dist/open.d.ts

@@ -1,34 +1,74 @@
-import type { JsonPointer } from './pointer.ts';
 import type { Source } from './sources.ts';
-import { type StandardSchemaV1 } from './validate.ts';
-export interface SessionOptions {
+import { type Path, type Segment, type StandardSchemaV1 } from './validate.ts';
+import { type IterOptions } from './args.ts';
+type InferOutput<Sch> = Sch extends StandardSchemaV1<unknown, infer O> ? O : never;
+type SelectMapShape<S> = {
+    -readonly [K in keyof S]: unknown;
+};
+/** Zero-based index of an array element. */
+export type IterIndex = number;
+export declare const DEFAULT_SOURCE_CHUNK_BYTES: number;
+export declare const DEFAULT_ITER_BATCH = 1000;
+export interface OpenOptions {
+    /**
+     * Slot budget for the structural-index cache: one slot per cached container
+     * plus one per tabled object member. When a scan tips the cache over this
+     * budget, the deepest (least navigationally useful) containers are evicted
+     * first, LRU-tiebroken, keeping the shallow backbone that resumes future
+     * scans. Bounds resident cache memory regardless of document size. `0`
+     * disables the cache entirely. Omit for the native default (1024).
+     */
+    indexCacheEntries?: number;
+    /**
+     * Max object members tabled per walked container in the structural-index
+     * cache. The table is a dense prefix; past the cap, lookups of later members
+     * resume-scan from the cap boundary. Lower trades cache memory for resume work
+     * on pathologically large objects. `0` disables object member indexing. Omit
+     * for the native default (unbounded).
+     */
+    objectMemberCap?: number;
     /**
-     * Maximum number of source chunks held resident at once. Each slot
-     * accounts for one chunk's bytes plus its bitmaps; the cache also
-     * enforces a derived byte ceiling at roughly `maxResidentChunks x
-     * source.chunkBytes x 2` to bound total native memory.
+     * Element-index stride between sampled array members in the structural-index
+     * cache. A later index resumes from the nearest array member at or before it, so
+     * a smaller stride means denser array members (more memory, shorter resume
+     * scans). `0` disables array-member indexing. Omit for the native default (16).
      *
-     * Defaults to 512 chunks.
+     * Setting both `objectMemberCap` and `arrayIndexInterval` to `0` disables the
+     * cache entirely (no source bytes are ever cached either way), as does
+     * `indexCacheEntries: 0`.
      */
-    maxResidentChunks?: number;
+    arrayIndexInterval?: number;
 }
-type InferOutput<Sch> = Sch extends StandardSchemaV1<unknown, infer O> ? O : never;
 export interface Cursor {
     /** Object-member key or array-element index that this cursor was yielded under by `walk`. `null` on the root cursor. */
     readonly key: string | number | null;
-    has<S extends string>(pointer: JsonPointer<S>): Promise<boolean>;
-    has<S extends string>(pointer: JsonPointer<S>, schema: StandardSchemaV1): Promise<boolean>;
-    get<S extends string>(pointer: JsonPointer<S>): Promise<unknown>;
-    get<S extends string, Sch extends StandardSchemaV1>(pointer: JsonPointer<S>, schema: Sch): Promise<InferOutput<Sch>>;
-    iter<S extends string>(pointer: JsonPointer<S>): AsyncIterable<unknown>;
-    iter<S extends string, Sch extends StandardSchemaV1>(pointer: JsonPointer<S>, schema: Sch): AsyncIterable<InferOutput<Sch>>;
-    walk<S extends string>(pointer: JsonPointer<S>): AsyncIterable<Cursor>;
+    has(...path: Segment[]): Promise<boolean>;
+    has(...args: [...Segment[], StandardSchemaV1]): Promise<boolean>;
+    get(...path: Segment[]): Promise<unknown>;
+    get<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): Promise<InferOutput<Sch>>;
+    count(...path: Segment[]): Promise<number>;
+    iter(...path: Segment[]): AsyncIterable<unknown[]>;
+    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): AsyncIterable<InferOutput<Sch>[]>;
+    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
+        withIndex: true;
+        schema: Sch;
+    }]): AsyncIterable<[IterIndex, InferOutput<Sch>][]>;
+    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
+        schema: Sch;
+    }]): AsyncIterable<InferOutput<Sch>[]>;
+    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
+        withIndex: true;
+        select: S;
+    }]): AsyncIterable<[IterIndex, SelectMapShape<S>][]>;
+    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
+        select: S;
+    }]): AsyncIterable<SelectMapShape<S>[]>;
+    iter(...args: [...Segment[], IterOptions & {
+        withIndex: true;
+    }]): AsyncIterable<[IterIndex, unknown][]>;
+    iter(...args: [...Segment[], IterOptions]): AsyncIterable<unknown[]>;
+    walk(...path: Segment[]): AsyncIterable<Cursor>;
 }
-/**
- * The cursor returned by `open()`. Owns the underlying `Source` and exposes
- * both an explicit `close()` and `Symbol.asyncDispose` so callers can choose
- * between manual cleanup and `await using` scoping.
- */
 export interface RootCursor extends Cursor, AsyncDisposable {
     /** Close the underlying source. Idempotent. */
     close(): Promise<void>;
@@ -36,14 +76,8 @@ export interface RootCursor extends Cursor, AsyncDisposable {
 /**
  * Open a cursor over a seekable source.
  *
- * Calls `source.open()` to acquire a reader, then constructs the native cursor
- * over it. The reader's `read(offset, buf)` is invoked with chunk-aligned
- * `offset` and a `buf` whose `byteLength` equals the configured chunk size;
- * the reader fills `buf` and resolves with `bytesRead`. `buf` is a view over
- * native-owned memory and **MUST** not be retained past the returned promise.
- *
  * The returned `RootCursor` owns the reader: `close()` (or `await using`)
  * drives the reader's own `close()` exactly once.
  */
-export declare function open(source: Source, options?: SessionOptions): Promise<RootCursor>;
+export declare function open(source: Source, options?: OpenOptions): Promise<RootCursor>;
 export {};

package/dist/open.js

@@ -1,30 +1,41 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_ITER_BATCH = exports.DEFAULT_SOURCE_CHUNK_BYTES = void 0;
 exports.open = open;
 const native_1 = require("@botejs/native");
+const path_ts_1 = require("./path.js");
 const validate_ts_1 = require("./validate.js");
+const args_ts_1 = require("./args.js");
+exports.DEFAULT_SOURCE_CHUNK_BYTES = 64 * 1024;
+exports.DEFAULT_ITER_BATCH = 1000;
 /**
  * Open a cursor over a seekable source.
  *
- * Calls `source.open()` to acquire a reader, then constructs the native cursor
- * over it. The reader's `read(offset, buf)` is invoked with chunk-aligned
- * `offset` and a `buf` whose `byteLength` equals the configured chunk size;
- * the reader fills `buf` and resolves with `bytesRead`. `buf` is a view over
- * native-owned memory and **MUST** not be retained past the returned promise.
- *
  * The returned `RootCursor` owns the reader: `close()` (or `await using`)
  * drives the reader's own `close()` exactly once.
  */
 async function open(source, options) {
+    const { indexCacheEntries, objectMemberCap, arrayIndexInterval } = options ?? {};
+    for (const [name, value] of [
+        ['indexCacheEntries', indexCacheEntries],
+        ['objectMemberCap', objectMemberCap],
+        ['arrayIndexInterval', arrayIndexInterval],
+    ]) {
+        if (value !== undefined && (!Number.isInteger(value) || value < 0)) {
+            throw new RangeError(`open: ${name} must be a non-negative integer (0 disables), got ${value}`);
+        }
+    }
     const reader = await source.open();
+    const chunkBytes = reader.chunkBytes ?? exports.DEFAULT_SOURCE_CHUNK_BYTES;
     let native;
     try {
         native = (0, native_1.open)({
             size: reader.size,
-            chunkBytes: reader.chunkBytes,
-            read: async ({ offset, buf }) => reader.read(offset, buf),
-        }, {
-            maxResidentChunks: options?.maxResidentChunks,
+            chunkBytes,
+            indexCacheEntries,
+            objectMemberCap,
+            arrayIndexInterval,
+            read: async ({ offset, length }) => reader.read(offset, length),
         });
     }
     catch (err) {
@@ -52,35 +63,60 @@ function wrap(native) {
         get key() {
             return native.key;
         },
-        async has(pointer, schema) {
+        async has(...args) {
+            const { path, tail: schema } = (0, args_ts_1.splitArgs)(args);
             if (!schema)
-                return native.has(pointer);
-            if (!(await native.has(pointer)))
+                return native.has(path);
+            if (!(await native.has(path)))
                 return false;
-            const result = await schema['~standard'].validate(await native.get(pointer));
-            return result.issues === undefined;
+            const result = await (0, validate_ts_1.validateItem)(schema, await native.get(path), path, 'skip');
+            return !('skip' in result);
+        },
+        async get(...args) {
+            const { path, tail: schema } = (0, args_ts_1.splitArgs)(args);
+            const value = await native.get(path);
+            if (!schema || value === undefined)
+                return value;
+            return (0, validate_ts_1.runStandardSchema)(schema, value, path);
         },
-        async get(pointer, schema) {
-            const value = await native.get(pointer);
-            return schema ? (0, validate_ts_1.runStandardSchema)(schema, value, pointer) : value;
+        count(...path) {
+            (0, path_ts_1.validatePath)(path);
+            return native.count(path);
         },
-        iter(pointer, schema) {
-            const inner = native.iter(pointer);
+        iter(...args) {
+            const { path, tail } = (0, args_ts_1.splitArgs)(args);
+            const { schema, select, batch, onInvalid, withIndex } = (0, args_ts_1.normalizeIterTail)(tail);
+            if (batch !== undefined && (!Number.isInteger(batch) || batch <= 0)) {
+                throw new RangeError(`iter: batch must be a positive integer, got ${batch}`);
+            }
+            const resolvedBatch = batch ?? exports.DEFAULT_ITER_BATCH;
+            const selectIr = select !== undefined ? (0, args_ts_1.serializeSelect)(select) : undefined;
+            const inner = native.iter(path, { selectIr, batch: resolvedBatch, withKey: withIndex });
             if (!schema)
                 return inner;
+            const policy = onInvalid ?? 'throw';
             return {
                 async *[Symbol.asyncIterator]() {
                     let i = 0;
-                    for await (const v of inner) {
-                        yield await (0, validate_ts_1.runStandardSchema)(schema, v, `${pointer}/${i++}`);
+                    for await (const b of inner) {
+                        const out = [];
+                        for (const v of b) {
+                            const value = withIndex ? v[1] : v;
+                            const result = await (0, validate_ts_1.validateItem)(schema, value, [...path, i++], policy);
+                            if ('skip' in result)
+                                continue;
+                            out.push(withIndex ? [v[0], result.value] : result.value);
+                        }
+                        yield out;
                     }
                 },
             };
         },
-        walk(pointer) {
+        walk(...path) {
+            (0, path_ts_1.validatePath)(path);
             return {
                 async *[Symbol.asyncIterator]() {
-                    for await (const child of native.walk(pointer)) {
+                    for await (const child of native.walk(path)) {
                         yield wrap(child);
                     }
                 },

package/dist/path.d.ts

@@ -0,0 +1,5 @@
+import type { Segment } from './validate.ts';
+/** Upper bound on numeric segments (napi takes them as `u32`). 2^32 - 1
+ *  comfortably covers any in-memory JSON array. */
+export declare const MAX_ARRAY_INDEX = 4294967295;
+export declare function validatePath(path: readonly unknown[]): asserts path is readonly Segment[];

package/dist/path.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MAX_ARRAY_INDEX = void 0;
+exports.validatePath = validatePath;
+/** Upper bound on numeric segments (napi takes them as `u32`). 2^32 - 1
+ *  comfortably covers any in-memory JSON array. */
+exports.MAX_ARRAY_INDEX = 0xffffffff;
+function validatePath(path) {
+    for (let i = 0; i < path.length; i++) {
+        const s = path[i];
+        if (typeof s === 'string')
+            continue;
+        if (typeof s === 'number' && Number.isInteger(s) && s >= 0 && s <= exports.MAX_ARRAY_INDEX)
+            continue;
+        throw new TypeError(`path segment ${i}: expected string or non-negative integer (<= ${exports.MAX_ARRAY_INDEX}), got ${describeBadSegment(s)}`);
+    }
+}
+function describeBadSegment(s) {
+    if (typeof s === 'number')
+        return `${s}`;
+    if (s === null)
+        return 'null';
+    return typeof s;
+}

package/dist/sources.d.ts

@@ -10,21 +10,17 @@ export interface SourceReader {
     /** Preferred read granularity in bytes. Must be a non-zero multiple of 64. */
     readonly chunkBytes?: number;
     /**
-     * Fill `buf` with up to `buf.byteLength` bytes starting at `offset` and
-     * resolve with the number of bytes written. The implementation must not
-     * retain a reference to `buf` or read from it after the returned promise
-     * resolves: `buf` is a view over native-owned memory whose lifetime ends
-     * once the promise settles.
+     * Read up to `length` bytes starting at `offset` and resolve with the
+     * bytes read. The returned `Uint8Array`'s `.byteLength` is the actual
+     * count, which must be `<= length`.
      */
-    read(offset: number, buf: Uint8Array): Promise<number>;
+    read(offset: number, length: number): Promise<Uint8Array>;
     /** Release resources held by the reader. Driven once by the `open()` lifecycle. */
     close?(): Promise<void> | void;
 }
 /**
- * Describes how to obtain a seekable byte stream. Construction is cheap and
- * synchronous - no I/O happens until `open()` runs, which the top-level
- * `open()` API drives. Provide your own object implementing this interface to
- * plug in custom backends.
+ * Describes how to obtain a seekable byte stream. Provide your own object implementing
+ * this interface to plug in custom backends.
  */
 export interface Source {
     /** Acquire the stream. Resolves to a `SourceReader` that owns any underlying resources. */

package/dist/sources.js

@@ -17,13 +17,7 @@ function fromBuffer(buf, options) {
         open: () => Promise.resolve({
             size: view.byteLength,
             chunkBytes,
-            read: async (offset, dst) => {
-                const end = Math.min(offset + dst.byteLength, view.byteLength);
-                const n = Math.max(0, end - offset);
-                if (n > 0)
-                    dst.set(view.subarray(offset, end));
-                return n;
-            },
+            read: (offset, length) => Promise.resolve(view.subarray(offset, Math.min(offset + length, view.byteLength))),
         }),
     };
 }
@@ -37,9 +31,16 @@ function fromFile(path, options) {
             return {
                 size: stat.size,
                 chunkBytes,
-                read: async (offset, dst) => {
-                    const { bytesRead } = await handle.read(dst, 0, dst.byteLength, offset);
-                    return bytesRead;
+                read: async (offset, length) => {
+                    const buf = Buffer.allocUnsafe(length);
+                    let filled = 0;
+                    while (filled < length) {
+                        const { bytesRead } = await handle.read(buf, filled, length - filled, offset + filled);
+                        if (bytesRead === 0)
+                            break;
+                        filled += bytesRead;
+                    }
+                    return buf.subarray(0, filled);
                 },
                 close: async () => {
                     if (closed)
@@ -85,17 +86,15 @@ function fromHttpRange(url, options) {
             return {
                 size,
                 chunkBytes,
-                read: async (offset, dst) => {
+                read: async (offset, length) => {
                     // HTTP ranges are inclusive on both ends.
-                    const end = Math.min(offset + dst.byteLength, size) - 1;
+                    const end = Math.min(offset + length, size) - 1;
                     const headers = new Headers(init?.headers);
                     headers.set('Range', `bytes=${offset}-${end}`);
                     headers.set('Accept-Encoding', 'identity');
                     const res = await fetch(url, { ...init, headers, method: 'GET', signal: controller.signal });
                     if (res.status === 206) {
-                        const body = new Uint8Array(await res.arrayBuffer());
-                        dst.set(body);
-                        return body.byteLength;
+                        return new Uint8Array(await res.arrayBuffer());
                     }
                     // A 200 means the server ignored our Range request and returned the full
                     // body. We throw here since the point of using ranges is to not have to

package/dist/validate.d.ts

@@ -1,8 +1,16 @@
 import type { StandardSchemaV1 } from '@standard-schema/spec';
 export type { StandardSchemaV1 };
+export type Segment = string | number;
+export type Path = readonly Segment[];
 export declare class ValidationError extends Error {
     readonly issues: readonly StandardSchemaV1.Issue[];
-    readonly pointer: string;
-    constructor(issues: readonly StandardSchemaV1.Issue[], pointer: string);
+    readonly path: Path;
+    constructor(issues: readonly StandardSchemaV1.Issue[], path: Path);
 }
-export declare function runStandardSchema<O>(schema: StandardSchemaV1<unknown, O>, value: unknown, pointer: string): Promise<O>;
+export declare function runStandardSchema<O>(schema: StandardSchemaV1<unknown, O>, value: unknown, path: Path): Promise<O>;
+export declare function validateItem<O>(schema: StandardSchemaV1<unknown, O>, value: unknown, path: Path, onInvalid: 'throw' | 'skip'): Promise<{
+    skip: true;
+} | {
+    value: O;
+}>;
+export declare function formatPath(path: Path): string;

package/dist/validate.js

@@ -2,20 +2,50 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ValidationError = void 0;
 exports.runStandardSchema = runStandardSchema;
+exports.validateItem = validateItem;
+exports.formatPath = formatPath;
 class ValidationError extends Error {
     issues;
-    pointer;
-    constructor(issues, pointer) {
-        super(`bote: schema validation failed at ${pointer || '/'}: ${issues[0]?.message ?? 'unknown'}`);
+    path;
+    constructor(issues, path) {
+        super(`bote: schema validation failed at ${formatPath(path)}: ${issues[0]?.message ?? 'unknown'}`);
         this.name = 'ValidationError';
         this.issues = issues;
-        this.pointer = pointer;
+        this.path = path;
     }
 }
 exports.ValidationError = ValidationError;
-async function runStandardSchema(schema, value, pointer) {
+async function runStandardSchema(schema, value, path) {
     const result = await schema['~standard'].validate(value);
     if (result.issues)
-        throw new ValidationError(result.issues, pointer);
+        throw new ValidationError(result.issues, path);
     return result.value;
 }
+async function validateItem(schema, value, path, onInvalid) {
+    const result = await schema['~standard'].validate(value);
+    if (result.issues) {
+        if (onInvalid === 'skip')
+            return { skip: true };
+        throw new ValidationError(result.issues, path);
+    }
+    return { value: result.value };
+}
+function formatPath(path) {
+    if (path.length === 0)
+        return '(root)';
+    let out = '';
+    for (let i = 0; i < path.length; i++) {
+        const seg = path[i];
+        if (typeof seg === 'number') {
+            out += `[${seg}]`;
+            continue;
+        }
+        if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(seg)) {
+            out += i === 0 ? seg : `.${seg}`;
+        }
+        else {
+            out += `[${JSON.stringify(seg)}]`;
+        }
+    }
+    return out;
+}

package/package.json

@@ -1,16 +1,17 @@
 {
   "name": "@botejs/core",
-  "version": "0.1.3",
+  "version": "0.2.0",
   "license": "MIT",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/jankdc/bote.git",
-    "directory": "packages/bote"
+    "directory": "packages/core"
   },
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "files": [
-    "dist"
+    "dist",
+    "README.md"
   ],
   "engines": {
     "node": ">= 18.17.0 < 19 || >= 20.3.0 < 21 || >= 21.1.0"
@@ -24,7 +25,7 @@
     "build:debug": "tsc --sourceMap",
     "test": "node --test --experimental-strip-types --no-warnings=ExperimentalWarning __test__/*.spec.ts",
     "lint": "oxlint src",
-    "prepublishOnly": "tsc"
+    "prepublishOnly": "cp ../../README.md ./README.md && tsc"
   },
   "dependencies": {
     "@botejs/native": "workspace:*"

package/dist/pointer.d.ts

@@ -1,5 +0,0 @@
-type ValidateTokenChars<S extends string> = S extends `${string}~${infer Rest}` ? Rest extends `0${infer After}` | `1${infer After}` ? ValidateTokenChars<After> : false : true;
-type ValidateTokens<S extends string> = S extends `${infer Token}/${infer Rest}` ? ValidateTokenChars<Token> extends true ? ValidateTokens<Rest> : false : ValidateTokenChars<S>;
-type IsJsonPointer<S extends string> = S extends '' ? true : S extends `/${infer Rest}` ? ValidateTokens<Rest> : false;
-export type JsonPointer<S extends string> = IsJsonPointer<S> extends true ? S : `Error: invalid JSON pointer "${S}"`;
-export {};

package/dist/pointer.js

@@ -1,3 +0,0 @@
-"use strict";
-// RFC 6901 JSON Pointer Static Typing Validator
-Object.defineProperty(exports, "__esModule", { value: true });