@botejs/core 0.1.4 → 0.3.0

package/README.md

@@ -8,57 +8,121 @@ npm install @botejs/core
 
 ```ts
 import { open, fromFile } from '@botejs/core'
+import { publish } from './message-bus'
 
-import * as z from 'zod' // or bring your own Standard Schema validator
+// e.g. { items: [...] }
+await using cursor = await open(fromFile('./some-large.json'))
 
-const User = z.object({
-  id: z.string(),
-  email: z.string(),
-})
+// items[0]
+const first = await cursor.get('items', 0)
+console.log(`first item: ${first}`)
+```
+
+given a **seekable** source (e.g. a file, an HTTP range) and a path, it retrieves values out of a JSON quickly, without loading the whole thing in-memory.
+
+here's a run (Apple M1 Pro 2021, ~500MB JSON array file, cold-cache, default settings):
+
+| operation      | approach   |      time | js heap peak Δ | rust heap peak |
+| -------------- | ---------- | --------: | -------------: | -------------: |
+| items[0]       | JSON.parse | 616.02 ms |        1.03 GB |            n/a |
+| items[535399]  | JSON.parse | 604.63 ms |        1.03 GB |            n/a |
+| items[1070797] | JSON.parse | 600.68 ms |        1.03 GB |            n/a |
+| items[0]       | bote       | 527.80 µs |       291.6 KB |       130.4 KB |
+| items[535399]  | bote       | 187.24 ms |       742.3 KB |        36.7 MB |
+| items[1070797] | bote       | 371.61 ms |       828.7 KB |        37.1 MB |
+
+## array access
+
+`iter` streams the elements of an array at a path, **a batch at a time**, so you never hold the whole collection in memory and not wait for the heat death of the universe if this yielded individually. each `for await` step yields an array of items (use `walk` to step over the members of an object):
+
+```ts
+// e.g. [{ id: 'user-1' }, { id: 'user-2' }, ...]
+await using cursor = await open(fromFile('./users.json'))
+
+// root is an array
+for await (const users of cursor.iter()) {
+  for (const user of users) {
+    console.log(user)
+  }
+}
+```
+
+pass an options object as the last argument to tune what comes back: `batch`, `select`, `schema`, `onInvalid`, and `withIndex`. if you want to know more of the options, see [`arrays.js`](./examples/arrays.js).
+
+## object access
+
+`walk` steps over the members of an object at a path, yielding a **`[key, cursor]`** pair per member. the key is the member name, the cursor is anchored at its value. each child cursor is first-class: it outlives the loop and can be `walk`ed again, which is what lets you descend a tree of unknown depth.
+
+```ts
+// e.g. { alice: { role: 'admin' }, bob: { role: 'guest' }, ... }
+await using cursor = await open(fromFile('./accounts.json'))
 
-type User = z.infer<typeof User>
+for await (const [name, account] of cursor.walk()) {
+  // name is the member name ('alice', 'bob', ...)
+  const role = await account.get('role')
+  console.log(`${name}: ${role}`)
+}
+```
 
-await using cursor = await open(fromFile('./your-big.json'))
+see [`recursive.js`](./examples/recursive.js) for advanced use-cases.
 
-// if you want one value
-const user0: unknown = await cursor.get('/1234/users/0')
+## hopping
 
-// for .get and .iter, you can supply a validator
-const user1: User = await cursor.get('/1234/users/1', User)
+`hop` resolves a path once and hands back a **cursor** anchored at that value (or `null` if the path isn't there):
 
-// if you want to iterate a list of values
-for await (const user of cursor.iter('/1234/users')) {
-  console.log(user)
+```ts
+// e.g. { report: { sections: [{ rows: [...] }, ...] } }
+await using cursor = await open(fromFile('./report.json'))
+
+const section = await cursor.hop('report', 'sections', 0)
+if (section) {
+  console.log(await section.count('rows'))
+  for await (const rows of section.iter('rows')) {
+    console.log(rows)
+  }
 }
+```
+
+## validation
+
+`get`, and `iter` takes a [Standard Schema](https://standardschema.dev) validator as their last argument (for `iter`, can also be passed in an `options` object). the value is validated and the return type is inferred from the schema, so reads come back typed instead of `unknown`:
+
+```ts
+import { open, fromFile } from '@botejs/core'
+import * as z from 'zod' // or any Standard Schema validator
+
+// a downstream API that wants a typed list of recipients
+declare function sendNewsletter(recipients: string[]): Promise<void>
+
+const User = z.object({
+  id: z.string(),
+  name: z.string(),
+  email: z.string(),
+})
+
+const cursor = await open(fromFile('./users.json'))
+
+// name: string
+const name = await cursor.get('users', 1000, 'name', User.shape.name)
 
-// if you want to iterate but not fully resolve values
-for await (const userCursor of cursor.walk('/1234/users')) {
-  const id = await userCursor.get('/id')
-  console.log({ id })
+for await (const users of cursor.iter('users', User)) {
+  // user: User[]
+  const emails = users.map((user) => user.email)
+  await sendNewsletter(emails)
 }
 
-// 'await using' would normally clean up resources for you
-// when it goes out of lexical scope. if you hate that,
-// you can do it explicitly as well.
 await cursor.close()
 ```
 
-given a **seekable** source (e.g. a file, an HTTP range) and a JSON pointer, it can retrieve values in a JSON quickly, without loading the whole thing in-memory.
+## memory
 
-here's a run (Apple M1 Pro 2021, 500MB JSON array file, cold-cache, default settings):
+bote keeps a small **structural-index** cache: as scans walk containers (arrays and object), it remembers where members live, so a later query that lands in an already walked container resumes near the target instead of from the top. it caches structure, never source bytes, so it can't grow unbounded with document size.
 
-| operation    | approach   |      time | js heap peak Δ | rust heap peak |
-| ------------ | ---------- | --------: | -------------: | -------------: |
-| items[0]     | JSON.parse |    1.75 s |        1.21 GB |            n/a |
-| items[len/2] | JSON.parse |    1.82 s |        1.21 GB |            n/a |
-| items[len-1] | JSON.parse |    1.76 s |        1.21 GB |            n/a |
-| items[0]     | bote       |   1.43 ms |        25.9 KB |        94.9 KB |
-| items[len/2] | bote       | 328.81 ms |         1.3 MB |        56.6 MB |
-| items[len-1] | bote       | 636.78 ms |         1.3 MB |        56.6 MB |
+the defaults are good, but `open` takes a few knobs: `indexCacheEntries`, `objectMemberCap`, and `arrayIndexInterval`. to bound memory tighter or turn the cache off. see [`memory.js`](./examples/memory.js) for what each does.
 
 ## sources
 
-bote currently only has `fromFile` and `fromHttpRange` as pre-built sources. create your own by implementing the `Source` interface. see [./packages/core/src/sources.ts](./packages/core/src/sources.ts) on how it works.
+bote ships `fromFile`, `fromHttpRange`, and `fromBuffer` as pre-built sources. create your own by implementing the `Source` interface. see [`sources-custom.ts`](./examples/sources-custom.ts) or [./packages/core/src/sources.ts](./packages/core/src/sources.ts) for how it works.
 
 ## status
 

package/dist/args.d.ts

@@ -0,0 +1,21 @@
+import type { Path, Segment, StandardSchemaV1 } from './validate.ts';
+export interface IterOptions {
+    select?: Segment | Path | Record<string, Segment | Path>;
+    /** How many items are yielded per batch. Higher is faster, but takes more memory to materialise those items. */
+    batch?: number;
+    /** Validate each yielded item against this schema (after `select`). */
+    schema?: StandardSchemaV1;
+    /** Policy for items failing `schema`. Default `'throw'`; `'skip'` drops them. */
+    onInvalid?: 'throw' | 'skip';
+    /** Yield `[index, value]` tuples instead of bare values, where `index` is
+     *  the zero-based position of the element in the source array. */
+    withIndex?: boolean;
+}
+export type VariadicPathArgs<TTail> = [...Segment[]] | [...Segment[], TTail];
+export declare function splitArgs<TTail>(args: VariadicPathArgs<TTail>): {
+    path: Segment[];
+    tail: TTail | undefined;
+};
+export declare function isSchema(value: unknown): value is StandardSchemaV1;
+export declare function normalizeIterTail(tail: StandardSchemaV1 | IterOptions | undefined): IterOptions;
+export declare function serializeSelect(select: Segment | Path | Record<string, Segment | Path>): string;

package/dist/args.js

@@ -0,0 +1,75 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.splitArgs = splitArgs;
+exports.isSchema = isSchema;
+exports.normalizeIterTail = normalizeIterTail;
+exports.serializeSelect = serializeSelect;
+const path_ts_1 = require("./path.js");
+function splitArgs(args) {
+    let pathArgs;
+    let tail;
+    if (args.length === 0) {
+        pathArgs = [];
+        tail = undefined;
+    }
+    else {
+        const last = args[args.length - 1];
+        if (last !== null && typeof last === 'object' && !Array.isArray(last)) {
+            pathArgs = args.slice(0, -1);
+            tail = last;
+        }
+        else {
+            pathArgs = args;
+            tail = undefined;
+        }
+    }
+    (0, path_ts_1.validatePath)(pathArgs);
+    return { path: pathArgs, tail };
+}
+function isSchema(value) {
+    return typeof value === 'object' && value !== null && '~standard' in value;
+}
+function normalizeIterTail(tail) {
+    if (!tail)
+        return {};
+    if (isSchema(tail))
+        return { schema: tail };
+    return tail;
+}
+function serializeSelect(select) {
+    if (typeof select === 'string' || typeof select === 'number') {
+        const one = [select];
+        (0, path_ts_1.validatePath)(one);
+        return JSON.stringify({ one });
+    }
+    if (Array.isArray(select)) {
+        (0, path_ts_1.validatePath)(select);
+        if (select.length === 0) {
+            throw new RangeError('iter: select sub-path must have at least one segment');
+        }
+        return JSON.stringify({ one: select });
+    }
+    if (select === null || typeof select !== 'object') {
+        throw new TypeError(`iter: select must be a segment, path, or field map, got ${describeSelect(select)}`);
+    }
+    const entries = Object.entries(select).map(([k, sub]) => {
+        const path = typeof sub === 'string' || typeof sub === 'number' ? [sub] : sub;
+        if (!Array.isArray(path)) {
+            throw new TypeError(`iter: select field ${JSON.stringify(k)} must be a segment or path, got ${describeSelect(sub)}`);
+        }
+        (0, path_ts_1.validatePath)(path);
+        if (path.length === 0) {
+            throw new RangeError(`iter: select field ${JSON.stringify(k)} sub-path must have at least one segment`);
+        }
+        return [k, path];
+    });
+    if (entries.length === 0) {
+        throw new RangeError('iter: select must have at least one field');
+    }
+    return JSON.stringify({ map: entries });
+}
+function describeSelect(value) {
+    if (value === null)
+        return 'null';
+    return Array.isArray(value) ? 'array' : typeof value;
+}

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-export type { JsonPointer } from './pointer.ts';
-export { open, type Cursor, type RootCursor, type SessionOptions } from './open.ts';
+export { type IterOptions } from './args.ts';
+export { ValidationError, PathError, formatPath, type Path, type Segment, type StandardSchemaV1, } from './validate.ts';
+export { open, DEFAULT_ITER_BATCH, MAX_ITER_BATCH, type Cursor, type RootCursor, type OpenOptions, type WalkEntry, type IterIndex as IterKey, } from './open.ts';
 export { fromBuffer, fromFile, fromHttpRange, type FactoryOptions, type Source, type SourceReader, type HttpRangeOptions, } from './sources.ts';
-export { ValidationError, type StandardSchemaV1 } from './validate.ts';

package/dist/index.js

@@ -1,17 +1,21 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ValidationError = exports.fromHttpRange = exports.fromFile = exports.fromBuffer = void 0;
+exports.fromHttpRange = exports.fromFile = exports.fromBuffer = exports.MAX_ITER_BATCH = exports.DEFAULT_ITER_BATCH = exports.formatPath = exports.PathError = exports.ValidationError = void 0;
 // Node 18 and Node 20.3 predate `Symbol.asyncDispose`; mirror what TS emits for
 // `await using` so the well-known symbol is available across our engine range.
 if (!Symbol.asyncDispose) {
     ;
     Symbol.asyncDispose = Symbol.for('Symbol.asyncDispose');
 }
+var validate_ts_1 = require("./validate.js");
+Object.defineProperty(exports, "ValidationError", { enumerable: true, get: function () { return validate_ts_1.ValidationError; } });
+Object.defineProperty(exports, "PathError", { enumerable: true, get: function () { return validate_ts_1.PathError; } });
+Object.defineProperty(exports, "formatPath", { enumerable: true, get: function () { return validate_ts_1.formatPath; } });
 var open_ts_1 = require("./open.js");
 Object.defineProperty(exports, "open", { enumerable: true, get: function () { return open_ts_1.open; } });
+Object.defineProperty(exports, "DEFAULT_ITER_BATCH", { enumerable: true, get: function () { return open_ts_1.DEFAULT_ITER_BATCH; } });
+Object.defineProperty(exports, "MAX_ITER_BATCH", { enumerable: true, get: function () { return open_ts_1.MAX_ITER_BATCH; } });
 var sources_ts_1 = require("./sources.js");
 Object.defineProperty(exports, "fromBuffer", { enumerable: true, get: function () { return sources_ts_1.fromBuffer; } });
 Object.defineProperty(exports, "fromFile", { enumerable: true, get: function () { return sources_ts_1.fromFile; } });
 Object.defineProperty(exports, "fromHttpRange", { enumerable: true, get: function () { return sources_ts_1.fromHttpRange; } });
-var validate_ts_1 = require("./validate.js");
-Object.defineProperty(exports, "ValidationError", { enumerable: true, get: function () { return validate_ts_1.ValidationError; } });

package/dist/open.d.ts

@@ -1,34 +1,77 @@
-import type { JsonPointer } from './pointer.ts';
 import type { Source } from './sources.ts';
-import { type StandardSchemaV1 } from './validate.ts';
-export interface SessionOptions {
+import { type Path, type Segment, type StandardSchemaV1 } from './validate.ts';
+import { type IterOptions } from './args.ts';
+type InferOutput<Sch> = Sch extends StandardSchemaV1<unknown, infer O> ? O : never;
+type SelectMapShape<S> = {
+    -readonly [K in keyof S]: unknown;
+};
+/** Zero-based index of an array element. */
+export type IterIndex = number;
+/** One `walk` step: the member's key paired with a cursor anchored at its value. */
+export type WalkEntry = [key: string, cursor: Cursor];
+export declare const DEFAULT_SOURCE_CHUNK_BYTES: number;
+export declare const DEFAULT_ITER_BATCH = 1000;
+export declare const MAX_ITER_BATCH = 1000000;
+export interface OpenOptions {
+    /**
+     * Slot budget for the structural-index cache: one slot per cached container
+     * plus one per tabled object member. When a scan tips the cache over this
+     * budget, the deepest (least navigationally useful) containers are evicted
+     * first, LRU-tiebroken, keeping the shallow backbone that resumes future
+     * scans. Bounds resident cache memory regardless of document size. `0`
+     * disables the cache entirely. Omit for the native default (1024).
+     */
+    indexCacheEntries?: number;
+    /**
+     * Max object members tabled per walked container in the structural-index
+     * cache. The table is a dense prefix; past the cap, lookups of later members
+     * resume-scan from the cap boundary. Lower trades cache memory for resume work
+     * on pathologically large objects. `0` disables object member indexing. Omit
+     * for the native default (unbounded).
+     */
+    objectMemberCap?: number;
     /**
-     * Maximum number of source chunks held resident at once. Each slot
-     * accounts for one chunk's bytes plus its bitmaps; the cache also
-     * enforces a derived byte ceiling at roughly `maxResidentChunks x
-     * source.chunkBytes x 2` to bound total native memory.
+     * Element-index stride between sampled array members in the structural-index
+     * cache. A later index resumes from the nearest array member at or before it, so
+     * a smaller stride means denser array members (more memory, shorter resume
+     * scans). `0` disables array-member indexing. Omit for the native default (16).
      *
-     * Defaults to 512 chunks.
+     * Setting both `objectMemberCap` and `arrayIndexInterval` to `0` disables the
+     * cache entirely (no source bytes are ever cached either way), as does
+     * `indexCacheEntries: 0`.
      */
-    maxResidentChunks?: number;
+    arrayIndexInterval?: number;
 }
-type InferOutput<Sch> = Sch extends StandardSchemaV1<unknown, infer O> ? O : never;
 export interface Cursor {
-    /** Object-member key or array-element index that this cursor was yielded under by `walk`. `null` on the root cursor. */
-    readonly key: string | number | null;
-    has<S extends string>(pointer: JsonPointer<S>): Promise<boolean>;
-    has<S extends string>(pointer: JsonPointer<S>, schema: StandardSchemaV1): Promise<boolean>;
-    get<S extends string>(pointer: JsonPointer<S>): Promise<unknown>;
-    get<S extends string, Sch extends StandardSchemaV1>(pointer: JsonPointer<S>, schema: Sch): Promise<InferOutput<Sch>>;
-    iter<S extends string>(pointer: JsonPointer<S>): AsyncIterable<unknown>;
-    iter<S extends string, Sch extends StandardSchemaV1>(pointer: JsonPointer<S>, schema: Sch): AsyncIterable<InferOutput<Sch>>;
-    walk<S extends string>(pointer: JsonPointer<S>): AsyncIterable<Cursor>;
+    hop(...path: Segment[]): Promise<Cursor | null>;
+    has(...path: Segment[]): Promise<boolean>;
+    has(...args: [...Segment[], StandardSchemaV1]): Promise<boolean>;
+    get(...path: Segment[]): Promise<unknown>;
+    get<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): Promise<InferOutput<Sch>>;
+    count(...path: Segment[]): Promise<number>;
+    iter(...path: Segment[]): AsyncIterable<unknown[]>;
+    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): AsyncIterable<InferOutput<Sch>[]>;
+    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
+        withIndex: true;
+        schema: Sch;
+    }]): AsyncIterable<[IterIndex, InferOutput<Sch>][]>;
+    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
+        schema: Sch;
+    }]): AsyncIterable<InferOutput<Sch>[]>;
+    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
+        withIndex: true;
+        select: S;
+    }]): AsyncIterable<[IterIndex, SelectMapShape<S>][]>;
+    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
+        select: S;
+    }]): AsyncIterable<SelectMapShape<S>[]>;
+    iter(...args: [...Segment[], IterOptions & {
+        withIndex: true;
+    }]): AsyncIterable<[IterIndex, unknown][]>;
+    iter(...args: [...Segment[], IterOptions]): AsyncIterable<unknown[]>;
+    walk(...path: Segment[]): AsyncIterable<WalkEntry>;
+    walk(...path: Segment[]): AsyncIterable<Cursor>;
 }
-/**
- * The cursor returned by `open()`. Owns the underlying `Source` and exposes
- * both an explicit `close()` and `Symbol.asyncDispose` so callers can choose
- * between manual cleanup and `await using` scoping.
- */
 export interface RootCursor extends Cursor, AsyncDisposable {
     /** Close the underlying source. Idempotent. */
     close(): Promise<void>;
@@ -36,14 +79,8 @@ export interface RootCursor extends Cursor, AsyncDisposable {
 /**
  * Open a cursor over a seekable source.
  *
- * Calls `source.open()` to acquire a reader, then constructs the native cursor
- * over it. The reader's `read(offset, buf)` is invoked with chunk-aligned
- * `offset` and a `buf` whose `byteLength` equals the configured chunk size;
- * the reader fills `buf` and resolves with `bytesRead`. `buf` is a view over
- * native-owned memory and **MUST** not be retained past the returned promise.
- *
  * The returned `RootCursor` owns the reader: `close()` (or `await using`)
  * drives the reader's own `close()` exactly once.
  */
-export declare function open(source: Source, options?: SessionOptions): Promise<RootCursor>;
+export declare function open(source: Source, options?: OpenOptions): Promise<RootCursor>;
 export {};

package/dist/open.js

@@ -1,44 +1,72 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.MAX_ITER_BATCH = exports.DEFAULT_ITER_BATCH = exports.DEFAULT_SOURCE_CHUNK_BYTES = void 0;
 exports.open = open;
 const native_1 = require("@botejs/native");
+const path_ts_1 = require("./path.js");
 const validate_ts_1 = require("./validate.js");
+const args_ts_1 = require("./args.js");
+exports.DEFAULT_SOURCE_CHUNK_BYTES = 64 * 1024;
+exports.DEFAULT_ITER_BATCH = 1000;
+exports.MAX_ITER_BATCH = 1_000_000;
 /**
  * Open a cursor over a seekable source.
  *
- * Calls `source.open()` to acquire a reader, then constructs the native cursor
- * over it. The reader's `read(offset, buf)` is invoked with chunk-aligned
- * `offset` and a `buf` whose `byteLength` equals the configured chunk size;
- * the reader fills `buf` and resolves with `bytesRead`. `buf` is a view over
- * native-owned memory and **MUST** not be retained past the returned promise.
- *
  * The returned `RootCursor` owns the reader: `close()` (or `await using`)
  * drives the reader's own `close()` exactly once.
  */
 async function open(source, options) {
+    const { indexCacheEntries, objectMemberCap, arrayIndexInterval } = options ?? {};
+    for (const [name, value] of [
+        ['indexCacheEntries', indexCacheEntries],
+        ['objectMemberCap', objectMemberCap],
+        ['arrayIndexInterval', arrayIndexInterval],
+    ]) {
+        if (value !== undefined && (!Number.isSafeInteger(value) || value < 0)) {
+            throw new RangeError(`open: ${name} must be a non-negative integer (0 disables), got ${value}`);
+        }
+    }
     const reader = await source.open();
+    const chunkBytes = reader.chunkBytes ?? exports.DEFAULT_SOURCE_CHUNK_BYTES;
     let native;
     try {
+        if (!Number.isInteger(reader.size) || reader.size < 0) {
+            throw new RangeError(`open: source size must be a non-negative integer, got ${reader.size}`);
+        }
+        if (!Number.isSafeInteger(chunkBytes) || chunkBytes <= 0) {
+            throw new RangeError(`open: chunkBytes must be a positive integer, got ${chunkBytes}`);
+        }
+        if (chunkBytes % 64 !== 0) {
+            throw new RangeError(`open: chunkBytes must be a multiple of 64, got ${chunkBytes}`);
+        }
         native = (0, native_1.open)({
             size: reader.size,
-            chunkBytes: reader.chunkBytes,
-            read: async ({ offset, buf }) => reader.read(offset, buf),
-        }, {
-            maxResidentChunks: options?.maxResidentChunks,
+            chunkBytes,
+            indexCacheEntries,
+            objectMemberCap,
+            arrayIndexInterval,
+            read: async ({ offset, length }) => reader.read(offset, length),
         });
     }
     catch (err) {
-        await closeReader(reader);
+        // Don't let a failing cleanup mask the original open error; attach it as cause.
+        try {
+            await closeReader(reader);
+        }
+        catch (closeErr) {
+            if (err instanceof Error)
+                err.cause ??= closeErr;
+        }
         throw err;
     }
-    let closed = false;
+    const state = { closed: false };
     const close = async () => {
-        if (closed)
+        if (state.closed)
             return;
-        closed = true;
+        state.closed = true;
         await closeReader(reader);
     };
-    return Object.assign(wrap(native), {
+    return Object.assign(wrap(native, state), {
         close,
         [Symbol.asyncDispose]: close,
     });
@@ -47,41 +75,142 @@ async function closeReader(reader) {
     if (reader.close)
         await reader.close();
 }
-function wrap(native) {
+/** Sentinel the native layer prefixes onto shape-contradiction errors (see
+ *  `session.rs` `SessionError::Path`). */
+const NATIVE_PATH_ERROR_PREFIX = 'bote.PathError: ';
+/** Rethrow a native shape-contradiction error as a `PathError` carrying the
+ *  caller's path; pass anything else through unchanged. */
+function asPathError(err, path) {
+    if (err instanceof Error && !(err instanceof validate_ts_1.PathError) && err.message.startsWith(NATIVE_PATH_ERROR_PREFIX)) {
+        return new validate_ts_1.PathError(err.message.slice(NATIVE_PATH_ERROR_PREFIX.length), path);
+    }
+    return err;
+}
+/** Throw a uniform error for any operation on a closed cursor, so use-after-close
+ *  is one defined contract regardless of source (some readers' reads keep working
+ *  after close, others throw an opaque I/O error). */
+function ensureOpen(state) {
+    if (state.closed)
+        throw new Error('bote: cursor is closed');
+}
+function wrap(native, state) {
     const cursor = {
-        get key() {
-            return native.key;
+        async hop(...path) {
+            ensureOpen(state);
+            (0, path_ts_1.validatePath)(path);
+            let child;
+            try {
+                child = await native.hop(path);
+            }
+            catch (err) {
+                throw asPathError(err, path);
+            }
+            return child ? wrap(child, state) : null;
         },
-        async has(pointer, schema) {
+        async has(...args) {
+            ensureOpen(state);
+            const { path, tail: schema } = (0, args_ts_1.splitArgs)(args);
+            if (schema !== undefined && !(0, args_ts_1.isSchema)(schema)) {
+                throw new TypeError('has: expected a Standard Schema as the trailing argument');
+            }
             if (!schema)
-                return native.has(pointer);
-            if (!(await native.has(pointer)))
+                return native.has(path);
+            if (!(await native.has(path)))
                 return false;
-            const result = await schema['~standard'].validate(await native.get(pointer));
-            return result.issues === undefined;
+            const result = await (0, validate_ts_1.validateItem)(schema, await native.get(path), path, 'skip');
+            return !('skip' in result);
         },
-        async get(pointer, schema) {
-            const value = await native.get(pointer);
-            return schema ? (0, validate_ts_1.runStandardSchema)(schema, value, pointer) : value;
-        },
-        iter(pointer, schema) {
-            const inner = native.iter(pointer);
+        async get(...args) {
+            ensureOpen(state);
+            const { path, tail: schema } = (0, args_ts_1.splitArgs)(args);
+            if (schema !== undefined && !(0, args_ts_1.isSchema)(schema)) {
+                throw new TypeError('get: expected a Standard Schema as the trailing argument');
+            }
+            let value;
+            try {
+                value = await native.get(path);
+            }
+            catch (err) {
+                throw asPathError(err, path);
+            }
             if (!schema)
-                return inner;
+                return value;
+            return (0, validate_ts_1.runStandardSchema)(schema, value, path);
+        },
+        async count(...path) {
+            ensureOpen(state);
+            (0, path_ts_1.validatePath)(path);
+            try {
+                return await native.count(path);
+            }
+            catch (err) {
+                throw asPathError(err, path);
+            }
+        },
+        iter(...args) {
+            ensureOpen(state);
+            const { path, tail } = (0, args_ts_1.splitArgs)(args);
+            const { schema, select, batch, onInvalid, withIndex } = (0, args_ts_1.normalizeIterTail)(tail);
+            if (batch !== undefined && (!Number.isInteger(batch) || batch <= 0 || batch > exports.MAX_ITER_BATCH)) {
+                throw new RangeError(`iter: batch must be an integer in 1..=${exports.MAX_ITER_BATCH}, got ${batch}`);
+            }
+            if (withIndex !== undefined && typeof withIndex !== 'boolean') {
+                throw new TypeError(`iter: withIndex must be a boolean, got ${typeof withIndex}`);
+            }
+            if (onInvalid !== undefined && onInvalid !== 'throw' && onInvalid !== 'skip') {
+                throw new RangeError(`iter: onInvalid must be "throw" or "skip", got ${JSON.stringify(onInvalid)}`);
+            }
+            const resolvedBatch = batch ?? exports.DEFAULT_ITER_BATCH;
+            const selectIr = select !== undefined ? (0, args_ts_1.serializeSelect)(select) : undefined;
+            const inner = native.iter(path, { selectIr, batch: resolvedBatch, withKey: withIndex });
+            if (!schema) {
+                return {
+                    async *[Symbol.asyncIterator]() {
+                        try {
+                            for await (const b of inner)
+                                yield b;
+                        }
+                        catch (err) {
+                            throw asPathError(err, path);
+                        }
+                    },
+                };
+            }
+            const policy = onInvalid ?? 'throw';
             return {
                 async *[Symbol.asyncIterator]() {
                     let i = 0;
-                    for await (const v of inner) {
-                        yield await (0, validate_ts_1.runStandardSchema)(schema, v, `${pointer}/${i++}`);
+                    try {
+                        for await (const b of inner) {
+                            const out = [];
+                            for (const v of b) {
+                                const value = withIndex ? v[1] : v;
+                                const result = await (0, validate_ts_1.validateItem)(schema, value, [...path, i++], policy);
+                                if ('skip' in result)
+                                    continue;
+                                out.push(withIndex ? [v[0], result.value] : result.value);
+                            }
+                            yield out;
+                        }
+                    }
+                    catch (err) {
+                        throw asPathError(err, path);
                     }
                 },
             };
         },
-        walk(pointer) {
+        walk(...path) {
+            ensureOpen(state);
+            (0, path_ts_1.validatePath)(path);
             return {
                 async *[Symbol.asyncIterator]() {
-                    for await (const child of native.walk(pointer)) {
-                        yield wrap(child);
+                    try {
+                        for await (const [key, child] of native.walk(path)) {
+                            yield [key, wrap(child, state)];
+                        }
+                    }
+                    catch (err) {
+                        throw asPathError(err, path);
                     }
                 },
             };

package/dist/path.d.ts

@@ -0,0 +1,5 @@
+import type { Segment } from './validate.ts';
+/** Upper bound on numeric segments (napi takes them as `u32`). 2^32 - 1
+ *  comfortably covers any in-memory JSON array. */
+export declare const MAX_ARRAY_INDEX = 4294967295;
+export declare function validatePath(path: readonly unknown[]): asserts path is readonly Segment[];

package/dist/path.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MAX_ARRAY_INDEX = void 0;
+exports.validatePath = validatePath;
+/** Upper bound on numeric segments (napi takes them as `u32`). 2^32 - 1
+ *  comfortably covers any in-memory JSON array. */
+exports.MAX_ARRAY_INDEX = 0xffffffff;
+function validatePath(path) {
+    for (let i = 0; i < path.length; i++) {
+        const s = path[i];
+        if (typeof s === 'string')
+            continue;
+        if (typeof s === 'number' && Number.isInteger(s) && s >= 0 && s <= exports.MAX_ARRAY_INDEX)
+            continue;
+        throw new TypeError(`path segment ${i}: expected string or non-negative integer (<= ${exports.MAX_ARRAY_INDEX}), got ${describeBadSegment(s)}`);
+    }
+}
+function describeBadSegment(s) {
+    if (typeof s === 'number')
+        return `${s}`;
+    if (s === null)
+        return 'null';
+    return typeof s;
+}

package/dist/sources.d.ts

@@ -10,21 +10,17 @@ export interface SourceReader {
     /** Preferred read granularity in bytes. Must be a non-zero multiple of 64. */
     readonly chunkBytes?: number;
     /**
-     * Fill `buf` with up to `buf.byteLength` bytes starting at `offset` and
-     * resolve with the number of bytes written. The implementation must not
-     * retain a reference to `buf` or read from it after the returned promise
-     * resolves: `buf` is a view over native-owned memory whose lifetime ends
-     * once the promise settles.
+     * Read up to `length` bytes starting at `offset` and resolve with the
+     * bytes read. The returned `Uint8Array`'s `.byteLength` is the actual
+     * count, which must be `<= length`.
      */
-    read(offset: number, buf: Uint8Array): Promise<number>;
+    read(offset: number, length: number): Promise<Uint8Array>;
     /** Release resources held by the reader. Driven once by the `open()` lifecycle. */
     close?(): Promise<void> | void;
 }
 /**
- * Describes how to obtain a seekable byte stream. Construction is cheap and
- * synchronous - no I/O happens until `open()` runs, which the top-level
- * `open()` API drives. Provide your own object implementing this interface to
- * plug in custom backends.
+ * Describes how to obtain a seekable byte stream. Provide your own object implementing
+ * this interface to plug in custom backends.
  */
 export interface Source {
     /** Acquire the stream. Resolves to a `SourceReader` that owns any underlying resources. */

package/dist/sources.js

@@ -17,13 +17,7 @@ function fromBuffer(buf, options) {
         open: () => Promise.resolve({
             size: view.byteLength,
             chunkBytes,
-            read: async (offset, dst) => {
-                const end = Math.min(offset + dst.byteLength, view.byteLength);
-                const n = Math.max(0, end - offset);
-                if (n > 0)
-                    dst.set(view.subarray(offset, end));
-                return n;
-            },
+            read: (offset, length) => Promise.resolve(view.subarray(offset, Math.min(offset + length, view.byteLength))),
         }),
     };
 }
@@ -37,9 +31,16 @@ function fromFile(path, options) {
             return {
                 size: stat.size,
                 chunkBytes,
-                read: async (offset, dst) => {
-                    const { bytesRead } = await handle.read(dst, 0, dst.byteLength, offset);
-                    return bytesRead;
+                read: async (offset, length) => {
+                    const buf = Buffer.allocUnsafe(length);
+                    let filled = 0;
+                    while (filled < length) {
+                        const { bytesRead } = await handle.read(buf, filled, length - filled, offset + filled);
+                        if (bytesRead === 0)
+                            break;
+                        filled += bytesRead;
+                    }
+                    return buf.subarray(0, filled);
                 },
                 close: async () => {
                     if (closed)
@@ -85,17 +86,15 @@ function fromHttpRange(url, options) {
             return {
                 size,
                 chunkBytes,
-                read: async (offset, dst) => {
+                read: async (offset, length) => {
                     // HTTP ranges are inclusive on both ends.
-                    const end = Math.min(offset + dst.byteLength, size) - 1;
+                    const end = Math.min(offset + length, size) - 1;
                     const headers = new Headers(init?.headers);
                     headers.set('Range', `bytes=${offset}-${end}`);
                     headers.set('Accept-Encoding', 'identity');
                     const res = await fetch(url, { ...init, headers, method: 'GET', signal: controller.signal });
                     if (res.status === 206) {
-                        const body = new Uint8Array(await res.arrayBuffer());
-                        dst.set(body);
-                        return body.byteLength;
+                        return new Uint8Array(await res.arrayBuffer());
                     }
                     // A 200 means the server ignored our Range request and returned the full
                     // body. We throw here since the point of using ranges is to not have to

package/dist/validate.d.ts

@@ -1,8 +1,20 @@
 import type { StandardSchemaV1 } from '@standard-schema/spec';
 export type { StandardSchemaV1 };
+export type Segment = string | number;
+export type Path = readonly Segment[];
 export declare class ValidationError extends Error {
     readonly issues: readonly StandardSchemaV1.Issue[];
-    readonly pointer: string;
-    constructor(issues: readonly StandardSchemaV1.Issue[], pointer: string);
+    readonly path: Path;
+    constructor(issues: readonly StandardSchemaV1.Issue[], path: Path);
 }
-export declare function runStandardSchema<O>(schema: StandardSchemaV1<unknown, O>, value: unknown, pointer: string): Promise<O>;
+export declare class PathError extends Error {
+    readonly path: Path;
+    constructor(reason: string, path: Path);
+}
+export declare function runStandardSchema<O>(schema: StandardSchemaV1<unknown, O>, value: unknown, path: Path): Promise<O>;
+export declare function validateItem<O>(schema: StandardSchemaV1<unknown, O>, value: unknown, path: Path, onInvalid: 'throw' | 'skip'): Promise<{
+    skip: true;
+} | {
+    value: O;
+}>;
+export declare function formatPath(path: Path): string;

package/dist/validate.js

@@ -1,21 +1,60 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ValidationError = void 0;
+exports.PathError = exports.ValidationError = void 0;
 exports.runStandardSchema = runStandardSchema;
+exports.validateItem = validateItem;
+exports.formatPath = formatPath;
 class ValidationError extends Error {
     issues;
-    pointer;
-    constructor(issues, pointer) {
-        super(`bote: schema validation failed at ${pointer || '/'}: ${issues[0]?.message ?? 'unknown'}`);
+    path;
+    constructor(issues, path) {
+        super(`bote: schema validation failed at ${formatPath(path)}: ${issues[0]?.message ?? 'unknown'}`);
         this.name = 'ValidationError';
         this.issues = issues;
-        this.pointer = pointer;
+        this.path = path;
     }
 }
 exports.ValidationError = ValidationError;
-async function runStandardSchema(schema, value, pointer) {
+class PathError extends Error {
+    path;
+    constructor(reason, path) {
+        super(`bote: cannot resolve ${formatPath(path)}: ${reason}`);
+        this.name = 'PathError';
+        this.path = path;
+    }
+}
+exports.PathError = PathError;
+async function runStandardSchema(schema, value, path) {
     const result = await schema['~standard'].validate(value);
     if (result.issues)
-        throw new ValidationError(result.issues, pointer);
+        throw new ValidationError(result.issues, path);
     return result.value;
 }
+async function validateItem(schema, value, path, onInvalid) {
+    const result = await schema['~standard'].validate(value);
+    if (result.issues) {
+        if (onInvalid === 'skip')
+            return { skip: true };
+        throw new ValidationError(result.issues, path);
+    }
+    return { value: result.value };
+}
+function formatPath(path) {
+    if (path.length === 0)
+        return '(root)';
+    let out = '';
+    for (let i = 0; i < path.length; i++) {
+        const seg = path[i];
+        if (typeof seg === 'number') {
+            out += `[${seg}]`;
+            continue;
+        }
+        if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(seg)) {
+            out += i === 0 ? seg : `.${seg}`;
+        }
+        else {
+            out += `[${JSON.stringify(seg)}]`;
+        }
+    }
+    return out;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botejs/core",
-  "version": "0.1.4",
+  "version": "0.3.0",
   "license": "MIT",
   "repository": {
     "type": "git",

package/dist/pointer.d.ts

@@ -1,5 +0,0 @@
-type ValidateTokenChars<S extends string> = S extends `${string}~${infer Rest}` ? Rest extends `0${infer After}` | `1${infer After}` ? ValidateTokenChars<After> : false : true;
-type ValidateTokens<S extends string> = S extends `${infer Token}/${infer Rest}` ? ValidateTokenChars<Token> extends true ? ValidateTokens<Rest> : false : ValidateTokenChars<S>;
-type IsJsonPointer<S extends string> = S extends '' ? true : S extends `/${infer Rest}` ? ValidateTokens<Rest> : false;
-export type JsonPointer<S extends string> = IsJsonPointer<S> extends true ? S : `Error: invalid JSON pointer "${S}"`;
-export {};

package/dist/pointer.js

@@ -1,3 +0,0 @@
-"use strict";
-// RFC 6901 JSON Pointer Static Typing Validator
-Object.defineProperty(exports, "__esModule", { value: true });