@botejs/core 0.3.0 → 0.5.0

package/README.md

@@ -24,43 +24,48 @@ here's a run (Apple M1 Pro 2021, ~500MB JSON array file, cold-cache, default set
 
 | operation      | approach   |      time | js heap peak Δ | rust heap peak |
 | -------------- | ---------- | --------: | -------------: | -------------: |
-| items[0]       | JSON.parse | 616.02 ms |        1.03 GB |            n/a |
-| items[535399]  | JSON.parse | 604.63 ms |        1.03 GB |            n/a |
-| items[1070797] | JSON.parse | 600.68 ms |        1.03 GB |            n/a |
-| items[0]       | bote       | 527.80 µs |       291.6 KB |       130.4 KB |
-| items[535399]  | bote       | 187.24 ms |       742.3 KB |        36.7 MB |
-| items[1070797] | bote       | 371.61 ms |       828.7 KB |        37.1 MB |
+| items[0]       | JSON.parse |    1.81 s |        1.21 GB |            n/a |
+| items[535399]  | JSON.parse |    1.74 s |        1.21 GB |            n/a |
+| items[1070797] | JSON.parse |    1.74 s |        1.21 GB |            n/a |
+| items[0]       | bote       |   1.29 ms |        63.3 KB |       130.8 KB |
+| items[535399]  | bote       | 193.49 ms |       191.5 KB |        36.7 MB |
+| items[1070797] | bote       | 379.98 ms |       189.8 KB |        37.2 MB |
 
 ## array access
 
-`iter` streams the elements of an array at a path, **a batch at a time**, so you never hold the whole collection in memory and not wait for the heat death of the universe if this yielded individually. each `for await` step yields an array of items (use `walk` to step over the members of an object):
+`iter` streams the children of a container at a path **one item at a time**, so you never hold the whole collection in memory. it works on either kind: array elements or object member values. each `for await` step yields a single item:
 
 ```ts
 // e.g. [{ id: 'user-1' }, { id: 'user-2' }, ...]
 await using cursor = await open(fromFile('./users.json'))
 
 // root is an array
-for await (const users of cursor.iter()) {
+for await (const user of cursor.iter()) {
+  console.log(user)
+}
+```
+
+the item loop is the ergonomic default; it costs a flat ~10% over a full walk. for hot paths, `.raw()` hands back the raw fetch arrays with no per-item tax (the `batch` option sets their size and the memory bound):
+
+```ts
+for await (const users of cursor.iter().raw()) {
   for (const user of users) {
     console.log(user)
   }
 }
 ```
 
-pass an options object as the last argument to tune what comes back: `batch`, `select`, `schema`, `onInvalid`, and `withIndex`. if you want to know more of the options, see [`arrays.js`](./examples/arrays.js).
-
 ## object access
 
-`walk` steps over the members of an object at a path, yielding a **`[key, cursor]`** pair per member. the key is the member name, the cursor is anchored at its value. each child cursor is first-class: it outlives the loop and can be `walk`ed again, which is what lets you descend a tree of unknown depth.
+`iter` over an object yields its **member values** in document order. add `withKey: true` to get **`[key, value]`** pairs instead, where `key` is the member name (for an array, `key` is the element's index). streamed either way, so a million-member object never lands on the heap at once:
 
 ```ts
 // e.g. { alice: { role: 'admin' }, bob: { role: 'guest' }, ... }
 await using cursor = await open(fromFile('./accounts.json'))
 
-for await (const [name, account] of cursor.walk()) {
-  // name is the member name ('alice', 'bob', ...)
-  const role = await account.get('role')
-  console.log(`${name}: ${role}`)
+for await (const [name, account] of cursor.iter({ withKey: true })) {
+  // name is the member name ('alice', 'bob', ...); account is its value
+  console.log(`${name}: ${account.role}`)
 }
 ```
 
@@ -77,8 +82,8 @@ await using cursor = await open(fromFile('./report.json'))
 const section = await cursor.hop('report', 'sections', 0)
 if (section) {
   console.log(await section.count('rows'))
-  for await (const rows of section.iter('rows')) {
-    console.log(rows)
+  for await (const row of section.iter('rows')) {
+    console.log(row)
   }
 }
 ```
@@ -105,12 +110,14 @@ const cursor = await open(fromFile('./users.json'))
 // name: string
 const name = await cursor.get('users', 1000, 'name', User.shape.name)
 
-for await (const users of cursor.iter('users', User)) {
-  // user: User[]
-  const emails = users.map((user) => user.email)
-  await sendNewsletter(emails)
+let emails: string[] = []
+// .raw() to hand each fetch's worth of recipients to the batched API at once
+for await (const user of cursor.iter('users', User)) {
+  // user: User
+  emails.push(user.email)
 }
 
+await sendNewsletter(emails)
 await cursor.close()
 ```
 

package/dist/args.d.ts

@@ -1,15 +1,20 @@
 import type { Path, Segment, StandardSchemaV1 } from './validate.ts';
 export interface IterOptions {
     select?: Segment | Path | Record<string, Segment | Path>;
-    /** How many items are yielded per batch. Higher is faster, but takes more memory to materialise those items. */
+    /** How many items cross the native boundary per fetch, which also bounds the
+     *  resident materialization window (the memory knob) and sets the array size
+     *  yielded by `IterStream.raw()`. The default item loop drains each fetch
+     *  one item at a time, so this doesn't change what item iteration yields, only
+     *  how much is fetched and held at once. Higher is faster but holds more in
+     *  memory. */
     batch?: number;
     /** Validate each yielded item against this schema (after `select`). */
     schema?: StandardSchemaV1;
     /** Policy for items failing `schema`. Default `'throw'`; `'skip'` drops them. */
     onInvalid?: 'throw' | 'skip';
-    /** Yield `[index, value]` tuples instead of bare values, where `index` is
-     *  the zero-based position of the element in the source array. */
-    withIndex?: boolean;
+    /** Yield `[key, value]` tuples instead of bare values. `key` is the member
+     *  name for objects and the zero-based index for arrays. */
+    withKey?: boolean;
 }
 export type VariadicPathArgs<TTail> = [...Segment[]] | [...Segment[], TTail];
 export declare function splitArgs<TTail>(args: VariadicPathArgs<TTail>): {

package/dist/args.js

@@ -1,11 +1,5 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.splitArgs = splitArgs;
-exports.isSchema = isSchema;
-exports.normalizeIterTail = normalizeIterTail;
-exports.serializeSelect = serializeSelect;
-const path_ts_1 = require("./path.js");
-function splitArgs(args) {
+import { validatePath } from "./path.js";
+export function splitArgs(args) {
     let pathArgs;
     let tail;
     if (args.length === 0) {
@@ -23,27 +17,27 @@ function splitArgs(args) {
             tail = undefined;
         }
     }
-    (0, path_ts_1.validatePath)(pathArgs);
+    validatePath(pathArgs);
     return { path: pathArgs, tail };
 }
-function isSchema(value) {
+export function isSchema(value) {
     return typeof value === 'object' && value !== null && '~standard' in value;
 }
-function normalizeIterTail(tail) {
+export function normalizeIterTail(tail) {
     if (!tail)
         return {};
     if (isSchema(tail))
         return { schema: tail };
     return tail;
 }
-function serializeSelect(select) {
+export function serializeSelect(select) {
     if (typeof select === 'string' || typeof select === 'number') {
         const one = [select];
-        (0, path_ts_1.validatePath)(one);
+        validatePath(one);
         return JSON.stringify({ one });
     }
     if (Array.isArray(select)) {
-        (0, path_ts_1.validatePath)(select);
+        validatePath(select);
         if (select.length === 0) {
             throw new RangeError('iter: select sub-path must have at least one segment');
         }
@@ -57,7 +51,7 @@ function serializeSelect(select) {
         if (!Array.isArray(path)) {
             throw new TypeError(`iter: select field ${JSON.stringify(k)} must be a segment or path, got ${describeSelect(sub)}`);
         }
-        (0, path_ts_1.validatePath)(path);
+        validatePath(path);
         if (path.length === 0) {
             throw new RangeError(`iter: select field ${JSON.stringify(k)} sub-path must have at least one segment`);
         }

package/dist/cursor.d.ts

@@ -0,0 +1,52 @@
+import type { Cursor as NativeCursor } from '@botejs/native';
+import { type IterStream } from './stream.ts';
+import { type Path, type Segment, type StandardSchemaV1 } from './validate.ts';
+import { type IterOptions } from './args.ts';
+type InferOutput<Sch> = Sch extends StandardSchemaV1<unknown, infer O> ? O : never;
+type SelectMapShape<S> = {
+    -readonly [K in keyof S]: unknown;
+};
+export type IterKey = string | number;
+export declare const DEFAULT_ITER_BATCH = 1000;
+export declare const MAX_ITER_BATCH = 1000000;
+export interface Cursor {
+    hop(...path: Segment[]): Promise<Cursor | null>;
+    has(...path: Segment[]): Promise<boolean>;
+    has(...args: [...Segment[], StandardSchemaV1]): Promise<boolean>;
+    get(...path: Segment[]): Promise<unknown>;
+    get<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): Promise<InferOutput<Sch>>;
+    count(...path: Segment[]): Promise<number>;
+    iter(...path: Segment[]): IterStream<unknown>;
+    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): IterStream<InferOutput<Sch>>;
+    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
+        withKey: true;
+        schema: Sch;
+    }]): IterStream<[IterKey, InferOutput<Sch>]>;
+    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
+        schema: Sch;
+    }]): IterStream<InferOutput<Sch>>;
+    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
+        withKey: true;
+        select: S;
+    }]): IterStream<[IterKey, SelectMapShape<S>]>;
+    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
+        select: S;
+    }]): IterStream<SelectMapShape<S>>;
+    iter(...args: [...Segment[], IterOptions & {
+        withKey: true;
+    }]): IterStream<[IterKey, unknown]>;
+    iter(...args: [...Segment[], IterOptions]): IterStream<unknown>;
+}
+export interface RootCursor extends Cursor, AsyncDisposable {
+    /** Close the underlying source. Idempotent. */
+    close(): Promise<void>;
+}
+export type CursorState = {
+    closed: boolean;
+};
+/** Throw a uniform error for any operation on a closed cursor, so use-after-close
+ *  is one defined contract regardless of source (some readers' reads keep working
+ *  after close, others throw an opaque I/O error). */
+export declare function ensureOpen(state: CursorState): void;
+export declare function wrap(native: NativeCursor, state: CursorState): Cursor;
+export {};

package/dist/cursor.js

@@ -0,0 +1,119 @@
+import { validatePath } from "./path.js";
+import { parseValue, deserializeError } from "./decode.js";
+import { makeStream } from "./stream.js";
+import { runStandardSchema, validateItem } from "./validate.js";
+import { splitArgs, isSchema, serializeSelect, normalizeIterTail, } from "./args.js";
+export const DEFAULT_ITER_BATCH = 1000;
+export const MAX_ITER_BATCH = 1_000_000;
+/** Throw a uniform error for any operation on a closed cursor, so use-after-close
+ *  is one defined contract regardless of source (some readers' reads keep working
+ *  after close, others throw an opaque I/O error). */
+export function ensureOpen(state) {
+    if (state.closed)
+        throw new Error('bote: cursor is closed');
+}
+export function wrap(native, state) {
+    const cursor = {
+        async hop(...path) {
+            ensureOpen(state);
+            validatePath(path);
+            let child;
+            try {
+                child = await native.hop(path);
+            }
+            catch (err) {
+                throw deserializeError(err, path);
+            }
+            return child ? wrap(child, state) : null;
+        },
+        async has(...args) {
+            ensureOpen(state);
+            const { path, tail: schema } = splitArgs(args);
+            if (schema !== undefined && !isSchema(schema)) {
+                throw new TypeError('has: expected a Standard Schema as the trailing argument');
+            }
+            if (!schema)
+                return native.has(path);
+            if (!(await native.has(path)))
+                return false;
+            const text = await native.get(path);
+            const value = text === undefined ? undefined : parseValue(text, path);
+            const result = await validateItem(schema, value, path, 'skip');
+            return !('skip' in result);
+        },
+        async get(...args) {
+            ensureOpen(state);
+            const { path, tail: schema } = splitArgs(args);
+            if (schema !== undefined && !isSchema(schema)) {
+                throw new TypeError('get: expected a Standard Schema as the trailing argument');
+            }
+            let value;
+            try {
+                const text = await native.get(path);
+                value = text === undefined ? undefined : parseValue(text, path);
+            }
+            catch (err) {
+                throw deserializeError(err, path);
+            }
+            if (!schema)
+                return value;
+            return runStandardSchema(schema, value, path);
+        },
+        async count(...path) {
+            ensureOpen(state);
+            validatePath(path);
+            try {
+                return await native.count(path);
+            }
+            catch (err) {
+                throw deserializeError(err, path);
+            }
+        },
+        iter(...args) {
+            ensureOpen(state);
+            const { path, tail } = splitArgs(args);
+            const { schema, select, batch, onInvalid, withKey } = normalizeIterTail(tail);
+            if (batch !== undefined && (!Number.isInteger(batch) || batch <= 0 || batch > MAX_ITER_BATCH)) {
+                throw new RangeError(`iter: batch must be an integer in 1..=${MAX_ITER_BATCH}, got ${batch}`);
+            }
+            if (withKey !== undefined && typeof withKey !== 'boolean') {
+                throw new TypeError(`iter: withKey must be a boolean, got ${typeof withKey}`);
+            }
+            if (onInvalid !== undefined && onInvalid !== 'throw' && onInvalid !== 'skip') {
+                throw new RangeError(`iter: onInvalid must be "throw" or "skip", got ${JSON.stringify(onInvalid)}`);
+            }
+            const resolvedBatch = batch ?? DEFAULT_ITER_BATCH;
+            const selectIr = select !== undefined ? serializeSelect(select) : undefined;
+            const wantKey = withKey ?? false;
+            const nativeWithKey = wantKey || schema !== undefined;
+            const inner = native.iter(path, { selectIr, batch: resolvedBatch, withKey: nativeWithKey });
+            if (!schema) {
+                return nativeStream(inner, path, resolvedBatch, (raw) => parseValue(raw, path));
+            }
+            const policy = onInvalid ?? 'throw';
+            return nativeStream(inner, path, resolvedBatch, async (raw) => {
+                const out = [];
+                for (const [key, value] of parseValue(raw, path)) {
+                    const result = await validateItem(schema, value, [...path, key], policy);
+                    if ('skip' in result)
+                        continue;
+                    out.push(wantKey ? [key, result.value] : result.value);
+                }
+                return out;
+            });
+        },
+    };
+    return cursor;
+}
+function nativeStream(inner, path, batchSize, mapBatch) {
+    async function* batches() {
+        try {
+            for await (const raw of inner)
+                yield await mapBatch(raw);
+        }
+        catch (err) {
+            throw deserializeError(err, path);
+        }
+    }
+    return makeStream(batches, batchSize);
+}

package/dist/decode.d.ts

@@ -0,0 +1,3 @@
+import { type Path } from './validate.ts';
+export declare function deserializeError(err: unknown, path: Path): unknown;
+export declare function parseValue(text: string, path: Path): unknown;

package/dist/decode.js

@@ -0,0 +1,20 @@
+import { PathError, formatPath } from "./validate.js";
+const NATIVE_PATH_ERROR = /^bote:path:([a-z_]+)(?::(\d+))?$/;
+export function deserializeError(err, path) {
+    if (err instanceof Error && !(err instanceof PathError)) {
+        const match = NATIVE_PATH_ERROR.exec(err.message);
+        if (match) {
+            const segment = match[2] === undefined ? undefined : Number(match[2]);
+            return new PathError(path, match[1], segment);
+        }
+    }
+    return err;
+}
+export function parseValue(text, path) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw new Error(`bote: malformed JSON value at ${formatPath(path)}`);
+    }
+}

package/dist/index.d.ts

@@ -1,4 +1,6 @@
 export { type IterOptions } from './args.ts';
-export { ValidationError, PathError, formatPath, type Path, type Segment, type StandardSchemaV1, } from './validate.ts';
-export { open, DEFAULT_ITER_BATCH, MAX_ITER_BATCH, type Cursor, type RootCursor, type OpenOptions, type WalkEntry, type IterIndex as IterKey, } from './open.ts';
+export { ValidationError, PathError, formatPath, type Path, type PathFaultCode, type Segment, type StandardSchemaV1, } from './validate.ts';
+export { DEFAULT_ITER_BATCH, MAX_ITER_BATCH, type Cursor, type RootCursor, type IterKey } from './cursor.ts';
 export { fromBuffer, fromFile, fromHttpRange, type FactoryOptions, type Source, type SourceReader, type HttpRangeOptions, } from './sources.ts';
+export { type IterStream } from './stream.ts';
+export { open, type OpenOptions } from './open.ts';

package/dist/index.js

@@ -1,21 +1,4 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fromHttpRange = exports.fromFile = exports.fromBuffer = exports.MAX_ITER_BATCH = exports.DEFAULT_ITER_BATCH = exports.formatPath = exports.PathError = exports.ValidationError = void 0;
-// Node 18 and Node 20.3 predate `Symbol.asyncDispose`; mirror what TS emits for
-// `await using` so the well-known symbol is available across our engine range.
-if (!Symbol.asyncDispose) {
-    ;
-    Symbol.asyncDispose = Symbol.for('Symbol.asyncDispose');
-}
-var validate_ts_1 = require("./validate.js");
-Object.defineProperty(exports, "ValidationError", { enumerable: true, get: function () { return validate_ts_1.ValidationError; } });
-Object.defineProperty(exports, "PathError", { enumerable: true, get: function () { return validate_ts_1.PathError; } });
-Object.defineProperty(exports, "formatPath", { enumerable: true, get: function () { return validate_ts_1.formatPath; } });
-var open_ts_1 = require("./open.js");
-Object.defineProperty(exports, "open", { enumerable: true, get: function () { return open_ts_1.open; } });
-Object.defineProperty(exports, "DEFAULT_ITER_BATCH", { enumerable: true, get: function () { return open_ts_1.DEFAULT_ITER_BATCH; } });
-Object.defineProperty(exports, "MAX_ITER_BATCH", { enumerable: true, get: function () { return open_ts_1.MAX_ITER_BATCH; } });
-var sources_ts_1 = require("./sources.js");
-Object.defineProperty(exports, "fromBuffer", { enumerable: true, get: function () { return sources_ts_1.fromBuffer; } });
-Object.defineProperty(exports, "fromFile", { enumerable: true, get: function () { return sources_ts_1.fromFile; } });
-Object.defineProperty(exports, "fromHttpRange", { enumerable: true, get: function () { return sources_ts_1.fromHttpRange; } });
+export { ValidationError, PathError, formatPath, } from "./validate.js";
+export { DEFAULT_ITER_BATCH, MAX_ITER_BATCH } from "./cursor.js";
+export { fromBuffer, fromFile, fromHttpRange, } from "./sources.js";
+export { open } from "./open.js";

package/dist/open.d.ts

@@ -1,17 +1,6 @@
+import { type RootCursor } from './cursor.ts';
 import type { Source } from './sources.ts';
-import { type Path, type Segment, type StandardSchemaV1 } from './validate.ts';
-import { type IterOptions } from './args.ts';
-type InferOutput<Sch> = Sch extends StandardSchemaV1<unknown, infer O> ? O : never;
-type SelectMapShape<S> = {
-    -readonly [K in keyof S]: unknown;
-};
-/** Zero-based index of an array element. */
-export type IterIndex = number;
-/** One `walk` step: the member's key paired with a cursor anchored at its value. */
-export type WalkEntry = [key: string, cursor: Cursor];
 export declare const DEFAULT_SOURCE_CHUNK_BYTES: number;
-export declare const DEFAULT_ITER_BATCH = 1000;
-export declare const MAX_ITER_BATCH = 1000000;
 export interface OpenOptions {
     /**
      * Slot budget for the structural-index cache: one slot per cached container
@@ -42,40 +31,6 @@ export interface OpenOptions {
      */
     arrayIndexInterval?: number;
 }
-export interface Cursor {
-    hop(...path: Segment[]): Promise<Cursor | null>;
-    has(...path: Segment[]): Promise<boolean>;
-    has(...args: [...Segment[], StandardSchemaV1]): Promise<boolean>;
-    get(...path: Segment[]): Promise<unknown>;
-    get<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): Promise<InferOutput<Sch>>;
-    count(...path: Segment[]): Promise<number>;
-    iter(...path: Segment[]): AsyncIterable<unknown[]>;
-    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): AsyncIterable<InferOutput<Sch>[]>;
-    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
-        withIndex: true;
-        schema: Sch;
-    }]): AsyncIterable<[IterIndex, InferOutput<Sch>][]>;
-    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
-        schema: Sch;
-    }]): AsyncIterable<InferOutput<Sch>[]>;
-    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
-        withIndex: true;
-        select: S;
-    }]): AsyncIterable<[IterIndex, SelectMapShape<S>][]>;
-    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
-        select: S;
-    }]): AsyncIterable<SelectMapShape<S>[]>;
-    iter(...args: [...Segment[], IterOptions & {
-        withIndex: true;
-    }]): AsyncIterable<[IterIndex, unknown][]>;
-    iter(...args: [...Segment[], IterOptions]): AsyncIterable<unknown[]>;
-    walk(...path: Segment[]): AsyncIterable<WalkEntry>;
-    walk(...path: Segment[]): AsyncIterable<Cursor>;
-}
-export interface RootCursor extends Cursor, AsyncDisposable {
-    /** Close the underlying source. Idempotent. */
-    close(): Promise<void>;
-}
 /**
  * Open a cursor over a seekable source.
  *
@@ -83,4 +38,3 @@ export interface RootCursor extends Cursor, AsyncDisposable {
  * drives the reader's own `close()` exactly once.
  */
 export declare function open(source: Source, options?: OpenOptions): Promise<RootCursor>;
-export {};

package/dist/open.js

@@ -1,21 +1,13 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.MAX_ITER_BATCH = exports.DEFAULT_ITER_BATCH = exports.DEFAULT_SOURCE_CHUNK_BYTES = void 0;
-exports.open = open;
-const native_1 = require("@botejs/native");
-const path_ts_1 = require("./path.js");
-const validate_ts_1 = require("./validate.js");
-const args_ts_1 = require("./args.js");
-exports.DEFAULT_SOURCE_CHUNK_BYTES = 64 * 1024;
-exports.DEFAULT_ITER_BATCH = 1000;
-exports.MAX_ITER_BATCH = 1_000_000;
+import { open as openNative } from '@botejs/native';
+import { wrap } from "./cursor.js";
+export const DEFAULT_SOURCE_CHUNK_BYTES = 64 * 1024;
 /**
  * Open a cursor over a seekable source.
  *
  * The returned `RootCursor` owns the reader: `close()` (or `await using`)
  * drives the reader's own `close()` exactly once.
  */
-async function open(source, options) {
+export async function open(source, options) {
     const { indexCacheEntries, objectMemberCap, arrayIndexInterval } = options ?? {};
     for (const [name, value] of [
         ['indexCacheEntries', indexCacheEntries],
@@ -27,7 +19,7 @@ async function open(source, options) {
         }
     }
     const reader = await source.open();
-    const chunkBytes = reader.chunkBytes ?? exports.DEFAULT_SOURCE_CHUNK_BYTES;
+    const chunkBytes = reader.chunkBytes ?? DEFAULT_SOURCE_CHUNK_BYTES;
     let native;
     try {
         if (!Number.isInteger(reader.size) || reader.size < 0) {
@@ -39,7 +31,7 @@ async function open(source, options) {
         if (chunkBytes % 64 !== 0) {
             throw new RangeError(`open: chunkBytes must be a multiple of 64, got ${chunkBytes}`);
         }
-        native = (0, native_1.open)({
+        native = openNative({
             size: reader.size,
             chunkBytes,
             indexCacheEntries,
@@ -75,146 +67,3 @@ async function closeReader(reader) {
     if (reader.close)
         await reader.close();
 }
-/** Sentinel the native layer prefixes onto shape-contradiction errors (see
- *  `session.rs` `SessionError::Path`). */
-const NATIVE_PATH_ERROR_PREFIX = 'bote.PathError: ';
-/** Rethrow a native shape-contradiction error as a `PathError` carrying the
- *  caller's path; pass anything else through unchanged. */
-function asPathError(err, path) {
-    if (err instanceof Error && !(err instanceof validate_ts_1.PathError) && err.message.startsWith(NATIVE_PATH_ERROR_PREFIX)) {
-        return new validate_ts_1.PathError(err.message.slice(NATIVE_PATH_ERROR_PREFIX.length), path);
-    }
-    return err;
-}
-/** Throw a uniform error for any operation on a closed cursor, so use-after-close
- *  is one defined contract regardless of source (some readers' reads keep working
- *  after close, others throw an opaque I/O error). */
-function ensureOpen(state) {
-    if (state.closed)
-        throw new Error('bote: cursor is closed');
-}
-function wrap(native, state) {
-    const cursor = {
-        async hop(...path) {
-            ensureOpen(state);
-            (0, path_ts_1.validatePath)(path);
-            let child;
-            try {
-                child = await native.hop(path);
-            }
-            catch (err) {
-                throw asPathError(err, path);
-            }
-            return child ? wrap(child, state) : null;
-        },
-        async has(...args) {
-            ensureOpen(state);
-            const { path, tail: schema } = (0, args_ts_1.splitArgs)(args);
-            if (schema !== undefined && !(0, args_ts_1.isSchema)(schema)) {
-                throw new TypeError('has: expected a Standard Schema as the trailing argument');
-            }
-            if (!schema)
-                return native.has(path);
-            if (!(await native.has(path)))
-                return false;
-            const result = await (0, validate_ts_1.validateItem)(schema, await native.get(path), path, 'skip');
-            return !('skip' in result);
-        },
-        async get(...args) {
-            ensureOpen(state);
-            const { path, tail: schema } = (0, args_ts_1.splitArgs)(args);
-            if (schema !== undefined && !(0, args_ts_1.isSchema)(schema)) {
-                throw new TypeError('get: expected a Standard Schema as the trailing argument');
-            }
-            let value;
-            try {
-                value = await native.get(path);
-            }
-            catch (err) {
-                throw asPathError(err, path);
-            }
-            if (!schema)
-                return value;
-            return (0, validate_ts_1.runStandardSchema)(schema, value, path);
-        },
-        async count(...path) {
-            ensureOpen(state);
-            (0, path_ts_1.validatePath)(path);
-            try {
-                return await native.count(path);
-            }
-            catch (err) {
-                throw asPathError(err, path);
-            }
-        },
-        iter(...args) {
-            ensureOpen(state);
-            const { path, tail } = (0, args_ts_1.splitArgs)(args);
-            const { schema, select, batch, onInvalid, withIndex } = (0, args_ts_1.normalizeIterTail)(tail);
-            if (batch !== undefined && (!Number.isInteger(batch) || batch <= 0 || batch > exports.MAX_ITER_BATCH)) {
-                throw new RangeError(`iter: batch must be an integer in 1..=${exports.MAX_ITER_BATCH}, got ${batch}`);
-            }
-            if (withIndex !== undefined && typeof withIndex !== 'boolean') {
-                throw new TypeError(`iter: withIndex must be a boolean, got ${typeof withIndex}`);
-            }
-            if (onInvalid !== undefined && onInvalid !== 'throw' && onInvalid !== 'skip') {
-                throw new RangeError(`iter: onInvalid must be "throw" or "skip", got ${JSON.stringify(onInvalid)}`);
-            }
-            const resolvedBatch = batch ?? exports.DEFAULT_ITER_BATCH;
-            const selectIr = select !== undefined ? (0, args_ts_1.serializeSelect)(select) : undefined;
-            const inner = native.iter(path, { selectIr, batch: resolvedBatch, withKey: withIndex });
-            if (!schema) {
-                return {
-                    async *[Symbol.asyncIterator]() {
-                        try {
-                            for await (const b of inner)
-                                yield b;
-                        }
-                        catch (err) {
-                            throw asPathError(err, path);
-                        }
-                    },
-                };
-            }
-            const policy = onInvalid ?? 'throw';
-            return {
-                async *[Symbol.asyncIterator]() {
-                    let i = 0;
-                    try {
-                        for await (const b of inner) {
-                            const out = [];
-                            for (const v of b) {
-                                const value = withIndex ? v[1] : v;
-                                const result = await (0, validate_ts_1.validateItem)(schema, value, [...path, i++], policy);
-                                if ('skip' in result)
-                                    continue;
-                                out.push(withIndex ? [v[0], result.value] : result.value);
-                            }
-                            yield out;
-                        }
-                    }
-                    catch (err) {
-                        throw asPathError(err, path);
-                    }
-                },
-            };
-        },
-        walk(...path) {
-            ensureOpen(state);
-            (0, path_ts_1.validatePath)(path);
-            return {
-                async *[Symbol.asyncIterator]() {
-                    try {
-                        for await (const [key, child] of native.walk(path)) {
-                            yield [key, wrap(child, state)];
-                        }
-                    }
-                    catch (err) {
-                        throw asPathError(err, path);
-                    }
-                },
-            };
-        },
-    };
-    return cursor;
-}

package/dist/path.js

@@ -1,18 +1,14 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.MAX_ARRAY_INDEX = void 0;
-exports.validatePath = validatePath;
 /** Upper bound on numeric segments (napi takes them as `u32`). 2^32 - 1
  *  comfortably covers any in-memory JSON array. */
-exports.MAX_ARRAY_INDEX = 0xffffffff;
-function validatePath(path) {
+export const MAX_ARRAY_INDEX = 0xffffffff;
+export function validatePath(path) {
     for (let i = 0; i < path.length; i++) {
         const s = path[i];
         if (typeof s === 'string')
             continue;
-        if (typeof s === 'number' && Number.isInteger(s) && s >= 0 && s <= exports.MAX_ARRAY_INDEX)
+        if (typeof s === 'number' && Number.isInteger(s) && s >= 0 && s <= MAX_ARRAY_INDEX)
             continue;
-        throw new TypeError(`path segment ${i}: expected string or non-negative integer (<= ${exports.MAX_ARRAY_INDEX}), got ${describeBadSegment(s)}`);
+        throw new TypeError(`path segment ${i}: expected string or non-negative integer (<= ${MAX_ARRAY_INDEX}), got ${describeBadSegment(s)}`);
     }
 }
 function describeBadSegment(s) {

package/dist/sources.js

@@ -1,16 +1,11 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fromBuffer = fromBuffer;
-exports.fromFile = fromFile;
-exports.fromHttpRange = fromHttpRange;
-const promises_1 = require("node:fs/promises");
+import { open as fsOpen } from 'node:fs/promises';
 /** Default chunk size, in bytes, for in-memory sources. */
 const DEFAULT_BUFFER_CHUNK_BYTES = 4 * 1024;
 /** Default chunk size, in bytes, for local files: matches typical filesystem readahead. */
 const DEFAULT_FILE_CHUNK_BYTES = 64 * 1024;
 /** Default chunk size, in bytes, for HTTP range reads: amortizes RTT across more data. */
 const DEFAULT_URL_CHUNK_BYTES = 256 * 1024;
-function fromBuffer(buf, options) {
+export function fromBuffer(buf, options) {
     const view = buf instanceof Uint8Array ? buf : new Uint8Array(buf);
     const chunkBytes = options?.chunkBytes ?? DEFAULT_BUFFER_CHUNK_BYTES;
     return {
@@ -21,11 +16,11 @@ function fromBuffer(buf, options) {
         }),
     };
 }
-function fromFile(path, options) {
+export function fromFile(path, options) {
     const chunkBytes = options?.chunkBytes ?? DEFAULT_FILE_CHUNK_BYTES;
     return {
         open: async () => {
-            const handle = await (0, promises_1.open)(path, 'r');
+            const handle = await fsOpen(path, 'r');
             const stat = await handle.stat();
             let closed = false;
             return {
@@ -52,7 +47,7 @@ function fromFile(path, options) {
         },
     };
 }
-function fromHttpRange(url, options) {
+export function fromHttpRange(url, options) {
     const init = options?.init;
     const chunkBytes = options?.chunkBytes ?? DEFAULT_URL_CHUNK_BYTES;
     return {

package/dist/stream.d.ts

@@ -0,0 +1,15 @@
+export interface IterStream<T> extends AsyncIterable<T> {
+    raw(): AsyncIterable<T[]>;
+    map<U>(fn: (item: T, index: number) => U | Promise<U>): IterStream<U>;
+    filter<U extends T>(fn: (item: T, index: number) => item is U): IterStream<U>;
+    filter(fn: (item: T, index: number) => boolean | Promise<boolean>): IterStream<T>;
+    take(limit: number): IterStream<T>;
+    drop(limit: number): IterStream<T>;
+    toArray(): Promise<T[]>;
+    forEach(fn: (item: T, index: number) => void | Promise<void>): Promise<void>;
+    reduce<A>(fn: (acc: A, item: T, index: number) => A | Promise<A>, init: A): Promise<A>;
+    find(fn: (item: T, index: number) => boolean | Promise<boolean>): Promise<T | undefined>;
+    some(fn: (item: T, index: number) => boolean | Promise<boolean>): Promise<boolean>;
+    every(fn: (item: T, index: number) => boolean | Promise<boolean>): Promise<boolean>;
+}
+export declare function makeStream<T>(batches: () => AsyncIterable<T[]>, batchSize: number, regroup?: boolean): IterStream<T>;

package/dist/stream.js

@@ -0,0 +1,166 @@
+export function makeStream(batches, batchSize, regroup = false) {
+    const derive = (next) => makeStream(next, batchSize, true);
+    const stream = {
+        [Symbol.asyncIterator]() {
+            return flatten(batches())[Symbol.asyncIterator]();
+        },
+        raw() {
+            return regroup ? regroupBatches(batches(), batchSize) : batches();
+        },
+        map(fn) {
+            return derive(() => mapBatches(batches(), fn));
+        },
+        filter(fn) {
+            return derive(() => filterBatches(batches(), fn));
+        },
+        take(limit) {
+            return derive(() => takeBatches(batches(), limit));
+        },
+        drop(limit) {
+            return derive(() => dropBatches(batches(), limit));
+        },
+        async toArray() {
+            const out = [];
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    out.push(batch[i]);
+                }
+            }
+            return out;
+        },
+        async forEach(fn) {
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    await fn(batch[i], index++);
+                }
+            }
+        },
+        async reduce(fn, init) {
+            let acc = init;
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    acc = await fn(acc, batch[i], index++);
+                }
+            }
+            return acc;
+        },
+        async find(fn) {
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    if (await fn(batch[i], index++)) {
+                        return batch[i];
+                    }
+                }
+            }
+            return undefined;
+        },
+        async some(fn) {
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    if (await fn(batch[i], index++)) {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        },
+        async every(fn) {
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    if (!(await fn(batch[i], index++))) {
+                        return false;
+                    }
+                }
+            }
+            return true;
+        },
+    };
+    return stream;
+}
+async function* flatten(batches) {
+    for await (const batch of batches) {
+        for (let i = 0; i < batch.length; i++) {
+            yield batch[i];
+        }
+    }
+}
+async function* regroupBatches(batches, size) {
+    let buf = [];
+    for await (const batch of batches) {
+        for (let i = 0; i < batch.length; i++) {
+            buf.push(batch[i]);
+            if (buf.length >= size) {
+                yield buf;
+                buf = [];
+            }
+        }
+    }
+    if (buf.length > 0) {
+        yield buf;
+    }
+}
+async function* mapBatches(batches, fn) {
+    let index = 0;
+    for await (const batch of batches) {
+        const out = new Array(batch.length);
+        for (let i = 0; i < batch.length; i++) {
+            const r = fn(batch[i], index++);
+            out[i] = isThenable(r) ? await r : r;
+        }
+        yield out;
+    }
+}
+async function* filterBatches(batches, fn) {
+    let index = 0;
+    for await (const batch of batches) {
+        const out = [];
+        for (let i = 0; i < batch.length; i++) {
+            const item = batch[i];
+            const r = fn(item, index++);
+            if (isThenable(r) ? await r : r) {
+                out.push(item);
+            }
+        }
+        if (out.length > 0) {
+            yield out;
+        }
+    }
+}
+async function* takeBatches(batches, limit) {
+    if (limit <= 0) {
+        return;
+    }
+    let remaining = limit;
+    for await (const batch of batches) {
+        if (batch.length < remaining) {
+            remaining -= batch.length;
+            yield batch;
+            continue;
+        }
+        yield batch.length === remaining ? batch : batch.slice(0, remaining);
+        return;
+    }
+}
+async function* dropBatches(batches, limit) {
+    let remaining = limit;
+    for await (const batch of batches) {
+        if (remaining === 0) {
+            yield batch;
+        }
+        else if (remaining >= batch.length) {
+            remaining -= batch.length;
+        }
+        else {
+            yield batch.slice(remaining);
+            remaining = 0;
+        }
+    }
+}
+function isThenable(value) {
+    return value != null && typeof value.then === 'function';
+}

package/dist/validate.d.ts

@@ -1,5 +1,6 @@
 import type { StandardSchemaV1 } from '@standard-schema/spec';
-export type { StandardSchemaV1 };
+import type { PathFaultCode } from '@botejs/native';
+export type { StandardSchemaV1, PathFaultCode };
 export type Segment = string | number;
 export type Path = readonly Segment[];
 export declare class ValidationError extends Error {
@@ -9,7 +10,9 @@ export declare class ValidationError extends Error {
 }
 export declare class PathError extends Error {
     readonly path: Path;
-    constructor(reason: string, path: Path);
+    /** The fault kind; stable across versions, safe to branch on. */
+    readonly code: PathFaultCode;
+    constructor(path: Path, code: PathFaultCode, segment?: number);
 }
 export declare function runStandardSchema<O>(schema: StandardSchemaV1<unknown, O>, value: unknown, path: Path): Promise<O>;
 export declare function validateItem<O>(schema: StandardSchemaV1<unknown, O>, value: unknown, path: Path, onInvalid: 'throw' | 'skip'): Promise<{

package/dist/validate.js

@@ -1,10 +1,4 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PathError = exports.ValidationError = void 0;
-exports.runStandardSchema = runStandardSchema;
-exports.validateItem = validateItem;
-exports.formatPath = formatPath;
-class ValidationError extends Error {
+export class ValidationError extends Error {
     issues;
     path;
     constructor(issues, path) {
@@ -14,23 +8,33 @@ class ValidationError extends Error {
         this.path = path;
     }
 }
-exports.ValidationError = ValidationError;
-class PathError extends Error {
+/** Human message per fault kind. The native layer ships only the code (and the
+ *  offending `segment` where it matters), so this is the single source of the
+ *  user-facing prose. Keyed by the Rust-generated [`PathFaultCode`]. */
+const PATH_FAULT_MESSAGE = {
+    through_scalar: (segment) => `path traverses a non-container value at segment ${segment}`,
+    wrong_kind: (segment) => `path segment ${segment} does not match the container kind`,
+    scalar_target: () => 'target value is not a container',
+};
+export class PathError extends Error {
     path;
-    constructor(reason, path) {
+    /** The fault kind; stable across versions, safe to branch on. */
+    code;
+    constructor(path, code, segment) {
+        const reason = (PATH_FAULT_MESSAGE[code] ?? (() => code))(segment);
         super(`bote: cannot resolve ${formatPath(path)}: ${reason}`);
         this.name = 'PathError';
         this.path = path;
+        this.code = code;
     }
 }
-exports.PathError = PathError;
-async function runStandardSchema(schema, value, path) {
+export async function runStandardSchema(schema, value, path) {
     const result = await schema['~standard'].validate(value);
     if (result.issues)
         throw new ValidationError(result.issues, path);
     return result.value;
 }
-async function validateItem(schema, value, path, onInvalid) {
+export async function validateItem(schema, value, path, onInvalid) {
     const result = await schema['~standard'].validate(value);
     if (result.issues) {
         if (onInvalid === 'skip')
@@ -39,7 +43,7 @@ async function validateItem(schema, value, path, onInvalid) {
     }
     return { value: result.value };
 }
-function formatPath(path) {
+export function formatPath(path) {
     if (path.length === 0)
         return '(root)';
     let out = '';

package/package.json

@@ -1,12 +1,19 @@
 {
   "name": "@botejs/core",
-  "version": "0.3.0",
+  "version": "0.5.0",
+  "type": "module",
   "license": "MIT",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/jankdc/bote.git",
     "directory": "packages/core"
   },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "files": [
@@ -14,7 +21,7 @@
     "README.md"
   ],
   "engines": {
-    "node": ">= 18.17.0 < 19 || >= 20.3.0 < 21 || >= 21.1.0"
+    "node": ">= 22.18.0"
   },
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
@@ -23,12 +30,12 @@
   "scripts": {
     "build": "tsc",
     "build:debug": "tsc --sourceMap",
-    "test": "node --test --experimental-strip-types --no-warnings=ExperimentalWarning __test__/*.spec.ts",
+    "test": "node --test __test__/*.spec.ts",
     "lint": "oxlint src",
     "prepublishOnly": "cp ../../README.md ./README.md && tsc"
   },
   "dependencies": {
-    "@botejs/native": "workspace:*"
+    "@botejs/native": "^0.5.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",