@botejs/core 0.4.0 → 0.5.0

package/README.md

@@ -24,43 +24,48 @@ here's a run (Apple M1 Pro 2021, ~500MB JSON array file, cold-cache, default set
 
 | operation      | approach   |      time | js heap peak Δ | rust heap peak |
 | -------------- | ---------- | --------: | -------------: | -------------: |
-| items[0]       | JSON.parse | 616.02 ms |        1.03 GB |            n/a |
-| items[535399]  | JSON.parse | 604.63 ms |        1.03 GB |            n/a |
-| items[1070797] | JSON.parse | 600.68 ms |        1.03 GB |            n/a |
-| items[0]       | bote       | 527.80 µs |       291.6 KB |       130.4 KB |
-| items[535399]  | bote       | 187.24 ms |       742.3 KB |        36.7 MB |
-| items[1070797] | bote       | 371.61 ms |       828.7 KB |        37.1 MB |
+| items[0]       | JSON.parse |    1.81 s |        1.21 GB |            n/a |
+| items[535399]  | JSON.parse |    1.74 s |        1.21 GB |            n/a |
+| items[1070797] | JSON.parse |    1.74 s |        1.21 GB |            n/a |
+| items[0]       | bote       |   1.29 ms |        63.3 KB |       130.8 KB |
+| items[535399]  | bote       | 193.49 ms |       191.5 KB |        36.7 MB |
+| items[1070797] | bote       | 379.98 ms |       189.8 KB |        37.2 MB |
 
 ## array access
 
-`iter` streams the elements of an array at a path, **a batch at a time**, so you never hold the whole collection in memory and not wait for the heat death of the universe if this yielded individually. each `for await` step yields an array of items (use `walk` to step over the members of an object):
+`iter` streams the children of a container at a path **one item at a time**, so you never hold the whole collection in memory. it works on either kind: array elements or object member values. each `for await` step yields a single item:
 
 ```ts
 // e.g. [{ id: 'user-1' }, { id: 'user-2' }, ...]
 await using cursor = await open(fromFile('./users.json'))
 
 // root is an array
-for await (const users of cursor.iter()) {
+for await (const user of cursor.iter()) {
+  console.log(user)
+}
+```
+
+the item loop is the ergonomic default; it costs a flat ~10% over a full walk. for hot paths, `.raw()` hands back the raw fetch arrays with no per-item tax (the `batch` option sets their size and the memory bound):
+
+```ts
+for await (const users of cursor.iter().raw()) {
   for (const user of users) {
     console.log(user)
   }
 }
 ```
 
-pass an options object as the last argument to tune what comes back: `batch`, `select`, `schema`, `onInvalid`, and `withIndex`. if you want to know more of the options, see [`arrays.js`](./examples/arrays.js).
-
 ## object access
 
-`walk` steps over the members of an object at a path, yielding a **`[key, cursor]`** pair per member. the key is the member name, the cursor is anchored at its value. each child cursor is first-class: it outlives the loop and can be `walk`ed again, which is what lets you descend a tree of unknown depth.
+`iter` over an object yields its **member values** in document order. add `withKey: true` to get **`[key, value]`** pairs instead, where `key` is the member name (for an array, `key` is the element's index). streamed either way, so a million-member object never lands on the heap at once:
 
 ```ts
 // e.g. { alice: { role: 'admin' }, bob: { role: 'guest' }, ... }
 await using cursor = await open(fromFile('./accounts.json'))
 
-for await (const [name, account] of cursor.walk()) {
-  // name is the member name ('alice', 'bob', ...)
-  const role = await account.get('role')
-  console.log(`${name}: ${role}`)
+for await (const [name, account] of cursor.iter({ withKey: true })) {
+  // name is the member name ('alice', 'bob', ...); account is its value
+  console.log(`${name}: ${account.role}`)
 }
 ```
 
@@ -77,8 +82,8 @@ await using cursor = await open(fromFile('./report.json'))
 const section = await cursor.hop('report', 'sections', 0)
 if (section) {
   console.log(await section.count('rows'))
-  for await (const rows of section.iter('rows')) {
-    console.log(rows)
+  for await (const row of section.iter('rows')) {
+    console.log(row)
   }
 }
 ```
@@ -105,12 +110,14 @@ const cursor = await open(fromFile('./users.json'))
 // name: string
 const name = await cursor.get('users', 1000, 'name', User.shape.name)
 
-for await (const users of cursor.iter('users', User)) {
-  // user: User[]
-  const emails = users.map((user) => user.email)
-  await sendNewsletter(emails)
+let emails: string[] = []
+// .raw() to hand each fetch's worth of recipients to the batched API at once
+for await (const user of cursor.iter('users', User)) {
+  // user: User
+  emails.push(user.email)
 }
 
+await sendNewsletter(emails)
 await cursor.close()
 ```
 

package/dist/args.d.ts

@@ -1,15 +1,20 @@
 import type { Path, Segment, StandardSchemaV1 } from './validate.ts';
 export interface IterOptions {
     select?: Segment | Path | Record<string, Segment | Path>;
-    /** How many items are yielded per batch. Higher is faster, but takes more memory to materialise those items. */
+    /** How many items cross the native boundary per fetch, which also bounds the
+     *  resident materialization window (the memory knob) and sets the array size
+     *  yielded by `IterStream.raw()`. The default item loop drains each fetch
+     *  one item at a time, so this doesn't change what item iteration yields, only
+     *  how much is fetched and held at once. Higher is faster but holds more in
+     *  memory. */
     batch?: number;
     /** Validate each yielded item against this schema (after `select`). */
     schema?: StandardSchemaV1;
     /** Policy for items failing `schema`. Default `'throw'`; `'skip'` drops them. */
     onInvalid?: 'throw' | 'skip';
-    /** Yield `[index, value]` tuples instead of bare values, where `index` is
-     *  the zero-based position of the element in the source array. */
-    withIndex?: boolean;
+    /** Yield `[key, value]` tuples instead of bare values. `key` is the member
+     *  name for objects and the zero-based index for arrays. */
+    withKey?: boolean;
 }
 export type VariadicPathArgs<TTail> = [...Segment[]] | [...Segment[], TTail];
 export declare function splitArgs<TTail>(args: VariadicPathArgs<TTail>): {

package/dist/cursor.d.ts

@@ -0,0 +1,52 @@
+import type { Cursor as NativeCursor } from '@botejs/native';
+import { type IterStream } from './stream.ts';
+import { type Path, type Segment, type StandardSchemaV1 } from './validate.ts';
+import { type IterOptions } from './args.ts';
+type InferOutput<Sch> = Sch extends StandardSchemaV1<unknown, infer O> ? O : never;
+type SelectMapShape<S> = {
+    -readonly [K in keyof S]: unknown;
+};
+export type IterKey = string | number;
+export declare const DEFAULT_ITER_BATCH = 1000;
+export declare const MAX_ITER_BATCH = 1000000;
+export interface Cursor {
+    hop(...path: Segment[]): Promise<Cursor | null>;
+    has(...path: Segment[]): Promise<boolean>;
+    has(...args: [...Segment[], StandardSchemaV1]): Promise<boolean>;
+    get(...path: Segment[]): Promise<unknown>;
+    get<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): Promise<InferOutput<Sch>>;
+    count(...path: Segment[]): Promise<number>;
+    iter(...path: Segment[]): IterStream<unknown>;
+    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): IterStream<InferOutput<Sch>>;
+    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
+        withKey: true;
+        schema: Sch;
+    }]): IterStream<[IterKey, InferOutput<Sch>]>;
+    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
+        schema: Sch;
+    }]): IterStream<InferOutput<Sch>>;
+    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
+        withKey: true;
+        select: S;
+    }]): IterStream<[IterKey, SelectMapShape<S>]>;
+    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
+        select: S;
+    }]): IterStream<SelectMapShape<S>>;
+    iter(...args: [...Segment[], IterOptions & {
+        withKey: true;
+    }]): IterStream<[IterKey, unknown]>;
+    iter(...args: [...Segment[], IterOptions]): IterStream<unknown>;
+}
+export interface RootCursor extends Cursor, AsyncDisposable {
+    /** Close the underlying source. Idempotent. */
+    close(): Promise<void>;
+}
+export type CursorState = {
+    closed: boolean;
+};
+/** Throw a uniform error for any operation on a closed cursor, so use-after-close
+ *  is one defined contract regardless of source (some readers' reads keep working
+ *  after close, others throw an opaque I/O error). */
+export declare function ensureOpen(state: CursorState): void;
+export declare function wrap(native: NativeCursor, state: CursorState): Cursor;
+export {};

package/dist/cursor.js

@@ -0,0 +1,119 @@
+import { validatePath } from "./path.js";
+import { parseValue, deserializeError } from "./decode.js";
+import { makeStream } from "./stream.js";
+import { runStandardSchema, validateItem } from "./validate.js";
+import { splitArgs, isSchema, serializeSelect, normalizeIterTail, } from "./args.js";
+export const DEFAULT_ITER_BATCH = 1000;
+export const MAX_ITER_BATCH = 1_000_000;
+/** Throw a uniform error for any operation on a closed cursor, so use-after-close
+ *  is one defined contract regardless of source (some readers' reads keep working
+ *  after close, others throw an opaque I/O error). */
+export function ensureOpen(state) {
+    if (state.closed)
+        throw new Error('bote: cursor is closed');
+}
+export function wrap(native, state) {
+    const cursor = {
+        async hop(...path) {
+            ensureOpen(state);
+            validatePath(path);
+            let child;
+            try {
+                child = await native.hop(path);
+            }
+            catch (err) {
+                throw deserializeError(err, path);
+            }
+            return child ? wrap(child, state) : null;
+        },
+        async has(...args) {
+            ensureOpen(state);
+            const { path, tail: schema } = splitArgs(args);
+            if (schema !== undefined && !isSchema(schema)) {
+                throw new TypeError('has: expected a Standard Schema as the trailing argument');
+            }
+            if (!schema)
+                return native.has(path);
+            if (!(await native.has(path)))
+                return false;
+            const text = await native.get(path);
+            const value = text === undefined ? undefined : parseValue(text, path);
+            const result = await validateItem(schema, value, path, 'skip');
+            return !('skip' in result);
+        },
+        async get(...args) {
+            ensureOpen(state);
+            const { path, tail: schema } = splitArgs(args);
+            if (schema !== undefined && !isSchema(schema)) {
+                throw new TypeError('get: expected a Standard Schema as the trailing argument');
+            }
+            let value;
+            try {
+                const text = await native.get(path);
+                value = text === undefined ? undefined : parseValue(text, path);
+            }
+            catch (err) {
+                throw deserializeError(err, path);
+            }
+            if (!schema)
+                return value;
+            return runStandardSchema(schema, value, path);
+        },
+        async count(...path) {
+            ensureOpen(state);
+            validatePath(path);
+            try {
+                return await native.count(path);
+            }
+            catch (err) {
+                throw deserializeError(err, path);
+            }
+        },
+        iter(...args) {
+            ensureOpen(state);
+            const { path, tail } = splitArgs(args);
+            const { schema, select, batch, onInvalid, withKey } = normalizeIterTail(tail);
+            if (batch !== undefined && (!Number.isInteger(batch) || batch <= 0 || batch > MAX_ITER_BATCH)) {
+                throw new RangeError(`iter: batch must be an integer in 1..=${MAX_ITER_BATCH}, got ${batch}`);
+            }
+            if (withKey !== undefined && typeof withKey !== 'boolean') {
+                throw new TypeError(`iter: withKey must be a boolean, got ${typeof withKey}`);
+            }
+            if (onInvalid !== undefined && onInvalid !== 'throw' && onInvalid !== 'skip') {
+                throw new RangeError(`iter: onInvalid must be "throw" or "skip", got ${JSON.stringify(onInvalid)}`);
+            }
+            const resolvedBatch = batch ?? DEFAULT_ITER_BATCH;
+            const selectIr = select !== undefined ? serializeSelect(select) : undefined;
+            const wantKey = withKey ?? false;
+            const nativeWithKey = wantKey || schema !== undefined;
+            const inner = native.iter(path, { selectIr, batch: resolvedBatch, withKey: nativeWithKey });
+            if (!schema) {
+                return nativeStream(inner, path, resolvedBatch, (raw) => parseValue(raw, path));
+            }
+            const policy = onInvalid ?? 'throw';
+            return nativeStream(inner, path, resolvedBatch, async (raw) => {
+                const out = [];
+                for (const [key, value] of parseValue(raw, path)) {
+                    const result = await validateItem(schema, value, [...path, key], policy);
+                    if ('skip' in result)
+                        continue;
+                    out.push(wantKey ? [key, result.value] : result.value);
+                }
+                return out;
+            });
+        },
+    };
+    return cursor;
+}
+function nativeStream(inner, path, batchSize, mapBatch) {
+    async function* batches() {
+        try {
+            for await (const raw of inner)
+                yield await mapBatch(raw);
+        }
+        catch (err) {
+            throw deserializeError(err, path);
+        }
+    }
+    return makeStream(batches, batchSize);
+}

package/dist/decode.d.ts

@@ -0,0 +1,3 @@
+import { type Path } from './validate.ts';
+export declare function deserializeError(err: unknown, path: Path): unknown;
+export declare function parseValue(text: string, path: Path): unknown;

package/dist/decode.js

@@ -0,0 +1,20 @@
+import { PathError, formatPath } from "./validate.js";
+const NATIVE_PATH_ERROR = /^bote:path:([a-z_]+)(?::(\d+))?$/;
+export function deserializeError(err, path) {
+    if (err instanceof Error && !(err instanceof PathError)) {
+        const match = NATIVE_PATH_ERROR.exec(err.message);
+        if (match) {
+            const segment = match[2] === undefined ? undefined : Number(match[2]);
+            return new PathError(path, match[1], segment);
+        }
+    }
+    return err;
+}
+export function parseValue(text, path) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw new Error(`bote: malformed JSON value at ${formatPath(path)}`);
+    }
+}

package/dist/index.d.ts

@@ -1,4 +1,6 @@
 export { type IterOptions } from './args.ts';
 export { ValidationError, PathError, formatPath, type Path, type PathFaultCode, type Segment, type StandardSchemaV1, } from './validate.ts';
-export { open, DEFAULT_ITER_BATCH, MAX_ITER_BATCH, type Cursor, type RootCursor, type OpenOptions, type WalkEntry, type IterIndex as IterKey, } from './open.ts';
+export { DEFAULT_ITER_BATCH, MAX_ITER_BATCH, type Cursor, type RootCursor, type IterKey } from './cursor.ts';
 export { fromBuffer, fromFile, fromHttpRange, type FactoryOptions, type Source, type SourceReader, type HttpRangeOptions, } from './sources.ts';
+export { type IterStream } from './stream.ts';
+export { open, type OpenOptions } from './open.ts';

package/dist/index.js

@@ -1,3 +1,4 @@
 export { ValidationError, PathError, formatPath, } from "./validate.js";
-export { open, DEFAULT_ITER_BATCH, MAX_ITER_BATCH, } from "./open.js";
+export { DEFAULT_ITER_BATCH, MAX_ITER_BATCH } from "./cursor.js";
 export { fromBuffer, fromFile, fromHttpRange, } from "./sources.js";
+export { open } from "./open.js";

package/dist/open.d.ts

@@ -1,17 +1,6 @@
+import { type RootCursor } from './cursor.ts';
 import type { Source } from './sources.ts';
-import { type Path, type Segment, type StandardSchemaV1 } from './validate.ts';
-import { type IterOptions } from './args.ts';
-type InferOutput<Sch> = Sch extends StandardSchemaV1<unknown, infer O> ? O : never;
-type SelectMapShape<S> = {
-    -readonly [K in keyof S]: unknown;
-};
-/** Zero-based index of an array element. */
-export type IterIndex = number;
-/** One `walk` step: the member's key paired with a cursor anchored at its value. */
-export type WalkEntry = [key: string, cursor: Cursor];
 export declare const DEFAULT_SOURCE_CHUNK_BYTES: number;
-export declare const DEFAULT_ITER_BATCH = 1000;
-export declare const MAX_ITER_BATCH = 1000000;
 export interface OpenOptions {
     /**
      * Slot budget for the structural-index cache: one slot per cached container
@@ -42,40 +31,6 @@ export interface OpenOptions {
      */
     arrayIndexInterval?: number;
 }
-export interface Cursor {
-    hop(...path: Segment[]): Promise<Cursor | null>;
-    has(...path: Segment[]): Promise<boolean>;
-    has(...args: [...Segment[], StandardSchemaV1]): Promise<boolean>;
-    get(...path: Segment[]): Promise<unknown>;
-    get<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): Promise<InferOutput<Sch>>;
-    count(...path: Segment[]): Promise<number>;
-    iter(...path: Segment[]): AsyncIterable<unknown[]>;
-    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): AsyncIterable<InferOutput<Sch>[]>;
-    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
-        withIndex: true;
-        schema: Sch;
-    }]): AsyncIterable<[IterIndex, InferOutput<Sch>][]>;
-    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
-        schema: Sch;
-    }]): AsyncIterable<InferOutput<Sch>[]>;
-    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
-        withIndex: true;
-        select: S;
-    }]): AsyncIterable<[IterIndex, SelectMapShape<S>][]>;
-    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
-        select: S;
-    }]): AsyncIterable<SelectMapShape<S>[]>;
-    iter(...args: [...Segment[], IterOptions & {
-        withIndex: true;
-    }]): AsyncIterable<[IterIndex, unknown][]>;
-    iter(...args: [...Segment[], IterOptions]): AsyncIterable<unknown[]>;
-    walk(...path: Segment[]): AsyncIterable<WalkEntry>;
-    walk(...path: Segment[]): AsyncIterable<Cursor>;
-}
-export interface RootCursor extends Cursor, AsyncDisposable {
-    /** Close the underlying source. Idempotent. */
-    close(): Promise<void>;
-}
 /**
  * Open a cursor over a seekable source.
  *
@@ -83,4 +38,3 @@ export interface RootCursor extends Cursor, AsyncDisposable {
  * drives the reader's own `close()` exactly once.
  */
 export declare function open(source: Source, options?: OpenOptions): Promise<RootCursor>;
-export {};

package/dist/open.js

@@ -1,10 +1,6 @@
 import { open as openNative } from '@botejs/native';
-import { validatePath } from "./path.js";
-import { runStandardSchema, validateItem, formatPath, PathError, } from "./validate.js";
-import { splitArgs, isSchema, serializeSelect, normalizeIterTail, } from "./args.js";
+import { wrap } from "./cursor.js";
 export const DEFAULT_SOURCE_CHUNK_BYTES = 64 * 1024;
-export const DEFAULT_ITER_BATCH = 1000;
-export const MAX_ITER_BATCH = 1_000_000;
 /**
  * Open a cursor over a seekable source.
  *
@@ -71,169 +67,3 @@ async function closeReader(reader) {
     if (reader.close)
         await reader.close();
 }
-const NATIVE_PATH_ERROR = /^bote:path:([a-z_]+)(?::(\d+))?$/;
-function deserializeError(err, path) {
-    if (err instanceof Error && !(err instanceof PathError)) {
-        const match = NATIVE_PATH_ERROR.exec(err.message);
-        if (match) {
-            const segment = match[2] === undefined ? undefined : Number(match[2]);
-            return new PathError(path, match[1], segment);
-        }
-    }
-    return err;
-}
-/** Throw a uniform error for any operation on a closed cursor, so use-after-close
- *  is one defined contract regardless of source (some readers' reads keep working
- *  after close, others throw an opaque I/O error). */
-function ensureOpen(state) {
-    if (state.closed)
-        throw new Error('bote: cursor is closed');
-}
-function wrap(native, state) {
-    const cursor = {
-        async hop(...path) {
-            ensureOpen(state);
-            validatePath(path);
-            let child;
-            try {
-                child = await native.hop(path);
-            }
-            catch (err) {
-                throw deserializeError(err, path);
-            }
-            return child ? wrap(child, state) : null;
-        },
-        async has(...args) {
-            ensureOpen(state);
-            const { path, tail: schema } = splitArgs(args);
-            if (schema !== undefined && !isSchema(schema)) {
-                throw new TypeError('has: expected a Standard Schema as the trailing argument');
-            }
-            if (!schema)
-                return native.has(path);
-            if (!(await native.has(path)))
-                return false;
-            const text = await native.get(path);
-            const value = text === undefined ? undefined : parseValue(text, path);
-            const result = await validateItem(schema, value, path, 'skip');
-            return !('skip' in result);
-        },
-        async get(...args) {
-            ensureOpen(state);
-            const { path, tail: schema } = splitArgs(args);
-            if (schema !== undefined && !isSchema(schema)) {
-                throw new TypeError('get: expected a Standard Schema as the trailing argument');
-            }
-            let value;
-            try {
-                const text = await native.get(path);
-                value = text === undefined ? undefined : parseValue(text, path);
-            }
-            catch (err) {
-                throw deserializeError(err, path);
-            }
-            if (!schema)
-                return value;
-            return runStandardSchema(schema, value, path);
-        },
-        async count(...path) {
-            ensureOpen(state);
-            validatePath(path);
-            try {
-                return await native.count(path);
-            }
-            catch (err) {
-                throw deserializeError(err, path);
-            }
-        },
-        iter(...args) {
-            ensureOpen(state);
-            const { path, tail } = splitArgs(args);
-            const { schema, select, batch, onInvalid, withIndex } = normalizeIterTail(tail);
-            if (batch !== undefined && (!Number.isInteger(batch) || batch <= 0 || batch > MAX_ITER_BATCH)) {
-                throw new RangeError(`iter: batch must be an integer in 1..=${MAX_ITER_BATCH}, got ${batch}`);
-            }
-            if (withIndex !== undefined && typeof withIndex !== 'boolean') {
-                throw new TypeError(`iter: withIndex must be a boolean, got ${typeof withIndex}`);
-            }
-            if (onInvalid !== undefined && onInvalid !== 'throw' && onInvalid !== 'skip') {
-                throw new RangeError(`iter: onInvalid must be "throw" or "skip", got ${JSON.stringify(onInvalid)}`);
-            }
-            const resolvedBatch = batch ?? DEFAULT_ITER_BATCH;
-            const selectIr = select !== undefined ? serializeSelect(select) : undefined;
-            const inner = native.iter(path, { selectIr, batch: resolvedBatch });
-            if (!schema) {
-                return {
-                    async *[Symbol.asyncIterator]() {
-                        let i = 0;
-                        try {
-                            for await (const b of inner) {
-                                const batch = parseValue(b, path);
-                                if (!withIndex) {
-                                    yield batch;
-                                    continue;
-                                }
-                                const out = new Array(batch.length);
-                                for (let j = 0; j < batch.length; j++) {
-                                    out[j] = [i++, batch[j]];
-                                }
-                                yield out;
-                            }
-                        }
-                        catch (err) {
-                            throw deserializeError(err, path);
-                        }
-                    },
-                };
-            }
-            const policy = onInvalid ?? 'throw';
-            return {
-                async *[Symbol.asyncIterator]() {
-                    let i = 0;
-                    try {
-                        for await (const b of inner) {
-                            const out = [];
-                            for (const v of parseValue(b, path)) {
-                                const index = i++;
-                                const result = await validateItem(schema, v, [...path, index], policy);
-                                if ('skip' in result) {
-                                    continue;
-                                }
-                                out.push(withIndex ? [index, result.value] : result.value);
-                            }
-                            yield out;
-                        }
-                    }
-                    catch (err) {
-                        throw deserializeError(err, path);
-                    }
-                },
-            };
-        },
-        walk(...path) {
-            ensureOpen(state);
-            validatePath(path);
-            return {
-                async *[Symbol.asyncIterator]() {
-                    try {
-                        for await (const [key, child] of native.walk(path)) {
-                            yield [key, wrap(child, state)];
-                        }
-                    }
-                    catch (err) {
-                        throw deserializeError(err, path);
-                    }
-                },
-            };
-        },
-    };
-    return cursor;
-}
-function parseValue(text, path) {
-    try {
-        return JSON.parse(text);
-    }
-    catch {
-        throw new Error(`bote: malformed JSON value at ${formatPath(path)}`);
-    }
-}

package/dist/stream.d.ts

@@ -0,0 +1,15 @@
+export interface IterStream<T> extends AsyncIterable<T> {
+    raw(): AsyncIterable<T[]>;
+    map<U>(fn: (item: T, index: number) => U | Promise<U>): IterStream<U>;
+    filter<U extends T>(fn: (item: T, index: number) => item is U): IterStream<U>;
+    filter(fn: (item: T, index: number) => boolean | Promise<boolean>): IterStream<T>;
+    take(limit: number): IterStream<T>;
+    drop(limit: number): IterStream<T>;
+    toArray(): Promise<T[]>;
+    forEach(fn: (item: T, index: number) => void | Promise<void>): Promise<void>;
+    reduce<A>(fn: (acc: A, item: T, index: number) => A | Promise<A>, init: A): Promise<A>;
+    find(fn: (item: T, index: number) => boolean | Promise<boolean>): Promise<T | undefined>;
+    some(fn: (item: T, index: number) => boolean | Promise<boolean>): Promise<boolean>;
+    every(fn: (item: T, index: number) => boolean | Promise<boolean>): Promise<boolean>;
+}
+export declare function makeStream<T>(batches: () => AsyncIterable<T[]>, batchSize: number, regroup?: boolean): IterStream<T>;

package/dist/stream.js

@@ -0,0 +1,166 @@
+export function makeStream(batches, batchSize, regroup = false) {
+    const derive = (next) => makeStream(next, batchSize, true);
+    const stream = {
+        [Symbol.asyncIterator]() {
+            return flatten(batches())[Symbol.asyncIterator]();
+        },
+        raw() {
+            return regroup ? regroupBatches(batches(), batchSize) : batches();
+        },
+        map(fn) {
+            return derive(() => mapBatches(batches(), fn));
+        },
+        filter(fn) {
+            return derive(() => filterBatches(batches(), fn));
+        },
+        take(limit) {
+            return derive(() => takeBatches(batches(), limit));
+        },
+        drop(limit) {
+            return derive(() => dropBatches(batches(), limit));
+        },
+        async toArray() {
+            const out = [];
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    out.push(batch[i]);
+                }
+            }
+            return out;
+        },
+        async forEach(fn) {
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    await fn(batch[i], index++);
+                }
+            }
+        },
+        async reduce(fn, init) {
+            let acc = init;
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    acc = await fn(acc, batch[i], index++);
+                }
+            }
+            return acc;
+        },
+        async find(fn) {
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    if (await fn(batch[i], index++)) {
+                        return batch[i];
+                    }
+                }
+            }
+            return undefined;
+        },
+        async some(fn) {
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    if (await fn(batch[i], index++)) {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        },
+        async every(fn) {
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    if (!(await fn(batch[i], index++))) {
+                        return false;
+                    }
+                }
+            }
+            return true;
+        },
+    };
+    return stream;
+}
+async function* flatten(batches) {
+    for await (const batch of batches) {
+        for (let i = 0; i < batch.length; i++) {
+            yield batch[i];
+        }
+    }
+}
+async function* regroupBatches(batches, size) {
+    let buf = [];
+    for await (const batch of batches) {
+        for (let i = 0; i < batch.length; i++) {
+            buf.push(batch[i]);
+            if (buf.length >= size) {
+                yield buf;
+                buf = [];
+            }
+        }
+    }
+    if (buf.length > 0) {
+        yield buf;
+    }
+}
+async function* mapBatches(batches, fn) {
+    let index = 0;
+    for await (const batch of batches) {
+        const out = new Array(batch.length);
+        for (let i = 0; i < batch.length; i++) {
+            const r = fn(batch[i], index++);
+            out[i] = isThenable(r) ? await r : r;
+        }
+        yield out;
+    }
+}
+async function* filterBatches(batches, fn) {
+    let index = 0;
+    for await (const batch of batches) {
+        const out = [];
+        for (let i = 0; i < batch.length; i++) {
+            const item = batch[i];
+            const r = fn(item, index++);
+            if (isThenable(r) ? await r : r) {
+                out.push(item);
+            }
+        }
+        if (out.length > 0) {
+            yield out;
+        }
+    }
+}
+async function* takeBatches(batches, limit) {
+    if (limit <= 0) {
+        return;
+    }
+    let remaining = limit;
+    for await (const batch of batches) {
+        if (batch.length < remaining) {
+            remaining -= batch.length;
+            yield batch;
+            continue;
+        }
+        yield batch.length === remaining ? batch : batch.slice(0, remaining);
+        return;
+    }
+}
+async function* dropBatches(batches, limit) {
+    let remaining = limit;
+    for await (const batch of batches) {
+        if (remaining === 0) {
+            yield batch;
+        }
+        else if (remaining >= batch.length) {
+            remaining -= batch.length;
+        }
+        else {
+            yield batch.slice(remaining);
+            remaining = 0;
+        }
+    }
+}
+function isThenable(value) {
+    return value != null && typeof value.then === 'function';
+}

package/dist/validate.js

@@ -15,8 +15,6 @@ const PATH_FAULT_MESSAGE = {
     through_scalar: (segment) => `path traverses a non-container value at segment ${segment}`,
     wrong_kind: (segment) => `path segment ${segment} does not match the container kind`,
     scalar_target: () => 'target value is not a container',
-    iter_on_object: () => 'iter target is an object; use walk() to iterate object members',
-    walk_on_array: () => 'walk target is an array; use iter() to iterate array elements',
 };
 export class PathError extends Error {
     path;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botejs/core",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "type": "module",
   "license": "MIT",
   "repository": {
@@ -35,7 +35,7 @@
     "prepublishOnly": "cp ../../README.md ./README.md && tsc"
   },
   "dependencies": {
-    "@botejs/native": "^0.4.0"
+    "@botejs/native": "^0.5.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",