@botejs/core 0.4.0 → 0.6.0

package/dist/open.d.ts

@@ -1,86 +1,47 @@
-import type { Source } from './sources.ts';
-import { type Path, type Segment, type StandardSchemaV1 } from './validate.ts';
-import { type IterOptions } from './args.ts';
-type InferOutput<Sch> = Sch extends StandardSchemaV1<unknown, infer O> ? O : never;
-type SelectMapShape<S> = {
-    -readonly [K in keyof S]: unknown;
-};
-/** Zero-based index of an array element. */
-export type IterIndex = number;
-/** One `walk` step: the member's key paired with a cursor anchored at its value. */
-export type WalkEntry = [key: string, cursor: Cursor];
+import { type RootCursor } from './cursor.ts';
+import type { SeekableSource, ForwardSource } from './source/base.ts';
 export declare const DEFAULT_SOURCE_CHUNK_BYTES: number;
-export declare const DEFAULT_ITER_BATCH = 1000;
-export declare const MAX_ITER_BATCH = 1000000;
 export interface OpenOptions {
     /**
-     * Slot budget for the structural-index cache: one slot per cached container
-     * plus one per tabled object member. When a scan tips the cache over this
-     * budget, the deepest (least navigationally useful) containers are evicted
-     * first, LRU-tiebroken, keeping the shallow backbone that resumes future
-     * scans. Bounds resident cache memory regardless of document size. `0`
-     * disables the cache entirely. Omit for the native default (1024).
+     * How much of the index that speeds up repeat lookups to keep in memory,
+     * measured in entries. Higher means faster repeat queries but more memory;
+     * lower means less memory but slower repeats. Set to `0` to turn the cache
+     * off. Defaults to 1024.
      */
     indexCacheEntries?: number;
     /**
-     * Max object members tabled per walked container in the structural-index
-     * cache. The table is a dense prefix; past the cap, lookups of later members
-     * resume-scan from the cap boundary. Lower trades cache memory for resume work
-     * on pathologically large objects. `0` disables object member indexing. Omit
-     * for the native default (unbounded).
+     * How many keys per object to index for fast lookup. Higher speeds up access
+     * to keys later in large objects but uses more memory; lower saves memory at
+     * the cost of slower lookups for those keys. Set to `0` to skip indexing
+     * object keys. Defaults to unlimited.
      */
     objectMemberCap?: number;
     /**
-     * Element-index stride between sampled array members in the structural-index
-     * cache. A later index resumes from the nearest array member at or before it, so
-     * a smaller stride means denser array members (more memory, shorter resume
-     * scans). `0` disables array-member indexing. Omit for the native default (16).
-     *
-     * Setting both `objectMemberCap` and `arrayIndexInterval` to `0` disables the
-     * cache entirely (no source bytes are ever cached either way), as does
-     * `indexCacheEntries: 0`.
+     * How often to index array positions, e.g. every 16th element. Lower means
+     * faster access to arbitrary array elements but more memory; higher saves
+     * memory at the cost of slower access. Set to `0` to skip indexing array
+     * positions. Defaults to 16.
      */
     arrayIndexInterval?: number;
 }
-export interface Cursor {
-    hop(...path: Segment[]): Promise<Cursor | null>;
-    has(...path: Segment[]): Promise<boolean>;
-    has(...args: [...Segment[], StandardSchemaV1]): Promise<boolean>;
-    get(...path: Segment[]): Promise<unknown>;
-    get<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): Promise<InferOutput<Sch>>;
-    count(...path: Segment[]): Promise<number>;
-    iter(...path: Segment[]): AsyncIterable<unknown[]>;
-    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], Sch]): AsyncIterable<InferOutput<Sch>[]>;
-    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
-        withIndex: true;
-        schema: Sch;
-    }]): AsyncIterable<[IterIndex, InferOutput<Sch>][]>;
-    iter<Sch extends StandardSchemaV1>(...args: [...Segment[], IterOptions & {
-        schema: Sch;
-    }]): AsyncIterable<InferOutput<Sch>[]>;
-    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
-        withIndex: true;
-        select: S;
-    }]): AsyncIterable<[IterIndex, SelectMapShape<S>][]>;
-    iter<S extends Record<string, Segment | Path>>(...args: [...Segment[], IterOptions & {
-        select: S;
-    }]): AsyncIterable<SelectMapShape<S>[]>;
-    iter(...args: [...Segment[], IterOptions & {
-        withIndex: true;
-    }]): AsyncIterable<[IterIndex, unknown][]>;
-    iter(...args: [...Segment[], IterOptions]): AsyncIterable<unknown[]>;
-    walk(...path: Segment[]): AsyncIterable<WalkEntry>;
-    walk(...path: Segment[]): AsyncIterable<Cursor>;
-}
-export interface RootCursor extends Cursor, AsyncDisposable {
-    /** Close the underlying source. Idempotent. */
-    close(): Promise<void>;
-}
 /**
- * Open a cursor over a seekable source.
+ * Options for a forward source: every cache knob is forbidden (the structural-index
+ * cache is forced off). `Omit` would collapse to `{}` and silently permit them, so
+ * the knobs are mapped to `never` to reject them at compile time.
+ */
+export type ForwardOpenOptions = {
+    [K in keyof OpenOptions]?: never;
+};
+/**
+ * Open a cursor over a source.
+ *
+ * A seekable source (`fromFile`/`fromBuffer`/`fromHttpRange`) supports the cache
+ * and repeated, out-of-order queries. A forward source (`fromReadable`/`fromHttpStream`)
+ * is a single forward pass: the cache is forced off, so its cache knobs are
+ * rejected at compile time and at runtime.
  *
- * The returned `RootCursor` owns the reader: `close()` (or `await using`)
- * drives the reader's own `close()` exactly once.
+ * The returned `RootCursor` owns the reader: `close()` (or `await using`) drives
+ * the reader's own `close()` exactly once.
  */
-export declare function open(source: Source, options?: OpenOptions): Promise<RootCursor>;
-export {};
+export declare function open(source: SeekableSource, options?: OpenOptions): Promise<RootCursor>;
+export declare function open(source: ForwardSource, options?: ForwardOpenOptions): Promise<RootCursor>;

package/dist/open.js

@@ -1,32 +1,32 @@
 import { open as openNative } from '@botejs/native';
-import { validatePath } from "./path.js";
-import { runStandardSchema, validateItem, formatPath, PathError, } from "./validate.js";
-import { splitArgs, isSchema, serializeSelect, normalizeIterTail, } from "./args.js";
+import { wrap } from "./cursor.js";
 export const DEFAULT_SOURCE_CHUNK_BYTES = 64 * 1024;
-export const DEFAULT_ITER_BATCH = 1000;
-export const MAX_ITER_BATCH = 1_000_000;
-/**
- * Open a cursor over a seekable source.
- *
- * The returned `RootCursor` owns the reader: `close()` (or `await using`)
- * drives the reader's own `close()` exactly once.
- */
+const CACHE_KNOBS = ['indexCacheEntries', 'objectMemberCap', 'arrayIndexInterval'];
 export async function open(source, options) {
-    const { indexCacheEntries, objectMemberCap, arrayIndexInterval } = options ?? {};
-    for (const [name, value] of [
-        ['indexCacheEntries', indexCacheEntries],
-        ['objectMemberCap', objectMemberCap],
-        ['arrayIndexInterval', arrayIndexInterval],
-    ]) {
+    if (!source.seekable) {
+        for (const name of CACHE_KNOBS) {
+            if (options?.[name] !== undefined) {
+                throw new RangeError(`open: ${name} is not allowed for a forward source; the structural-index cache is forced off`);
+            }
+        }
+    }
+    for (const name of CACHE_KNOBS) {
+        const value = options?.[name];
         if (value !== undefined && (!Number.isSafeInteger(value) || value < 0)) {
             throw new RangeError(`open: ${name} must be a non-negative integer (0 disables), got ${value}`);
         }
     }
+    // A forward source disables every cache dimension, so the engine never resolves
+    // a cached container offset into a backward read on a stream it cannot rewind.
+    const knobs = source.seekable ? options : { indexCacheEntries: 0, objectMemberCap: 0, arrayIndexInterval: 0 };
     const reader = await source.open();
     const chunkBytes = reader.chunkBytes ?? DEFAULT_SOURCE_CHUNK_BYTES;
     let native;
     try {
-        if (!Number.isInteger(reader.size) || reader.size < 0) {
+        if (reader.seekable && reader.size === undefined) {
+            throw new RangeError('open: a seekable source must report a size');
+        }
+        if (reader.size !== undefined && (!Number.isInteger(reader.size) || reader.size < 0)) {
             throw new RangeError(`open: source size must be a non-negative integer, got ${reader.size}`);
         }
         if (!Number.isSafeInteger(chunkBytes) || chunkBytes <= 0) {
@@ -38,10 +38,10 @@ export async function open(source, options) {
         native = openNative({
             size: reader.size,
             chunkBytes,
-            indexCacheEntries,
-            objectMemberCap,
-            arrayIndexInterval,
-            read: async ({ offset, length }) => reader.read(offset, length),
+            objectMemberCap: knobs?.objectMemberCap,
+            indexCacheEntries: knobs?.indexCacheEntries,
+            arrayIndexInterval: knobs?.arrayIndexInterval,
+            read: ({ offset, length }) => reader.read(offset, length),
         });
     }
     catch (err) {
@@ -50,15 +50,17 @@ export async function open(source, options) {
             await closeReader(reader);
         }
         catch (closeErr) {
-            if (err instanceof Error)
+            if (err instanceof Error) {
                 err.cause ??= closeErr;
+            }
         }
         throw err;
     }
     const state = { closed: false };
     const close = async () => {
-        if (state.closed)
+        if (state.closed) {
             return;
+        }
         state.closed = true;
         await closeReader(reader);
     };
@@ -68,172 +70,7 @@ export async function open(source, options) {
     });
 }
 async function closeReader(reader) {
-    if (reader.close)
+    if (reader.close) {
         await reader.close();
-}
-const NATIVE_PATH_ERROR = /^bote:path:([a-z_]+)(?::(\d+))?$/;
-function deserializeError(err, path) {
-    if (err instanceof Error && !(err instanceof PathError)) {
-        const match = NATIVE_PATH_ERROR.exec(err.message);
-        if (match) {
-            const segment = match[2] === undefined ? undefined : Number(match[2]);
-            return new PathError(path, match[1], segment);
-        }
-    }
-    return err;
-}
-/** Throw a uniform error for any operation on a closed cursor, so use-after-close
- *  is one defined contract regardless of source (some readers' reads keep working
- *  after close, others throw an opaque I/O error). */
-function ensureOpen(state) {
-    if (state.closed)
-        throw new Error('bote: cursor is closed');
-}
-function wrap(native, state) {
-    const cursor = {
-        async hop(...path) {
-            ensureOpen(state);
-            validatePath(path);
-            let child;
-            try {
-                child = await native.hop(path);
-            }
-            catch (err) {
-                throw deserializeError(err, path);
-            }
-            return child ? wrap(child, state) : null;
-        },
-        async has(...args) {
-            ensureOpen(state);
-            const { path, tail: schema } = splitArgs(args);
-            if (schema !== undefined && !isSchema(schema)) {
-                throw new TypeError('has: expected a Standard Schema as the trailing argument');
-            }
-            if (!schema)
-                return native.has(path);
-            if (!(await native.has(path)))
-                return false;
-            const text = await native.get(path);
-            const value = text === undefined ? undefined : parseValue(text, path);
-            const result = await validateItem(schema, value, path, 'skip');
-            return !('skip' in result);
-        },
-        async get(...args) {
-            ensureOpen(state);
-            const { path, tail: schema } = splitArgs(args);
-            if (schema !== undefined && !isSchema(schema)) {
-                throw new TypeError('get: expected a Standard Schema as the trailing argument');
-            }
-            let value;
-            try {
-                const text = await native.get(path);
-                value = text === undefined ? undefined : parseValue(text, path);
-            }
-            catch (err) {
-                throw deserializeError(err, path);
-            }
-            if (!schema)
-                return value;
-            return runStandardSchema(schema, value, path);
-        },
-        async count(...path) {
-            ensureOpen(state);
-            validatePath(path);
-            try {
-                return await native.count(path);
-            }
-            catch (err) {
-                throw deserializeError(err, path);
-            }
-        },
-        iter(...args) {
-            ensureOpen(state);
-            const { path, tail } = splitArgs(args);
-            const { schema, select, batch, onInvalid, withIndex } = normalizeIterTail(tail);
-            if (batch !== undefined && (!Number.isInteger(batch) || batch <= 0 || batch > MAX_ITER_BATCH)) {
-                throw new RangeError(`iter: batch must be an integer in 1..=${MAX_ITER_BATCH}, got ${batch}`);
-            }
-            if (withIndex !== undefined && typeof withIndex !== 'boolean') {
-                throw new TypeError(`iter: withIndex must be a boolean, got ${typeof withIndex}`);
-            }
-            if (onInvalid !== undefined && onInvalid !== 'throw' && onInvalid !== 'skip') {
-                throw new RangeError(`iter: onInvalid must be "throw" or "skip", got ${JSON.stringify(onInvalid)}`);
-            }
-            const resolvedBatch = batch ?? DEFAULT_ITER_BATCH;
-            const selectIr = select !== undefined ? serializeSelect(select) : undefined;
-            const inner = native.iter(path, { selectIr, batch: resolvedBatch });
-            if (!schema) {
-                return {
-                    async *[Symbol.asyncIterator]() {
-                        let i = 0;
-                        try {
-                            for await (const b of inner) {
-                                const batch = parseValue(b, path);
-                                if (!withIndex) {
-                                    yield batch;
-                                    continue;
-                                }
-                                const out = new Array(batch.length);
-                                for (let j = 0; j < batch.length; j++) {
-                                    out[j] = [i++, batch[j]];
-                                }
-                                yield out;
-                            }
-                        }
-                        catch (err) {
-                            throw deserializeError(err, path);
-                        }
-                    },
-                };
-            }
-            const policy = onInvalid ?? 'throw';
-            return {
-                async *[Symbol.asyncIterator]() {
-                    let i = 0;
-                    try {
-                        for await (const b of inner) {
-                            const out = [];
-                            for (const v of parseValue(b, path)) {
-                                const index = i++;
-                                const result = await validateItem(schema, v, [...path, index], policy);
-                                if ('skip' in result) {
-                                    continue;
-                                }
-                                out.push(withIndex ? [index, result.value] : result.value);
-                            }
-                            yield out;
-                        }
-                    }
-                    catch (err) {
-                        throw deserializeError(err, path);
-                    }
-                },
-            };
-        },
-        walk(...path) {
-            ensureOpen(state);
-            validatePath(path);
-            return {
-                async *[Symbol.asyncIterator]() {
-                    try {
-                        for await (const [key, child] of native.walk(path)) {
-                            yield [key, wrap(child, state)];
-                        }
-                    }
-                    catch (err) {
-                        throw deserializeError(err, path);
-                    }
-                },
-            };
-        },
-    };
-    return cursor;
-}
-function parseValue(text, path) {
-    try {
-        return JSON.parse(text);
-    }
-    catch {
-        throw new Error(`bote: malformed JSON value at ${formatPath(path)}`);
     }
 }

package/dist/path.d.ts

@@ -1,5 +1,7 @@
-import type { Segment } from './validate.ts';
+export type Segment = string | number;
+export type Path = readonly Segment[];
 /** Upper bound on numeric segments (napi takes them as `u32`). 2^32 - 1
  *  comfortably covers any in-memory JSON array. */
 export declare const MAX_ARRAY_INDEX = 4294967295;
 export declare function validatePath(path: readonly unknown[]): asserts path is readonly Segment[];
+export declare function formatPath(path: Path): string;

package/dist/path.js

@@ -4,17 +4,41 @@ export const MAX_ARRAY_INDEX = 0xffffffff;
 export function validatePath(path) {
     for (let i = 0; i < path.length; i++) {
         const s = path[i];
-        if (typeof s === 'string')
+        if (typeof s === 'string') {
             continue;
-        if (typeof s === 'number' && Number.isInteger(s) && s >= 0 && s <= MAX_ARRAY_INDEX)
+        }
+        if (typeof s === 'number' && Number.isInteger(s) && s >= 0 && s <= MAX_ARRAY_INDEX) {
             continue;
+        }
         throw new TypeError(`path segment ${i}: expected string or non-negative integer (<= ${MAX_ARRAY_INDEX}), got ${describeBadSegment(s)}`);
     }
 }
+export function formatPath(path) {
+    if (path.length === 0) {
+        return '(root)';
+    }
+    let out = '';
+    for (let i = 0; i < path.length; i++) {
+        const seg = path[i];
+        if (typeof seg === 'number') {
+            out += `[${seg}]`;
+            continue;
+        }
+        if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(seg)) {
+            out += i === 0 ? seg : `.${seg}`;
+        }
+        else {
+            out += `[${JSON.stringify(seg)}]`;
+        }
+    }
+    return out;
+}
 function describeBadSegment(s) {
-    if (typeof s === 'number')
+    if (typeof s === 'number') {
         return `${s}`;
-    if (s === null)
+    }
+    if (s === null) {
         return 'null';
+    }
     return typeof s;
 }

package/dist/source/base.d.ts

@@ -0,0 +1,70 @@
+/**
+ * The bytes a `read` resolves to, plus an end-of-stream flag. `eof` is `true`
+ * iff this read reached the end of the underlying stream. A seekable reader can
+ * compute it from `size`; a forward reader discovers it as the stream drains.
+ */
+export interface ReadResult {
+    readonly data: Uint8Array;
+    readonly eof: boolean;
+}
+/**
+ * A handle on an opened byte stream. The reader owns whatever resources back the
+ * stream (a file handle, a `fetch` body, an `AbortController`, etc.) and surfaces
+ * them through `close()`. Constructed by `Source.open()`; never by callers directly.
+ *
+ * `seekable` declares the access model:
+ *   - `true`: `read(offset, length)` may be called at any offset, in any order.
+ *     `size` is required. This random access is what lets the structural-index
+ *     cache resume scans near a target.
+ *   - `false`: a single forward pass. `read` is called with non-decreasing
+ *     offsets; the cache is forced off. `size` may be omitted (the end is found
+ *     via `eof`). Rewinding to an earlier offset re-acquires the stream (see
+ *     `fromReadable`'s `rewind` option) or throws {@link ForwardReplayError}.
+ */
+export interface Reader {
+    /** Whether reads may target any offset/order (`true`) or a single forward pass (`false`). */
+    readonly seekable: boolean;
+    /** Total length in bytes. Required for seekable readers; optional for forward ones. */
+    readonly size?: number;
+    /** Preferred read granularity in bytes. Must be a non-zero multiple of 64. */
+    readonly chunkBytes?: number;
+    /**
+     * Read up to `length` bytes starting at `offset`. `data.byteLength` is the
+     * actual count read (`<= length`); `eof` is `true` iff the read reached the
+     * end of the stream.
+     */
+    read(offset: number, length: number): Promise<ReadResult>;
+    /** Release resources held by the reader. Driven once by the `open()` lifecycle. */
+    close?(): Promise<void> | void;
+}
+/**
+ * Describes how to obtain a byte stream. `open()` is called once per cursor for
+ * a seekable source; a forward source's reader re-acquires its stream on demand.
+ * Provide your own object to plug in a custom backend.
+ */
+export interface Source {
+    /** Mirrors {@link Reader.seekable}; lets `open()` enforce the right knobs at compile time. */
+    readonly seekable: boolean;
+    /** Acquire the stream. Resolves to a `Reader` that owns any underlying resources. */
+    open(): Promise<Reader>;
+}
+/**
+ * A {@link Source} statically known to be seekable (random access, cache-eligible).
+ * `open()` discriminates on this, so a custom backend must brand its `seekable`
+ * as the literal `true` (annotate the object `SeekableSource`) to be accepted.
+ */
+export type SeekableSource = Source & {
+    readonly seekable: true;
+};
+/**
+ * A {@link Source} statically known to be forward-only (single pass, cache forced
+ * off). Brand a custom backend's object `ForwardSource` so `open()` accepts it and
+ * rejects cache knobs at compile time.
+ */
+export type ForwardSource = Source & {
+    readonly seekable: false;
+};
+export interface FactoryOptions {
+    /** Override the factory's default chunk size. Must be a non-zero multiple of 64. */
+    chunkBytes?: number;
+}

package/dist/source/base.js

@@ -0,0 +1 @@
+export {};

package/dist/source/forward.d.ts

@@ -0,0 +1,49 @@
+import type { FactoryOptions, ForwardSource } from './base.ts';
+/**
+ * A function that produces a fresh readable stream each time it is called. A
+ * forward reader invokes it once up front, and again on every re-acquisition, so
+ * it is also the seam for per-acquisition customization (a freshly minted auth
+ * token, a new `AbortSignal`, etc.).
+ */
+export type ReadableProducer = () => NodeJS.ReadableStream | ReadableStream<Uint8Array> | Promise<NodeJS.ReadableStream | ReadableStream<Uint8Array>>;
+export interface ReadableOptions extends FactoryOptions {
+    /** Known total length, if any. Forwarded to the engine so it can skip rediscovering the end. */
+    size?: number;
+    /** Transform applied to every (re)acquired stream, e.g. `s => s.pipeThrough(new DecompressionStream('gzip'))`. */
+    decode?: (raw: ReadableStream<Uint8Array>) => ReadableStream<Uint8Array>;
+    /**
+     * What a later query that must re-read from an earlier offset does. Defaults
+     * to `'forbid'`. The three settings trade resident memory for re-read ability:
+     *   - `'forbid'`: a single forward pass; a rewind throws {@link ForwardReplayError}.
+     *   - `'replay'`: re-acquire the stream from the start. Only safe when the
+     *     producer is idempotent (yields the same bytes each call). No extra memory.
+     *   - `'buffer'`: snapshot the whole stream into memory on first read, enabling
+     *     random access at O(n) resident memory.
+     */
+    rewind?: 'forbid' | 'replay' | 'buffer';
+}
+export interface HttpRequestOptions extends Omit<ReadableOptions, 'size'> {
+    /** Merged into every `fetch` (headers, credentials, signal, etc.). */
+    init?: RequestInit;
+}
+/**
+ * A forward-only source backed by a re-openable readable stream. `produce` is
+ * called to acquire each pass: once up front, and again on a rewind when
+ * `rewind: 'replay'` is set. A plain `Readable` instance cannot be re-streamed,
+ * so pass a thunk (`() => createReadStream(path)`), not a live stream.
+ *
+ * Because every cursor operation is an independent scan from the start, a single
+ * forward pass serves exactly one query; a second query (or `hop` then `get`)
+ * rewinds. By default that throws {@link ForwardReplayError}; opt into
+ * `rewind: 'replay'` (idempotent producer) or `rewind: 'buffer'` (in-memory
+ * snapshot) for multi-query access.
+ */
+export declare function fromReadable(produce: ReadableProducer, options?: ReadableOptions): ForwardSource;
+/**
+ * A forward-only source over an HTTP request (GET is default), streamed in a single pass. A
+ * convenience wrapper around {@link fromReadable} whose producer re-fetches `url`
+ * (reusing `init`, so auth headers, credentials, and an `AbortSignal` survive
+ * each acquisition). For repeated or random access over HTTP, prefer the seekable
+ * {@link fromHttpRange}.
+ */
+export declare function fromHttpRequest(url: string, options?: HttpRequestOptions): ForwardSource;