@botejs/core 0.4.0 → 0.6.0

package/dist/source/forward.js

@@ -0,0 +1,219 @@
+import { Readable } from 'node:stream';
+import { ForwardReplayError } from "../error.js";
+/** Default chunk size, in bytes, for forward streams: a large pull keeps a streamed scan moving. */
+const DEFAULT_STREAM_CHUNK_BYTES = 256 * 1024;
+const EMPTY = new Uint8Array(0);
+/**
+ * A forward-only source backed by a re-openable readable stream. `produce` is
+ * called to acquire each pass: once up front, and again on a rewind when
+ * `rewind: 'replay'` is set. A plain `Readable` instance cannot be re-streamed,
+ * so pass a thunk (`() => createReadStream(path)`), not a live stream.
+ *
+ * Because every cursor operation is an independent scan from the start, a single
+ * forward pass serves exactly one query; a second query (or `hop` then `get`)
+ * rewinds. By default that throws {@link ForwardReplayError}; opt into
+ * `rewind: 'replay'` (idempotent producer) or `rewind: 'buffer'` (in-memory
+ * snapshot) for multi-query access.
+ */
+export function fromReadable(produce, options) {
+    const chunkBytes = options?.chunkBytes ?? DEFAULT_STREAM_CHUNK_BYTES;
+    const size = options?.size;
+    const decode = options?.decode;
+    const rewind = options?.rewind ?? 'forbid';
+    return {
+        seekable: false,
+        open: () => makeForwardReader(produce, { chunkBytes, size, decode, rewind }),
+    };
+}
+/**
+ * A forward-only source over an HTTP request (GET is default), streamed in a single pass. A
+ * convenience wrapper around {@link fromReadable} whose producer re-fetches `url`
+ * (reusing `init`, so auth headers, credentials, and an `AbortSignal` survive
+ * each acquisition). For repeated or random access over HTTP, prefer the seekable
+ * {@link fromHttpRange}.
+ */
+export function fromHttpRequest(url, options) {
+    const { init, ...readable } = options ?? {};
+    const produce = async () => {
+        const res = await fetch(url, { ...init });
+        if (!res.ok) {
+            throw new Error(`${url} failed: ${res.status} ${res.statusText}`);
+        }
+        if (!res.body) {
+            throw new Error(`${url} returned no body`);
+        }
+        return res.body;
+    };
+    return fromReadable(produce, readable);
+}
+async function makeForwardReader(produce, config) {
+    const { chunkBytes, size, decode, rewind } = config;
+    const replay = rewind === 'replay';
+    const buffer = rewind === 'buffer';
+    let reader = null;
+    let pos = 0; // bytes consumed from the active pass (served + skipped)
+    let leftover = null; // tail of the last pulled chunk, not yet served
+    let done = false;
+    let snapshot = null;
+    let chain = Promise.resolve();
+    const acquire = async () => {
+        let web = toWebStream(await produce());
+        if (decode) {
+            web = decode(web);
+        }
+        reader = web.getReader();
+        pos = 0;
+        leftover = null;
+        done = false;
+    };
+    const release = async () => {
+        const active = reader;
+        reader = null;
+        if (active) {
+            try {
+                await active.cancel();
+            }
+            catch {
+                // cancelling an already-errored/closed stream is not actionable
+            }
+        }
+    };
+    // Next run of bytes available from the active pass, or null once it is drained.
+    const pull = async () => {
+        if (leftover) {
+            const chunk = leftover;
+            leftover = null;
+            return chunk;
+        }
+        if (done || !reader) {
+            return null;
+        }
+        const { value, done: streamDone } = await reader.read();
+        if (streamDone) {
+            done = true;
+            return null;
+        }
+        return value;
+    };
+    const readForward = async (offset, length) => {
+        if (offset < pos) {
+            if (!replay) {
+                throw new ForwardReplayError(offset, pos);
+            }
+            await release();
+            await acquire();
+        }
+        if (!reader && !done) {
+            await acquire();
+        }
+        while (pos < offset) {
+            const chunk = await pull();
+            if (chunk === null) {
+                break; // stream ended before the requested offset; the read returns eof
+            }
+            const skip = offset - pos;
+            if (chunk.byteLength <= skip) {
+                pos += chunk.byteLength;
+            }
+            else {
+                leftover = chunk.subarray(skip);
+                pos += skip;
+            }
+        }
+        const parts = [];
+        let got = 0;
+        while (got < length) {
+            const chunk = await pull();
+            if (chunk === null) {
+                break;
+            }
+            if (chunk.byteLength === 0) {
+                continue;
+            }
+            const take = Math.min(chunk.byteLength, length - got);
+            parts.push(take === chunk.byteLength ? chunk : chunk.subarray(0, take));
+            if (take < chunk.byteLength) {
+                leftover = chunk.subarray(take);
+            }
+            pos += take;
+            got += take;
+        }
+        return { data: concat(parts, got), eof: done && leftover === null };
+    };
+    const readSnapshot = (offset, length) => {
+        const data = snapshot;
+        if (offset >= data.byteLength) {
+            return { data: EMPTY, eof: true };
+        }
+        const end = Math.min(offset + length, data.byteLength);
+        return { data: data.subarray(offset, end), eof: end >= data.byteLength };
+    };
+    // A forward pass has shared position state, so overlapping scans (e.g. two
+    // un-awaited queries) must not interleave their pulls. Serializing reads keeps
+    // bookkeeping intact and turns concurrent misuse into a ForwardReplayError.
+    const read = (offset, length) => {
+        const result = chain.then(async () => {
+            if (buffer) {
+                if (!snapshot) {
+                    snapshot = await drainAll(produce, decode);
+                }
+                return readSnapshot(offset, length);
+            }
+            return readForward(offset, length);
+        });
+        chain = result.catch(() => { });
+        return result;
+    };
+    if (!buffer) {
+        await acquire();
+    }
+    return { seekable: false, size, chunkBytes, read, close: release };
+}
+async function drainAll(produce, decode) {
+    let web = toWebStream(await produce());
+    if (decode) {
+        web = decode(web);
+    }
+    const reader = web.getReader();
+    const parts = [];
+    let total = 0;
+    try {
+        for (;;) {
+            const { value, done } = await reader.read();
+            if (done) {
+                break;
+            }
+            if (value && value.byteLength > 0) {
+                parts.push(value);
+                total += value.byteLength;
+            }
+        }
+    }
+    finally {
+        try {
+            await reader.cancel();
+        }
+        catch {
+            // already drained/errored
+        }
+    }
+    return concat(parts, total);
+}
+function toWebStream(stream) {
+    if (typeof stream.getReader === 'function') {
+        return stream;
+    }
+    return Readable.toWeb(stream);
+}
+function concat(parts, total) {
+    if (parts.length === 1 && parts[0].byteLength === total) {
+        return parts[0];
+    }
+    const out = new Uint8Array(total);
+    let offset = 0;
+    for (const part of parts) {
+        out.set(part, offset);
+        offset += part.byteLength;
+    }
+    return out;
+}

package/dist/source/seekable.d.ts

@@ -0,0 +1,8 @@
+import type { FactoryOptions, SeekableSource } from './base.ts';
+export interface HttpRangeOptions extends FactoryOptions {
+    /** Merged into every request (headers, credentials, signal, etc.). */
+    init?: RequestInit;
+}
+export declare function fromBuffer(buf: Uint8Array | ArrayBuffer, options?: FactoryOptions): SeekableSource;
+export declare function fromFile(path: string, options?: FactoryOptions): SeekableSource;
+export declare function fromHttpRange(url: string, options?: HttpRangeOptions): SeekableSource;

package/dist/{sources.js → source/seekable.js}

@@ -9,37 +9,47 @@ export function fromBuffer(buf, options) {
     const view = buf instanceof Uint8Array ? buf : new Uint8Array(buf);
     const chunkBytes = options?.chunkBytes ?? DEFAULT_BUFFER_CHUNK_BYTES;
     return {
+        seekable: true,
         open: () => Promise.resolve({
+            seekable: true,
             size: view.byteLength,
             chunkBytes,
-            read: (offset, length) => Promise.resolve(view.subarray(offset, Math.min(offset + length, view.byteLength))),
+            read: (offset, length) => {
+                const data = view.subarray(offset, Math.min(offset + length, view.byteLength));
+                return Promise.resolve({ data, eof: offset + data.byteLength >= view.byteLength });
+            },
         }),
     };
 }
 export function fromFile(path, options) {
     const chunkBytes = options?.chunkBytes ?? DEFAULT_FILE_CHUNK_BYTES;
     return {
+        seekable: true,
         open: async () => {
             const handle = await fsOpen(path, 'r');
             const stat = await handle.stat();
+            const size = stat.size;
             let closed = false;
             return {
-                size: stat.size,
+                seekable: true,
+                size,
                 chunkBytes,
                 read: async (offset, length) => {
                     const buf = Buffer.allocUnsafe(length);
                     let filled = 0;
                     while (filled < length) {
                         const { bytesRead } = await handle.read(buf, filled, length - filled, offset + filled);
-                        if (bytesRead === 0)
+                        if (bytesRead === 0) {
                             break;
+                        }
                         filled += bytesRead;
                     }
-                    return buf.subarray(0, filled);
+                    return { data: buf.subarray(0, filled), eof: offset + filled >= size };
                 },
                 close: async () => {
-                    if (closed)
+                    if (closed) {
                         return;
+                    }
                     closed = true;
                     await handle.close();
                 },
@@ -51,6 +61,7 @@ export function fromHttpRange(url, options) {
     const init = options?.init;
     const chunkBytes = options?.chunkBytes ?? DEFAULT_URL_CHUNK_BYTES;
     return {
+        seekable: true,
         open: async () => {
             const controller = new AbortController();
             const userSignal = init?.signal;
@@ -79,6 +90,7 @@ export function fromHttpRange(url, options) {
             }
             let closed = false;
             return {
+                seekable: true,
                 size,
                 chunkBytes,
                 read: async (offset, length) => {
@@ -87,21 +99,23 @@ export function fromHttpRange(url, options) {
                     const headers = new Headers(init?.headers);
                     headers.set('Range', `bytes=${offset}-${end}`);
                     headers.set('Accept-Encoding', 'identity');
-                    const res = await fetch(url, { ...init, headers, method: 'GET', signal: controller.signal });
+                    const res = await fetch(url, { ...init, headers, signal: controller.signal });
                     if (res.status === 206) {
-                        return new Uint8Array(await res.arrayBuffer());
+                        const data = new Uint8Array(await res.arrayBuffer());
+                        return { data, eof: offset + data.byteLength >= size };
                     }
                     // A 200 means the server ignored our Range request and returned the full
                     // body. We throw here since the point of using ranges is to not have to
                     // buffer the whole thing in memory.
                     if (res.status === 200) {
-                        throw new Error(`Range GET ${url} (bytes=${offset}-${end}) ignored Range and returned 200.`);
+                        throw new Error(`Range ${url} (bytes=${offset}-${end}) ignored Range and returned 200.`);
                     }
-                    throw new Error(`Range GET ${url} (bytes=${offset}-${end}) failed: ${res.status}`);
+                    throw new Error(`Range ${url} (bytes=${offset}-${end}) failed: ${res.status}`);
                 },
                 close: async () => {
-                    if (closed)
+                    if (closed) {
                         return;
+                    }
                     closed = true;
                     controller.abort();
                 },

package/dist/stream.d.ts

@@ -0,0 +1,15 @@
+export interface IterStream<T> extends AsyncIterable<T> {
+    raw(): AsyncIterable<T[]>;
+    map<U>(fn: (item: T, index: number) => U | Promise<U>): IterStream<U>;
+    filter<U extends T>(fn: (item: T, index: number) => item is U): IterStream<U>;
+    filter(fn: (item: T, index: number) => boolean | Promise<boolean>): IterStream<T>;
+    take(limit: number): IterStream<T>;
+    drop(limit: number): IterStream<T>;
+    toArray(): Promise<T[]>;
+    forEach(fn: (item: T, index: number) => void | Promise<void>): Promise<void>;
+    reduce<A>(fn: (acc: A, item: T, index: number) => A | Promise<A>, init: A): Promise<A>;
+    find(fn: (item: T, index: number) => boolean | Promise<boolean>): Promise<T | undefined>;
+    some(fn: (item: T, index: number) => boolean | Promise<boolean>): Promise<boolean>;
+    every(fn: (item: T, index: number) => boolean | Promise<boolean>): Promise<boolean>;
+}
+export declare function makeStream<T>(batches: () => AsyncIterable<T[]>, batchSize: number, regroup?: boolean): IterStream<T>;

package/dist/stream.js

@@ -0,0 +1,166 @@
+export function makeStream(batches, batchSize, regroup = false) {
+    const derive = (next) => makeStream(next, batchSize, true);
+    const stream = {
+        [Symbol.asyncIterator]() {
+            return flatten(batches())[Symbol.asyncIterator]();
+        },
+        raw() {
+            return regroup ? regroupBatches(batches(), batchSize) : batches();
+        },
+        map(fn) {
+            return derive(() => mapBatches(batches(), fn));
+        },
+        filter(fn) {
+            return derive(() => filterBatches(batches(), fn));
+        },
+        take(limit) {
+            return derive(() => takeBatches(batches(), limit));
+        },
+        drop(limit) {
+            return derive(() => dropBatches(batches(), limit));
+        },
+        async toArray() {
+            const out = [];
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    out.push(batch[i]);
+                }
+            }
+            return out;
+        },
+        async forEach(fn) {
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    await fn(batch[i], index++);
+                }
+            }
+        },
+        async reduce(fn, init) {
+            let acc = init;
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    acc = await fn(acc, batch[i], index++);
+                }
+            }
+            return acc;
+        },
+        async find(fn) {
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    if (await fn(batch[i], index++)) {
+                        return batch[i];
+                    }
+                }
+            }
+            return undefined;
+        },
+        async some(fn) {
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    if (await fn(batch[i], index++)) {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        },
+        async every(fn) {
+            let index = 0;
+            for await (const batch of batches()) {
+                for (let i = 0; i < batch.length; i++) {
+                    if (!(await fn(batch[i], index++))) {
+                        return false;
+                    }
+                }
+            }
+            return true;
+        },
+    };
+    return stream;
+}
+async function* flatten(batches) {
+    for await (const batch of batches) {
+        for (let i = 0; i < batch.length; i++) {
+            yield batch[i];
+        }
+    }
+}
+async function* regroupBatches(batches, size) {
+    let buf = [];
+    for await (const batch of batches) {
+        for (let i = 0; i < batch.length; i++) {
+            buf.push(batch[i]);
+            if (buf.length >= size) {
+                yield buf;
+                buf = [];
+            }
+        }
+    }
+    if (buf.length > 0) {
+        yield buf;
+    }
+}
+async function* mapBatches(batches, fn) {
+    let index = 0;
+    for await (const batch of batches) {
+        const out = new Array(batch.length);
+        for (let i = 0; i < batch.length; i++) {
+            const r = fn(batch[i], index++);
+            out[i] = isThenable(r) ? await r : r;
+        }
+        yield out;
+    }
+}
+async function* filterBatches(batches, fn) {
+    let index = 0;
+    for await (const batch of batches) {
+        const out = [];
+        for (let i = 0; i < batch.length; i++) {
+            const item = batch[i];
+            const r = fn(item, index++);
+            if (isThenable(r) ? await r : r) {
+                out.push(item);
+            }
+        }
+        if (out.length > 0) {
+            yield out;
+        }
+    }
+}
+async function* takeBatches(batches, limit) {
+    if (limit <= 0) {
+        return;
+    }
+    let remaining = limit;
+    for await (const batch of batches) {
+        if (batch.length < remaining) {
+            remaining -= batch.length;
+            yield batch;
+            continue;
+        }
+        yield batch.length === remaining ? batch : batch.slice(0, remaining);
+        return;
+    }
+}
+async function* dropBatches(batches, limit) {
+    let remaining = limit;
+    for await (const batch of batches) {
+        if (remaining === 0) {
+            yield batch;
+        }
+        else if (remaining >= batch.length) {
+            remaining -= batch.length;
+        }
+        else {
+            yield batch.slice(remaining);
+            remaining = 0;
+        }
+    }
+}
+function isThenable(value) {
+    return value != null && typeof value.then === 'function';
+}

package/dist/validate.d.ts

@@ -1,23 +1,9 @@
 import type { StandardSchemaV1 } from '@standard-schema/spec';
-import type { PathFaultCode } from '@botejs/native';
-export type { StandardSchemaV1, PathFaultCode };
-export type Segment = string | number;
-export type Path = readonly Segment[];
-export declare class ValidationError extends Error {
-    readonly issues: readonly StandardSchemaV1.Issue[];
-    readonly path: Path;
-    constructor(issues: readonly StandardSchemaV1.Issue[], path: Path);
-}
-export declare class PathError extends Error {
-    readonly path: Path;
-    /** The fault kind; stable across versions, safe to branch on. */
-    readonly code: PathFaultCode;
-    constructor(path: Path, code: PathFaultCode, segment?: number);
-}
+import { type Path } from './path.ts';
+export type { StandardSchemaV1 };
 export declare function runStandardSchema<O>(schema: StandardSchemaV1<unknown, O>, value: unknown, path: Path): Promise<O>;
 export declare function validateItem<O>(schema: StandardSchemaV1<unknown, O>, value: unknown, path: Path, onInvalid: 'throw' | 'skip'): Promise<{
     skip: true;
 } | {
     value: O;
 }>;
-export declare function formatPath(path: Path): string;

package/dist/validate.js

@@ -1,66 +1,18 @@
-export class ValidationError extends Error {
-    issues;
-    path;
-    constructor(issues, path) {
-        super(`bote: schema validation failed at ${formatPath(path)}: ${issues[0]?.message ?? 'unknown'}`);
-        this.name = 'ValidationError';
-        this.issues = issues;
-        this.path = path;
-    }
-}
-/** Human message per fault kind. The native layer ships only the code (and the
- *  offending `segment` where it matters), so this is the single source of the
- *  user-facing prose. Keyed by the Rust-generated [`PathFaultCode`]. */
-const PATH_FAULT_MESSAGE = {
-    through_scalar: (segment) => `path traverses a non-container value at segment ${segment}`,
-    wrong_kind: (segment) => `path segment ${segment} does not match the container kind`,
-    scalar_target: () => 'target value is not a container',
-    iter_on_object: () => 'iter target is an object; use walk() to iterate object members',
-    walk_on_array: () => 'walk target is an array; use iter() to iterate array elements',
-};
-export class PathError extends Error {
-    path;
-    /** The fault kind; stable across versions, safe to branch on. */
-    code;
-    constructor(path, code, segment) {
-        const reason = (PATH_FAULT_MESSAGE[code] ?? (() => code))(segment);
-        super(`bote: cannot resolve ${formatPath(path)}: ${reason}`);
-        this.name = 'PathError';
-        this.path = path;
-        this.code = code;
-    }
-}
+import { ValidationError } from "./error.js";
 export async function runStandardSchema(schema, value, path) {
     const result = await schema['~standard'].validate(value);
-    if (result.issues)
+    if (result.issues) {
         throw new ValidationError(result.issues, path);
+    }
     return result.value;
 }
 export async function validateItem(schema, value, path, onInvalid) {
     const result = await schema['~standard'].validate(value);
     if (result.issues) {
-        if (onInvalid === 'skip')
+        if (onInvalid === 'skip') {
             return { skip: true };
+        }
         throw new ValidationError(result.issues, path);
     }
     return { value: result.value };
 }
-export function formatPath(path) {
-    if (path.length === 0)
-        return '(root)';
-    let out = '';
-    for (let i = 0; i < path.length; i++) {
-        const seg = path[i];
-        if (typeof seg === 'number') {
-            out += `[${seg}]`;
-            continue;
-        }
-        if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(seg)) {
-            out += i === 0 ? seg : `.${seg}`;
-        }
-        else {
-            out += `[${JSON.stringify(seg)}]`;
-        }
-    }
-    return out;
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botejs/core",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "type": "module",
   "license": "MIT",
   "repository": {
@@ -31,11 +31,12 @@
     "build": "tsc",
     "build:debug": "tsc --sourceMap",
     "test": "node --test __test__/*.spec.ts",
+    "typecheck": "tsc --noEmit -p tsconfig.json && tsc --noEmit -p __test__/tsconfig.json",
     "lint": "oxlint src",
     "prepublishOnly": "cp ../../README.md ./README.md && tsc"
   },
   "dependencies": {
-    "@botejs/native": "^0.4.0"
+    "@botejs/native": "^0.6.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

package/dist/sources.d.ts

@@ -1,39 +0,0 @@
-/**
- * A handle on an opened seekable byte stream. The reader owns whatever
- * resources back the stream (a file handle, an `AbortController`, etc.) and
- * surfaces them through `close()`. Constructed by `Source.open()`; never by
- * library callers directly.
- */
-export interface SourceReader {
-    /** Total length of the underlying byte stream. */
-    readonly size: number;
-    /** Preferred read granularity in bytes. Must be a non-zero multiple of 64. */
-    readonly chunkBytes?: number;
-    /**
-     * Read up to `length` bytes starting at `offset` and resolve with the
-     * bytes read. The returned `Uint8Array`'s `.byteLength` is the actual
-     * count, which must be `<= length`.
-     */
-    read(offset: number, length: number): Promise<Uint8Array>;
-    /** Release resources held by the reader. Driven once by the `open()` lifecycle. */
-    close?(): Promise<void> | void;
-}
-/**
- * Describes how to obtain a seekable byte stream. Provide your own object implementing
- * this interface to plug in custom backends.
- */
-export interface Source {
-    /** Acquire the stream. Resolves to a `SourceReader` that owns any underlying resources. */
-    open(): Promise<SourceReader>;
-}
-export interface FactoryOptions {
-    /** Override the factory's default chunk size. Must be a non-zero multiple of 64. */
-    chunkBytes?: number;
-}
-export interface HttpRangeOptions extends FactoryOptions {
-    /** Merged into every request (headers, credentials, signal, etc.). */
-    init?: RequestInit;
-}
-export declare function fromBuffer(buf: Uint8Array | ArrayBuffer, options?: FactoryOptions): Source;
-export declare function fromFile(path: string, options?: FactoryOptions): Source;
-export declare function fromHttpRange(url: string, options?: HttpRangeOptions): Source;