@oh-my-pi/pi-utils 18.1.5 → 18.1.6

package/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## [Unreleased]
 
+## [18.1.6] - 2026-09-03
+
+### Added
+
+- Added `IncomingDoc` (`@oh-my-pi/pi-utils/incoming-json`) for incrementally reading path-addressed JSON data as text arrives, including string chunks and lines, array elements, and keyed object values, with structured errors for missing, incomplete, aborted, malformed, or mismatched data.
+- Added `Serial` for running asynchronous operations sequentially in call order.
+
+### Fixed
+
+- Fixed relaxed JSON parsing for single-quoted strings followed by line or block comments.
+
 ## [18.1.5] - 2026-09-03
 
 ### Added

package/dist/types/async.d.ts

@@ -20,3 +20,13 @@ export declare class AsyncDrain<T> {
     /** Runs the pending batch handler immediately and returns its completion promise. */
     flush(): Promise<void>;
 }
+/**
+ * Runs async operations one at a time in call order. Each `run` starts after
+ * the previous operation settles (success or failure) and returns that
+ * operation's own promise, so a rejected step never poisons the queue. Used by
+ * stateful cursors whose concurrent pulls must not interleave.
+ */
+export declare class Serial {
+    #private;
+    run<T>(op: () => Promise<T>): Promise<T>;
+}

package/dist/types/incoming-json.d.ts

@@ -0,0 +1,232 @@
+/**
+ * Typed cursors over a JSON document while its text is still arriving.
+ *
+ * {@link IncomingDoc.channel} returns a push-side {@link IncomingFeed} and a
+ * read-side {@link IncomingDoc}. The producer appends text fragments, then
+ * explicitly calls {@link IncomingFeed.finish} or {@link IncomingFeed.abort};
+ * a feed that is never closed leaves every pending pull waiting forever.
+ * There is one shared append-only buffer; cursors are cheap, immutable
+ * path handles over it, and every pull is an ordinary promise that
+ * re-scans the buffer whenever the feed changes. There are no snapshots,
+ * per-field events, or fan-out channels.
+ *
+ * A scalar completes at its closing quote/delimiter, and a container
+ * completes only when its closing delimiter arrives. Finished-but-truncated
+ * input rejects with kind `incomplete`; abandoned input rejects with
+ * `aborted`. String chunks contain only decoded text whose meaning is
+ * stable, so an escape or Unicode escape may span any number of fragments.
+ *
+ * Pulling an {@link IncomingObject.key} makes that key required: a missing
+ * or mistyped value is a structured {@link IncomingJsonError}. Object members
+ * never pulled are skipped without validation. {@link IncomingDoc.whole} is
+ * the explicit whole-document pull and runs only after successful input
+ * completion.
+ *
+ * Object cursors bind the first occurrence of a duplicate key, whereas
+ * complete-value pulls (`value()`, `collect()`, `whole()`) go through the
+ * final parser (`parseJsonWithRepair`), whose objects are last-write-wins.
+ *
+ * Mid-stream cursors tolerate incomplete tokens but read double-quoted
+ * strings with the final parser's strict closing rule: an unescaped inner
+ * `"` can never swallow a sibling key or value. A pulled scalar completes
+ * only once a value terminator follows it, like numbers, so structural
+ * garbage after a value surfaces as `incomplete` rather than a silently
+ * misparsed pull. Single-quote recovery (`'it's'`) is shared with the final
+ * parser and passes both.
+ *
+ * @example
+ * ```ts
+ * const { feed, doc } = IncomingDoc.channel();
+ * const args = doc.root().object();
+ * const content = args.key("content").string();
+ * feed.push('{"path":"a.ts","content":"hel');
+ * await content.nextChunk(); // "hel"
+ * feed.push('lo"}');
+ * feed.finish();
+ * await content.nextChunk(); // "lo"
+ * await content.nextChunk(); // undefined
+ * await args.key("path").value<string>(); // "a.ts"
+ * ```
+ */
+/** Location component in a pulled JSON path: an object member name or an array index. */
+export type PullPathSegment = string | number;
+/** JSON shape observed by a started pull. */
+export type IncomingValueKind = "null" | "boolean" | "number" | "string" | "array" | "object";
+/** Why a pull could not produce the requested shape. */
+export type PullIssueKind = 
+/** The requested member was absent when its container completed. */
+"missing"
+/** The producer finished before the pulled value's closing token. */
+ | "incomplete"
+/** The producer abandoned the input before the pull completed. */
+ | "aborted"
+/** A complete pulled value could not be parsed. */
+ | "malformed"
+/** A value was present with a different JSON shape (see `found`). */
+ | "mismatch";
+/** Structured failure while awaiting an incoming JSON value. */
+export declare class IncomingJsonError extends Error {
+    /** Full key/index path pulled by the consumer. */
+    readonly path: readonly PullPathSegment[];
+    /** Shape requested by the typed cursor. */
+    readonly expected: string;
+    /** Why the pull could not produce that shape. */
+    readonly kind: PullIssueKind;
+    /** Shape observed in the input when `kind` is `mismatch`. */
+    readonly found?: string;
+    constructor(path: readonly PullPathSegment[], expected: string, kind: PullIssueKind, options?: {
+        found?: string;
+        cause?: unknown;
+    });
+}
+type Shape = {
+    kind: "null" | "array" | "object";
+} | {
+    kind: "boolean";
+    value: boolean;
+} | {
+    kind: "number";
+    value: number;
+} | {
+    kind: "string";
+    value: string;
+    stableLen: number;
+};
+interface Located {
+    tag: "located";
+    /** Source offset of the value's first char. */
+    start: number;
+    /** Source offset just past the value's closing token; `undefined` while it is still open. */
+    end: number | undefined;
+    shape: Shape;
+}
+/** Readiness predicate a pull waits for once its value has been located. */
+type Ready = (located: Located) => boolean;
+/** Append-only buffer, terminal state, and change notification shared by one feed and its cursors. */
+declare class Shared {
+    #private;
+    text: string;
+    end: "open" | "finished" | "aborted";
+    /** Resolves once the buffer or terminal state changes after this call. */
+    get changed(): Promise<void>;
+    notify(): void;
+    /** Await input completion; rejects with `aborted` when the feed was abandoned. */
+    finished(): Promise<void>;
+    /**
+     * Await the value at `path` until `ready` accepts it. Resolves `undefined`
+     * when the value's container completed without it.
+     */
+    pull(path: readonly PullPathSegment[], expected: string, ready: Ready): Promise<Located | undefined>;
+}
+/**
+ * Push side of an {@link IncomingDoc} channel. Call {@link finish} to mark
+ * the document complete or {@link abort} to abandon it; both are idempotent
+ * and every pending pull settles on the first one.
+ */
+export declare class IncomingFeed {
+    #private;
+    constructor(shared: Shared);
+    /** Append one text fragment and wake every pending cursor. Throws once the feed is closed. */
+    push(fragment: string): void;
+    /** Mark the input complete. */
+    finish(): void;
+    /** Abandon the input; pending and future pulls reject with kind `aborted`. */
+    abort(): void;
+}
+/** Read side of one growing JSON document. */
+export declare class IncomingDoc {
+    #private;
+    constructor(shared: Shared);
+    /** Create a push feed and its read-side document. */
+    static channel(): {
+        feed: IncomingFeed;
+        doc: IncomingDoc;
+    };
+    /** Text received so far. */
+    get text(): string;
+    /** Await explicit input completion; rejects with kind `aborted` if the feed was abandoned. */
+    finished(): Promise<void>;
+    /**
+     * Parse the entire finished document with the final tolerant parser.
+     * Waits for {@link IncomingFeed.finish}; aborted input is not decoded and a
+     * document that fails the final parse rejects with kind `malformed`.
+     */
+    whole<T = unknown>(): Promise<T>;
+    /** Cursor for the root JSON value. */
+    root(): IncomingJson;
+}
+/** Cursor for one JSON value in the incoming document, addressed by path. */
+export declare class IncomingJson {
+    #private;
+    readonly path: readonly PullPathSegment[];
+    constructor(shared: Shared, path: readonly PullPathSegment[]);
+    /** Await the value's first token and report its JSON shape. */
+    kind(): Promise<IncomingValueKind>;
+    /** Await and parse the complete value. Containers go through the final tolerant parser. */
+    value<T = unknown>(): Promise<T>;
+    /** Await a complete number. */
+    number(): Promise<number>;
+    /** Await a complete boolean. */
+    boolean(): Promise<boolean>;
+    /** Await a complete `null`. */
+    null(): Promise<null>;
+    /** View this value as an incremental decoded string. */
+    string(): IncomingString;
+    /** View this value as an array of element cursors. */
+    array(): IncomingArray;
+    /** View this value as an object with keyed cursors. */
+    object(): IncomingObject;
+}
+/**
+ * Incremental decoded string consumer. Chunks are emitted in order without
+ * overlap and are always prefixes of the final decoded string;
+ * {@link text} returns the complete string independently of whether chunks
+ * were consumed. Async iteration yields chunks. Concurrent `nextChunk` /
+ * `nextLine` calls are served in call order, like a stream reader: a call
+ * whose result is abandoned still consumes its chunk.
+ */
+export declare class IncomingString implements AsyncIterable<string> {
+    #private;
+    constructor(shared: Shared, path: readonly PullPathSegment[]);
+    /** Await the next stable decoded chunk, or `undefined` after the closing quote. */
+    nextChunk(): Promise<string | undefined>;
+    /**
+     * Await the next complete decoded line, retaining its trailing newline.
+     * A final unterminated line is returned once the closing quote arrives.
+     */
+    nextLine(): Promise<string | undefined>;
+    /** Iterate complete decoded lines. */
+    lines(): AsyncGenerator<string, void, undefined>;
+    [Symbol.asyncIterator](): AsyncGenerator<string, void, undefined>;
+    /** Await the closing quote and return the complete decoded string. */
+    text(): Promise<string>;
+}
+/**
+ * Linear cursor over elements of an incoming array. Async iteration yields
+ * element cursors. Concurrent `next` calls are served in call order.
+ */
+export declare class IncomingArray implements AsyncIterable<IncomingJson> {
+    #private;
+    constructor(shared: Shared, path: readonly PullPathSegment[]);
+    /** Await the start of the next element; `undefined` only after the closing bracket. */
+    next(): Promise<IncomingJson | undefined>;
+    [Symbol.asyncIterator](): AsyncGenerator<IncomingJson, void, undefined>;
+    /** Await the closing bracket and collect the fully parsed elements. */
+    collect<T = unknown>(): Promise<T[]>;
+}
+/** Keyed cursor and final collection for an incoming object. */
+export declare class IncomingObject {
+    #private;
+    constructor(shared: Shared, path: readonly PullPathSegment[]);
+    /**
+     * Cursor bound to the first occurrence of `name`. Pulling it makes the
+     * key required: a missing member rejects with kind `missing`.
+     */
+    key(name: string): IncomingJson;
+    /**
+     * Await the closing brace and collect the object through the final parser,
+     * whose duplicate keys are last-write-wins unlike {@link key}.
+     */
+    collect<T = Record<string, unknown>>(): Promise<T>;
+}
+export {};

package/dist/types/index.d.ts

@@ -10,6 +10,7 @@ export * from "./format.js";
 export * from "./frontmatter.js";
 export * from "./fs-error.js";
 export * from "./glob.js";
+export * from "./incoming-json.js";
 export * from "./json.js";
 export * from "./json-parse.js";
 export * as logger from "./logger.js";

package/dist/types/json-lexer.d.ts

@@ -0,0 +1,116 @@
+/**
+ * Tolerant JSON lexer shared by the final parser (`parseJsonWithRepair`), the
+ * streaming partial builder (`parseStreamingJson`), and the incoming cursors
+ * (`IncomingDoc`). {@link JsonLexerMode} selects how truncation and unescaped
+ * inner double quotes are treated.
+ *
+ * The grammar is a forgiving superset of JSON covering malformations commonly
+ * produced by language models:
+ *
+ * - single-quoted strings and unquoted object keys (JSON5);
+ * - trailing / stray commas, and `//` + block comments;
+ * - Python literals `True` / `False` / `None`, plus `0x` / `0b` numbers;
+ * - raw control characters and invalid `\x` escapes inside strings (kept literally);
+ * - unescaped quotes inside strings — a single quote only closes a string when
+ *   followed by a value terminator, recovering apostrophes such as `'it's'`;
+ *   the same recovery applies to double quotes in `streaming` mode only,
+ *   everywhere else they close strictly;
+ * - unquoted string values in value position — an unrecognized bareword such
+ *   as `{"paths": packages/foo/*}` is recovered as a string up to the next
+ *   `,` / `}` / `]` / newline.
+ */
+export declare const QUOTE = 34;
+export declare const SQUOTE = 39;
+export declare const BACKSLASH = 92;
+export declare const SLASH = 47;
+export declare const COLON = 58;
+export declare const COMMA = 44;
+export declare const LBRACE = 123;
+export declare const RBRACE = 125;
+export declare const LBRACKET = 91;
+export declare const RBRACKET = 93;
+/** Valid chars after `\` in a strict JSON escape: `" \ / b f n r t u`. */
+export declare const VALID_ESCAPE_CHAR: Uint8Array<ArrayBuffer>;
+export declare function isHexDigit(cp: number): boolean;
+/** JSON insignificant whitespace (RFC 8259 §2). */
+export declare function isWhitespace(cp: number): boolean;
+/** First char of a numeric token: sign, dot, or digit. */
+export declare function isNumberStart(cp: number): boolean;
+/**
+ * Grammar tolerance selected by the lexer's consumer.
+ *
+ * - `strict`: final parse — complete input required, double quotes close strictly.
+ * - `streaming`: mid-stream snapshot — incomplete tokens tolerated and unescaped
+ *   inner double quotes recovered for display.
+ * - `incoming`: incremental typed pulls — incomplete tokens tolerated, but double
+ *   quotes close strictly so pulled values match the final parse.
+ */
+export type JsonLexerMode = "strict" | "streaming" | "incoming";
+/** Decoded state of a string token at the current streaming edge. */
+export interface JsonStringProgress {
+    /** Decoded content so far (complete when `complete` is true). */
+    value: string;
+    /**
+     * Length of the prefix of `value` whose meaning cannot change when more
+     * input arrives. Excludes a trailing split escape, a high surrogate whose
+     * low half may still follow, and everything from a quote whose close/inner
+     * reading is still undecidable at the buffer edge.
+     */
+    stableLen: number;
+    /** Whether the closing quote was consumed. */
+    complete: boolean;
+}
+/**
+ * Cursor over the input with the tolerant token readers. `pos` is the current
+ * offset; readers advance it. In `strict` mode a truncated or malformed token
+ * throws `SyntaxError`; the lenient modes report progress or return
+ * `undefined` so the caller can roll back.
+ */
+export declare class JsonLexer {
+    #private;
+    readonly src: string;
+    readonly mode: JsonLexerMode;
+    pos: number;
+    constructor(src: string, mode: JsonLexerMode, pos?: number);
+    get atEnd(): boolean;
+    /** Char code at the cursor; `NaN` at end of input (so every comparison is false). */
+    peek(): number;
+    /** Skip whitespace plus `//` line and `/* *\/` block comments. */
+    ws(): void;
+    /**
+     * Read a string starting at the opening `quote`, retaining the information
+     * an incremental consumer needs. Strict mode throws on an unterminated
+     * string; lenient modes consume to the end of input and report progress.
+     */
+    string(quote: number): JsonStringProgress;
+    /**
+     * Read a numeric token with JS `Number()` semantics (decimal with optional
+     * sign / leading or trailing dot / exponent, plus `0x` hex and `0b` binary).
+     * Non-finite or malformed tokens throw in strict mode and return
+     * `undefined` in lenient modes; the cursor is left past the token either way.
+     */
+    number(): number | undefined;
+    /**
+     * Match a keyword literal at the cursor; consumes only on success and
+     * returns `undefined` otherwise. Requires a non-identifier boundary so
+     * `Truex` / `nullish` are not misread as the keyword followed by junk.
+     */
+    keyword(): boolean | null | undefined;
+    /** Read an unquoted object key: everything up to `:` / `,` / `}` / whitespace. May be empty. */
+    unquotedKey(): string;
+    /**
+     * Recover an unquoted string value, e.g. `{"paths": packages/foo/*}`:
+     * consume until `,` / `}` / `]` / newline and trim trailing whitespace.
+     * Recovery still fails — so a final parse never accepts a half-formed or
+     * non-finite argument — when the token:
+     * - hits end-of-input before a delimiter (truncated value);
+     * - contains a `"`, `{`, `[`, or a key-like `:` — this grammar accepts
+     *   unquoted keys, so a missed comma (`{"a": foo "b": 1}`) would otherwise
+     *   silently swallow the following field. A colon followed by `/` or `\`
+     *   stays literal so URL and Windows-path values recover;
+     * - is a non-finite atom ({@link NON_RECOVERABLE_BAREWORDS}).
+     *
+     * Failure throws in strict mode and returns `undefined` in lenient modes.
+     */
+    bareword(): string | undefined;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@oh-my-pi/pi-utils",
-	"version": "18.1.5",
+	"version": "18.1.6",
 	"description": "Shared utilities for pi packages",
 	"homepage": "https://omp.sh",
 	"author": "Stencil Labs, Inc.",
@@ -31,7 +31,7 @@
 		"fmt": "oxfmt --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts'"
 	},
 	"dependencies": {
-		"@oh-my-pi/pi-natives": "18.1.5"
+		"@oh-my-pi/pi-natives": "18.1.6"
 	},
 	"devDependencies": {
 		"@types/bun": "^1.3.14"

package/src/async.ts

@@ -108,3 +108,19 @@ export class AsyncDrain<T> {
 		return this.#promise;
 	}
 }
+
+/**
+ * Runs async operations one at a time in call order. Each `run` starts after
+ * the previous operation settles (success or failure) and returns that
+ * operation's own promise, so a rejected step never poisons the queue. Used by
+ * stateful cursors whose concurrent pulls must not interleave.
+ */
+export class Serial {
+	#tail: Promise<unknown> = Promise.resolve();
+
+	run<T>(op: () => Promise<T>): Promise<T> {
+		const result = this.#tail.then(op, op);
+		this.#tail = result.catch(() => {});
+		return result;
+	}
+}