cesr-ts 0.2.3 → 0.3.1

package/README.md

@@ -9,14 +9,17 @@ CESR annotation.
 npm install cesr-ts
 ```
 
-## Library usage
+## Library quick start
 
 ```ts
-import { createParser } from "cesr-ts";
+import { annotate, createParser } from "cesr-ts";
 
 const parser = createParser();
-const out = parser.feed(new TextEncoder().encode("..."));
+const out = parser.feed(new TextEncoder().encode("...CESR..."));
 const last = parser.flush();
+
+const text = '{"v":"KERI10JSON00002e_","t":"rpy","d":"Eabc"}';
+const annotated = annotate(text, { domainHint: "txt", pretty: true });
 ```
 
 ## CLI usage
@@ -30,3 +33,33 @@ npx cesr-annotate --in mystream.cesr --pretty
 ```bash
 deno task cesr:annotate --in mystream.cesr --pretty
 ```
+
+## Benchmarking (from source)
+
+Run standard parser benchmark baselines:
+
+```bash
+deno task bench:cesr
+```
+
+Run a benchmark on an arbitrary stream:
+
+```bash
+deno task bench:cesr:parser --in ../../samples/cesr-streams/CESR_1_0-oor-auth-vc.cesr
+cat ../../samples/cesr-streams/CESR_1_0-oor-auth-vc.cesr | deno task bench:cesr:parser --iterations 20 --warmup 3
+```
+
+## Using cesr-ts through tufa
+
+`keri-ts` exposes CESR annotation through `tufa annotate`, which is often the
+easiest on-ramp:
+
+```bash
+tufa version
+tufa annotate --in mystream.cesr --pretty
+```
+
+## License
+
+Licensed under the Apache License, Version 2.0 (`Apache-2.0`). See the
+top-level `LICENSE` file in this repository.

package/esm/src/adapters/async-iterable.js

@@ -2,16 +2,16 @@ import { createParser } from "../core/parser-engine.js";
 export async function* toAsyncFrames(source, options = {}) {
     const parser = createParser(options);
     for await (const chunk of source) {
-        const emissions = parser.feed(chunk);
-        for (const emission of emissions) {
-            if (emission.type === "error")
-                throw emission.error;
-            yield emission.frame;
+        const frames = parser.feed(chunk);
+        for (const frame of frames) {
+            if (frame.type === "error")
+                throw frame.error;
+            yield frame.frame;
         }
     }
-    for (const emission of parser.flush()) {
-        if (emission.type === "error")
-            throw emission.error;
-        yield emission.frame;
+    for (const frame of parser.flush()) {
+        if (frame.type === "error")
+            throw frame.error;
+        yield frame.frame;
     }
 }

package/esm/src/annotate/annotator.js

@@ -1,5 +1,5 @@
 import { parseBytes } from "../core/parser-engine.js";
-import { renderAnnotatedFrames } from "./render.js";
+import { renderAnnotatedFrames, renderWrapperAnnotatedStream, } from "./render.js";
 const DEFAULT_OPTIONS = Object.freeze({
     commentMode: "inline",
     indent: 2,
@@ -14,25 +14,33 @@ function resolveOptions(options) {
         ...options,
     };
 }
-function framesOrThrow(emissions) {
-    const frames = [];
-    for (const emission of emissions) {
-        if (emission.type === "error") {
-            throw emission.error;
+function parsedFramesOrThrow(frames) {
+    const parsedFrames = [];
+    for (const frame of frames) {
+        if (frame.type === "error") {
+            throw frame.error;
         }
-        frames.push(emission.frame);
+        parsedFrames.push(frame.frame);
     }
-    return frames;
+    return parsedFrames;
 }
 export function annotateFrames(input, options) {
     const opts = resolveOptions(options);
     const bytes = typeof input === "string"
         ? new TextEncoder().encode(input)
         : input;
-    const frames = framesOrThrow(parseBytes(bytes));
+    const frames = parsedFramesOrThrow(parseBytes(bytes));
     return renderAnnotatedFrames(frames, opts);
 }
 export function annotate(input, options) {
-    const frames = annotateFrames(input, options);
+    const opts = resolveOptions(options);
+    const bytes = typeof input === "string"
+        ? new TextEncoder().encode(input)
+        : input;
+    const wrapperAnnotated = renderWrapperAnnotatedStream(bytes, opts);
+    if (wrapperAnnotated !== null) {
+        return wrapperAnnotated;
+    }
+    const frames = annotateFrames(bytes, opts);
     return frames.map((frame) => frame.lines.join("\n")).join("\n");
 }

package/esm/src/annotate/comments.js

@@ -1,5 +1,6 @@
 import { COUNTER_CODE_NAMES_V1, COUNTER_CODE_NAMES_V2, } from "../tables/counter.tables.generated.js";
 import { MATTER_CODE_NAMES } from "../tables/matter.tables.generated.js";
+import { resolveCounterCodeNameTable } from "../tables/counter-version-registry.js";
 const NATIVE_FIELD_LABELS = Object.freeze({
     v: "version string",
     t: "ilk",
@@ -26,12 +27,8 @@ export function counterCodeName(code) {
     return "Counter";
 }
 export function counterCodeNameForVersion(code, version) {
-    if (version.major >= 2) {
-        return COUNTER_CODE_NAMES_V2[code] ??
-            "Counter";
-    }
-    return COUNTER_CODE_NAMES_V1[code] ??
-        "Counter";
+    const table = resolveCounterCodeNameTable(version);
+    return table[code] ?? "Counter";
 }
 export function matterCodeName(code) {
     return MATTER_CODE_NAMES[code] ?? "Matter";

package/esm/src/annotate/render.js

@@ -1,10 +1,12 @@
+import { parseBytes } from "../core/parser-engine.js";
 import { DeserializeError, GroupSizeError, ShortageError, UnknownCodeError, } from "../core/errors.js";
 import { sniff } from "../parser/cold-start.js";
 import { parseCounter } from "../primitives/counter.js";
 import { parseMatter } from "../primitives/matter.js";
 import { parseIndexer } from "../primitives/indexer.js";
 import { parseAttachmentDispatchCompat } from "../parser/group-dispatch.js";
-import { counterCodeNameForVersion, matterCodeName, nativeLabelName, } from "./comments.js";
+import { counterCodeName, counterCodeNameForVersion, matterCodeName, nativeLabelName, } from "./comments.js";
+import { b64ToInt, intToB64 } from "../core/bytes.js";
 const TEXT_DECODER = new TextDecoder();
 const TEXT_ENCODER = new TextEncoder();
 const OPAQUE_TOKEN_COMMENT = "opaque token";
@@ -15,6 +17,12 @@ const WRAPPER_GROUP_NAMES = new Set([
     "BodyWithAttachmentGroup",
     "BigBodyWithAttachmentGroup",
 ]);
+const TOP_LEVEL_WRAPPER_GROUP_NAMES = new Set([
+    "GenericGroup",
+    "BigGenericGroup",
+    "BodyWithAttachmentGroup",
+    "BigBodyWithAttachmentGroup",
+]);
 function toHex(bytes) {
     return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
 }
@@ -109,27 +117,37 @@ function parseCounterCompat(input, version, domain) {
         return parseCounter(input, alternate, domain);
     }
 }
+function decodeVersionCounter(counter) {
+    const triplet = counter.qb64.length >= 3
+        ? counter.qb64.slice(-3)
+        : intToB64(counter.count, 3);
+    const majorRaw = b64ToInt(triplet[0] ?? "A");
+    const minorRaw = b64ToInt(triplet[1] ?? "A");
+    return {
+        major: majorRaw === 1 ? 1 : 2,
+        minor: minorRaw,
+    };
+}
 function renderGroupItems(lines, items, version, indent, options) {
     for (const item of items) {
-        if (typeof item === "string") {
-            const parsed = describeToken(item, "txt", version);
-            emitLine(lines, item, parsed.comment, indent, options);
+        if (item.kind === "qb64") {
+            const parsed = describeToken(item.qb64, "txt", version);
+            const comment = item.opaque
+                ? OPAQUE_WRAPPER_PAYLOAD_COMMENT
+                : parsed.comment;
+            emitLine(lines, item.qb64, comment, indent, options);
             continue;
         }
-        if (item instanceof Uint8Array) {
-            emitLine(lines, `0x${toHex(item)}`, "raw qb2 quadlet fragment", indent, options);
+        if (item.kind === "qb2") {
+            emitLine(lines, `0x${toHex(item.qb2)}`, item.opaque ? OPAQUE_WRAPPER_PAYLOAD_COMMENT : "raw qb2 triplet token", indent, options);
             continue;
         }
-        if (Array.isArray(item)) {
-            renderGroupItems(lines, item, version, indent + options.indent, options);
+        if (item.kind === "tuple") {
+            renderGroupItems(lines, item.items, version, indent + options.indent, options);
             continue;
         }
-        if (item && typeof item === "object") {
-            const nested = item;
-            if (typeof nested.code === "string" && typeof nested.name === "string") {
-                emitLine(lines, `${nested.code}`, `${nested.name} nested group`, indent, options);
-            }
-            continue;
+        if (item.kind === "group") {
+            emitLine(lines, `${item.code}`, `${item.name} nested group`, indent, options);
         }
     }
 }
@@ -187,14 +205,14 @@ function renderAttachmentGroupRaw(lines, raw, version, indent, options) {
     return parsed.consumed;
 }
 function renderNativeBody(lines, frame, options, version) {
-    const raw = frame.serder.raw;
+    const raw = frame.body.raw;
     const domain = asDomain(raw);
     if (domain !== "txt" && domain !== "bny") {
         return;
     }
     const counter = parseCounterCompat(raw, version, domain);
     emitLine(lines, counter.qb64, `${counterCodeNameForVersion(counter.code, version)} count=${counter.count}`, 0, options);
-    for (const field of frame.serder.native?.fields ?? []) {
+    for (const field of frame.body.native?.fields ?? []) {
         const label = nativeLabelName(field.label);
         const comment = label
             ? `${label} (${matterCodeName(field.code)})`
@@ -203,9 +221,15 @@ function renderNativeBody(lines, frame, options, version) {
     }
 }
 function renderMessageBody(lines, frame, options) {
-    const rawBody = TEXT_DECODER.decode(frame.serder.raw);
+    const rawBody = TEXT_DECODER.decode(frame.body.raw);
+    const isOpaqueCesrBody = frame.body.kind === "CESR" &&
+        frame.body.ked === null;
+    if (isOpaqueCesrBody) {
+        emitLine(lines, rawBody, `OPAQUE CESR body (non-serder fallback, hex=${toHex(frame.body.raw)})`, 0, options);
+        return;
+    }
     let body = rawBody;
-    if (options.pretty && frame.serder.kind === "JSON") {
+    if (options.pretty && frame.body.kind === "JSON") {
         try {
             body = JSON.stringify(JSON.parse(rawBody), null, 2);
         }
@@ -218,17 +242,17 @@ function renderMessageBody(lines, frame, options) {
     }
     const info = [
         `SERDER`,
-        frame.serder.proto,
-        frame.serder.kind,
-        frame.serder.ilk ? `ilk=${frame.serder.ilk}` : null,
-        frame.serder.said ? `said=${frame.serder.said}` : null,
+        frame.body.proto,
+        frame.body.kind,
+        frame.body.ilk ? `ilk=${frame.body.ilk}` : null,
+        frame.body.said ? `said=${frame.body.said}` : null,
     ].filter(Boolean).join(" ");
     emitLine(lines, body, info, 0, options);
 }
 function renderFrame(frame, index, options) {
     const lines = [];
-    const version = frame.serder.gvrsn ?? frame.serder.pvrsn;
-    const domain = asDomain(frame.serder.raw);
+    const version = frame.body.gvrsn ?? frame.body.pvrsn;
+    const domain = asDomain(frame.body.raw);
     if (domain === "txt" || domain === "bny") {
         renderNativeBody(lines, frame, options, version);
     }
@@ -240,6 +264,81 @@ function renderFrame(frame, index, options) {
     }
     return { index, frame, lines };
 }
+function renderFrameChunk(lines, input, indent, options) {
+    const parsed = parseBytes(input);
+    const frames = [];
+    for (const event of parsed) {
+        if (event.type === "error") {
+            throw event.error;
+        }
+        frames.push(event.frame);
+    }
+    const rendered = renderAnnotatedFrames(frames, options);
+    for (const frame of rendered) {
+        for (const line of frame.lines) {
+            lines.push(`${spaces(indent)}${line}`);
+        }
+    }
+}
+function renderWrapperAwareStream(lines, input, inheritedVersion, indent, options) {
+    let offset = 0;
+    let activeVersion = inheritedVersion;
+    let usedWrapper = false;
+    while (offset < input.length) {
+        const slice = input.slice(offset);
+        const domain = asDomain(slice);
+        if (domain !== "txt" && domain !== "bny") {
+            if (!usedWrapper)
+                return false;
+            renderFrameChunk(lines, slice, indent, options);
+            return true;
+        }
+        const counter = parseCounterCompat(slice, activeVersion, domain);
+        const headerSize = domain === "bny" ? counter.fullSizeB2 : counter.fullSize;
+        const name = counterCodeName(counter.code);
+        if (name === "KERIACDCGenusVersion") {
+            emitLine(lines, counter.qb64, `${name} count=${counter.count}`, indent, options);
+            activeVersion = decodeVersionCounter(counter);
+            offset += headerSize;
+            usedWrapper = true;
+            continue;
+        }
+        if (!TOP_LEVEL_WRAPPER_GROUP_NAMES.has(name)) {
+            if (!usedWrapper)
+                return false;
+            renderFrameChunk(lines, slice, indent, options);
+            return true;
+        }
+        const unit = domain === "bny" ? 3 : 4;
+        const payloadSize = counter.count * unit;
+        const total = headerSize + payloadSize;
+        if (slice.length < total) {
+            throw new ShortageError(total, slice.length);
+        }
+        emitLine(lines, counter.qb64, `${name} count=${counter.count}`, indent, options);
+        const payload = slice.slice(headerSize, total);
+        const nestedHandled = renderWrapperAwareStream(lines, payload, activeVersion, indent + options.indent, options);
+        if (!nestedHandled) {
+            renderFrameChunk(lines, payload, indent + options.indent, options);
+        }
+        offset += total;
+        usedWrapper = true;
+    }
+    return usedWrapper;
+}
+/**
+ * Render stream-level wrapper groups (GenericGroup/BodyWithAttachmentGroup)
+ * so denot round-trips preserve wrapper counters. Returns null when input does
+ * not start in a wrapper-oriented domain and caller should use frame rendering.
+ */
+export function renderWrapperAnnotatedStream(input, options) {
+    const lines = [];
+    const rendered = renderWrapperAwareStream(lines, input, { major: 2, minor: 0 }, 0, options);
+    if (!rendered) {
+        return null;
+    }
+    return lines.join("\n");
+}
 /** Render parsed CESR frames into line-oriented, human-annotated text blocks. */
 export function renderAnnotatedFrames(frames, options) {
     return frames.map((frame, index) => renderFrame(frame, index + 1, options));

package/esm/src/bench/parser-benchmark.js

@@ -0,0 +1,134 @@
+import { createParser } from "../core/parser-engine.js";
+function asPositiveInteger(value, fallback) {
+    if (typeof value !== "number" || !Number.isFinite(value)) {
+        return fallback;
+    }
+    return Math.max(0, Math.trunc(value));
+}
+/**
+ * Normalize chunking to a legal parser-feed size.
+ *
+ * Boundary rules:
+ * - `<= 0` means full-stream feed for each iteration.
+ * - values larger than stream size collapse to full-stream feed.
+ */
+function normalizeChunkSize(chunkSize, totalBytes) {
+    const normalized = asPositiveInteger(chunkSize, 0);
+    if (normalized <= 0 || normalized >= totalBytes) {
+        return totalBytes;
+    }
+    return normalized;
+}
+/**
+ * Produce deterministic feed slices for one parse run.
+ *
+ * Invariant:
+ * - slices preserve original stream ordering.
+ * - slices are view-backed (`subarray`) to avoid benchmark-side copy noise.
+ */
+function buildChunks(input, chunkSize) {
+    if (chunkSize >= input.length) {
+        return [input];
+    }
+    const chunks = [];
+    for (let offset = 0; offset < input.length; offset += chunkSize) {
+        chunks.push(input.subarray(offset, Math.min(offset + chunkSize, input.length)));
+    }
+    return chunks;
+}
+/**
+ * Count parser frame/error events for one parser emission batch.
+ */
+function countFrames(events) {
+    let frameCount = 0;
+    let errorCount = 0;
+    for (const event of events) {
+        if (event.type === "frame") {
+            frameCount++;
+            continue;
+        }
+        errorCount++;
+    }
+    return { frameCount, errorCount };
+}
+/**
+ * Execute one complete parser pass over the provided stream.
+ *
+ * Boundary contract:
+ * - Always feeds configured chunks and then flushes once.
+ * - Returns event counts only; callers own timing and aggregation.
+ */
+export function parseCesrStreamOnce(input, options = {}) {
+    const chunkSize = normalizeChunkSize(options.chunkSize, input.length);
+    const parser = createParser(options.parserOptions);
+    let frameCount = 0;
+    let errorCount = 0;
+    for (const chunk of buildChunks(input, chunkSize)) {
+        const summary = countFrames(parser.feed(chunk));
+        frameCount += summary.frameCount;
+        errorCount += summary.errorCount;
+    }
+    const flushSummary = countFrames(parser.flush());
+    frameCount += flushSummary.frameCount;
+    errorCount += flushSummary.errorCount;
+    return { frameCount, errorCount };
+}
+/**
+ * Benchmark parser throughput across warmup + measured iterations.
+ *
+ * Invariants:
+ * - warmup runs are excluded from final metrics.
+ * - when `failOnParseError` is true, any parse error fails fast to prevent
+ *   silently benchmarking degraded correctness.
+ */
+export function benchmarkCesrParser(input, options = {}) {
+    if (input.length === 0) {
+        throw new Error("Benchmark input stream must not be empty");
+    }
+    const iterations = asPositiveInteger(options.iterations, 50);
+    if (iterations <= 0) {
+        throw new Error("iterations must be greater than 0");
+    }
+    const warmupIterations = asPositiveInteger(options.warmupIterations, 5);
+    const chunkSize = normalizeChunkSize(options.chunkSize, input.length);
+    const failOnParseError = options.failOnParseError ?? true;
+    for (let i = 0; i < warmupIterations; i++) {
+        parseCesrStreamOnce(input, {
+            chunkSize,
+            parserOptions: options.parserOptions,
+        });
+    }
+    let totalFrames = 0;
+    let totalErrors = 0;
+    const startMs = performance.now();
+    for (let i = 0; i < iterations; i++) {
+        const run = parseCesrStreamOnce(input, {
+            chunkSize,
+            parserOptions: options.parserOptions,
+        });
+        totalFrames += run.frameCount;
+        totalErrors += run.errorCount;
+        if (failOnParseError && run.errorCount > 0) {
+            throw new Error(`Benchmark run produced parse errors (run=${i + 1}, errorCount=${run.errorCount})`);
+        }
+    }
+    const elapsedMs = Math.max(performance.now() - startMs, Number.EPSILON);
+    const totalBytes = input.length * iterations;
+    const throughputBytesPerSec = (totalBytes * 1000) / elapsedMs;
+    const throughputMiBPerSec = throughputBytesPerSec / (1024 * 1024);
+    const framesPerSec = (totalFrames * 1000) / elapsedMs;
+    return {
+        iterations,
+        warmupIterations,
+        chunkSize,
+        bytesPerIteration: input.length,
+        totalBytes,
+        totalFrames,
+        totalErrors,
+        elapsedMs,
+        avgIterationMs: elapsedMs / iterations,
+        throughputBytesPerSec,
+        throughputMiBPerSec,
+        framesPerSec,
+    };
+}

package/esm/src/core/bytes.js

@@ -1,6 +1,12 @@
 export const encoder = new TextEncoder();
 export const decoder = new TextDecoder();
 const B64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
+/**
+ * Concatenates Uint8Array byte arrays into one byte array
+ *
+ * @param chunks chunks to concatenate
+ * @returns concatenated bytearray
+ */
 export function concatBytes(...chunks) {
     const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
     const out = new Uint8Array(total);

package/esm/src/core/errors.js

@@ -43,3 +43,27 @@ export class GroupSizeError extends ParserError {
 }
 export class DeserializeError extends ParserError {
 }
+/** Failure while constructing syntax artifacts from token bytes. */
+export class SyntaxParseError extends ParserError {
+    constructor(message, cause, offset, context) {
+        super(message, offset, context);
+        Object.defineProperty(this, "cause", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: cause
+        });
+    }
+}
+/** Failure while interpreting syntax artifacts into semantic fields. */
+export class SemanticInterpretationError extends ParserError {
+    constructor(message, cause, offset, context) {
+        super(message, offset, context);
+        Object.defineProperty(this, "cause", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: cause
+        });
+    }
+}