keri-ts 0.3.1 → 0.4.0

package/README.md

@@ -83,5 +83,5 @@ deno task tufa version
 
 ## License
 
-Licensed under the Apache License, Version 2.0 (`Apache-2.0`). See the
-top-level `LICENSE` file in this repository.
+Licensed under the Apache License, Version 2.0 (`Apache-2.0`). See the top-level
+`LICENSE` file in this repository.

package/esm/cesr/mod.js

@@ -0,0 +1,2 @@
+export * from "./src/index.js";
+export * from "./src/version.js";

package/esm/cesr/src/adapters/async-iterable.js

@@ -0,0 +1,17 @@
+import { createParser } from "../core/parser-engine.js";
+export async function* toAsyncFrames(source, options = {}) {
+    const parser = createParser(options);
+    for await (const chunk of source) {
+        const frames = parser.feed(chunk);
+        for (const frame of frames) {
+            if (frame.type === "error")
+                throw frame.error;
+            yield frame.frame;
+        }
+    }
+    for (const frame of parser.flush()) {
+        if (frame.type === "error")
+            throw frame.error;
+        yield frame.frame;
+    }
+}

package/esm/cesr/src/adapters/effection.js

@@ -0,0 +1,9 @@
+import { action } from "effection";
+import { toAsyncFrames } from "./async-iterable.js";
+export function* toEffectionFrames(source, options = {}) {
+    const frames = yield* action((resolve) => {
+        resolve(toAsyncFrames(source, options));
+        return () => { };
+    });
+    return { frames };
+}

package/esm/cesr/src/annotate/annotator.js

@@ -0,0 +1,43 @@
+import { b } from "../core/bytes.js";
+import { parseBytes } from "../core/parser-engine.js";
+import { renderAnnotatedFrames, renderWrapperAnnotatedStream } from "./render.js";
+const DEFAULT_OPTIONS = Object.freeze({
+    commentMode: "inline",
+    indent: 2,
+    showOffsets: false,
+    showRawHex: false,
+    domainHint: "auto",
+    pretty: false,
+});
+function resolveOptions(options) {
+    return {
+        ...DEFAULT_OPTIONS,
+        ...options,
+    };
+}
+function parsedFramesOrThrow(frames) {
+    const parsedFrames = [];
+    for (const frame of frames) {
+        if (frame.type === "error") {
+            throw frame.error;
+        }
+        parsedFrames.push(frame.frame);
+    }
+    return parsedFrames;
+}
+export function annotateFrames(input, options) {
+    const opts = resolveOptions(options);
+    const bytes = typeof input === "string" ? b(input) : input;
+    const frames = parsedFramesOrThrow(parseBytes(bytes));
+    return renderAnnotatedFrames(frames, opts);
+}
+export function annotate(input, options) {
+    const opts = resolveOptions(options);
+    const bytes = typeof input === "string" ? b(input) : input;
+    const wrapperAnnotated = renderWrapperAnnotatedStream(bytes, opts);
+    if (wrapperAnnotated !== null) {
+        return wrapperAnnotated;
+    }
+    const frames = annotateFrames(bytes, opts);
+    return frames.map((frame) => frame.lines.join("\n")).join("\n");
+}

package/esm/cesr/src/annotate/comments.js

@@ -0,0 +1,40 @@
+import { resolveCounterCodeNameTable } from "../tables/counter-version-registry.js";
+import { COUNTER_CODE_NAMES_V1, COUNTER_CODE_NAMES_V2, } from "../tables/counter.tables.generated.js";
+import { MATTER_CODE_NAMES } from "../tables/matter.tables.generated.js";
+const NATIVE_FIELD_LABELS = Object.freeze({
+    v: "version string",
+    t: "ilk",
+    d: "SAID",
+    i: "AID prefix",
+    s: "sequence number",
+    p: "prior event digest",
+    kt: "signing threshold",
+    k: "current keys",
+    nt: "next threshold",
+    n: "next keys digest",
+    bt: "witness threshold",
+    b: "witness list",
+    c: "traits/config",
+    a: "anchors/data",
+});
+export function counterCodeName(code) {
+    const v2 = COUNTER_CODE_NAMES_V2[code];
+    if (v2)
+        return v2;
+    const v1 = COUNTER_CODE_NAMES_V1[code];
+    if (v1)
+        return v1;
+    return "Counter";
+}
+export function counterCodeNameForVersion(code, version) {
+    const table = resolveCounterCodeNameTable(version);
+    return table[code] ?? "Counter";
+}
+export function matterCodeName(code) {
+    return MATTER_CODE_NAMES[code] ?? "Matter";
+}
+export function nativeLabelName(label) {
+    if (!label)
+        return null;
+    return NATIVE_FIELD_LABELS[label] ?? label;
+}

package/esm/cesr/src/annotate/denot.js

@@ -0,0 +1,17 @@
+const TEXT_ENCODER = new TextEncoder();
+export function denot(annotated) {
+    const tokens = [];
+    for (const line of annotated.split(/\r?\n/)) {
+        const trimmed = line.trim();
+        if (!trimmed || trimmed.startsWith("#")) {
+            continue;
+        }
+        const commentIndex = line.indexOf("#");
+        const sansComment = (commentIndex >= 0 ? line.slice(0, commentIndex) : line)
+            .trim();
+        if (!sansComment)
+            continue;
+        tokens.push(sansComment);
+    }
+    return TEXT_ENCODER.encode(tokens.join(""));
+}

package/esm/cesr/src/annotate/render.js

@@ -0,0 +1,336 @@
+import { b64ToInt, intToB64 } from "../core/bytes.js";
+import { DeserializeError, GroupSizeError, ShortageError, UnknownCodeError, } from "../core/errors.js";
+import { parseBytes } from "../core/parser-engine.js";
+import { sniff } from "../parser/cold-start.js";
+import { parseAttachmentDispatchCompat } from "../parser/group-dispatch.js";
+import { Counter, parseCounter } from "../primitives/counter.js";
+import { Indexer } from "../primitives/indexer.js";
+import { Matter, parseMatter } from "../primitives/matter.js";
+import { isCounterGroupLike, isPrimitiveTuple } from "../primitives/primitive.js";
+import { UnknownPrimitive } from "../primitives/unknown.js";
+import { counterCodeName, counterCodeNameForVersion, matterCodeName, nativeLabelName, } from "./comments.js";
+const TEXT_DECODER = new TextDecoder();
+const OPAQUE_TOKEN_COMMENT = "opaque token";
+const OPAQUE_WRAPPER_PAYLOAD_COMMENT = "opaque wrapper payload";
+const WRAPPER_GROUP_NAMES = new Set([
+    "AttachmentGroup",
+    "BigAttachmentGroup",
+    "BodyWithAttachmentGroup",
+    "BigBodyWithAttachmentGroup",
+]);
+const TOP_LEVEL_WRAPPER_GROUP_NAMES = new Set([
+    "GenericGroup",
+    "BigGenericGroup",
+    "BodyWithAttachmentGroup",
+    "BigBodyWithAttachmentGroup",
+]);
+function toHex(bytes) {
+    return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+function isRecoverableParseError(error) {
+    return error instanceof UnknownCodeError
+        || error instanceof DeserializeError
+        || error instanceof ShortageError
+        || error instanceof GroupSizeError;
+}
+function spaces(count) {
+    return " ".repeat(Math.max(0, count));
+}
+function asDomain(raw) {
+    try {
+        const cold = sniff(raw);
+        if (cold === "txt" || cold === "bny" || cold === "msg")
+            return cold;
+    }
+    catch (error) {
+        if (!isRecoverableParseError(error)) {
+            throw error;
+        }
+        // Fallback handled below.
+    }
+    return "msg";
+}
+function emitLine(lines, value, comment, indent, options) {
+    const pad = spaces(indent);
+    if (options.commentMode === "above") {
+        lines.push(`${pad}# ${comment}`);
+        lines.push(`${pad}${value}`);
+        return;
+    }
+    lines.push(`${pad}${value} # ${comment}`);
+}
+function parseCounterCompat(input, version, domain) {
+    try {
+        return parseCounter(input, version, domain);
+    }
+    catch (error) {
+        if (!isRecoverableParseError(error)) {
+            throw error;
+        }
+        const alternate = version.major >= 2
+            ? { major: 1, minor: 0 }
+            : { major: 2, minor: 0 };
+        return parseCounter(input, alternate, domain);
+    }
+}
+function decodeVersionCounter(counter) {
+    const triplet = counter.qb64.length >= 3
+        ? counter.qb64.slice(-3)
+        : intToB64(counter.count, 3);
+    const majorRaw = b64ToInt(triplet[0] ?? "A");
+    const minorRaw = b64ToInt(triplet[1] ?? "A");
+    return {
+        major: majorRaw === 1 ? 1 : 2,
+        minor: minorRaw,
+    };
+}
+function renderGroupItems(lines, items, version, indent, options) {
+    for (const item of items) {
+        if (isPrimitiveTuple(item)) {
+            renderGroupItems(lines, item, version, indent + options.indent, options);
+            continue;
+        }
+        if (isCounterGroupLike(item)) {
+            emitLine(lines, item.qb64, `${item.name} nested group`, indent, options);
+            renderGroupItems(lines, [...item.items], version, indent + options.indent, options);
+            continue;
+        }
+        if (item instanceof UnknownPrimitive) {
+            const value = item.sourceDomain === "bny"
+                ? `0x${toHex(item.qb2)}`
+                : item.qb64;
+            emitLine(lines, value, OPAQUE_WRAPPER_PAYLOAD_COMMENT, indent, options);
+            continue;
+        }
+        if (item instanceof Counter) {
+            emitLine(lines, item.qb64, `${counterCodeNameForVersion(item.code, version)} counter`, indent, options);
+            continue;
+        }
+        if (item instanceof Indexer) {
+            emitLine(lines, item.qb64, `Indexer ${item.code}`, indent, options);
+            continue;
+        }
+        if (item instanceof Matter) {
+            emitLine(lines, item.qb64, matterCodeName(item.code), indent, options);
+            continue;
+        }
+        emitLine(lines, String(item), OPAQUE_TOKEN_COMMENT, indent, options);
+    }
+}
+function renderAttachmentGroupRaw(lines, raw, version, indent, options) {
+    const domain = asDomain(raw);
+    if (domain !== "txt" && domain !== "bny") {
+        return 0;
+    }
+    const parsed = parseAttachmentDispatchCompat(raw, version, domain);
+    const counter = parseCounterCompat(raw, version, domain);
+    emitLine(lines, counter.qb64, `${parsed.group.name} count=${counter.count}`, indent, options);
+    const headerSize = domain === "bny" ? counter.fullSizeB2 : counter.fullSize;
+    const payload = raw.slice(headerSize, parsed.consumed);
+    if (WRAPPER_GROUP_NAMES.has(parsed.group.name)) {
+        // Wrapper groups contain nested groups by convention, but real-world
+        // streams may include trailing opaque segments. Keep annotation resilient.
+        let offset = 0;
+        while (offset < payload.length) {
+            const slice = payload.slice(offset);
+            const sliceCold = asDomain(slice);
+            if (sliceCold === "msg") {
+                break;
+            }
+            if (sliceCold === "txt" && slice[0] !== "-".charCodeAt(0)) {
+                break;
+            }
+            let consumed = 0;
+            try {
+                consumed = renderAttachmentGroupRaw(lines, slice, version, indent + options.indent, options);
+            }
+            catch (error) {
+                if (!isRecoverableParseError(error)) {
+                    throw error;
+                }
+                break;
+            }
+            if (consumed <= 0)
+                break;
+            offset += consumed;
+        }
+        if (offset < payload.length) {
+            const remainder = payload.slice(offset);
+            const unit = domain === "bny" ? 3 : 4;
+            for (let i = 0; i < remainder.length; i += unit) {
+                const chunk = remainder.slice(i, Math.min(i + unit, remainder.length));
+                const rendered = domain === "bny"
+                    ? `0x${toHex(chunk)}`
+                    : TEXT_DECODER.decode(chunk);
+                emitLine(lines, rendered, OPAQUE_WRAPPER_PAYLOAD_COMMENT, indent + options.indent, options);
+            }
+        }
+        return parsed.consumed;
+    }
+    renderGroupItems(lines, parsed.group.items, version, indent + options.indent, options);
+    return parsed.consumed;
+}
+function renderNativeBody(lines, frame, options, version) {
+    const raw = frame.body.raw;
+    const domain = asDomain(raw);
+    if (domain !== "txt" && domain !== "bny") {
+        return;
+    }
+    const counter = parseCounterCompat(raw, version, domain);
+    emitLine(lines, counter.qb64, `${counterCodeNameForVersion(counter.code, version)} count=${counter.count}`, 0, options);
+    const unit = domain === "bny" ? 3 : 4;
+    const headerSize = domain === "bny" ? counter.fullSizeB2 : counter.fullSize;
+    const payloadSize = counter.count * unit;
+    const end = Math.min(raw.length, headerSize + payloadSize);
+    let offset = headerSize;
+    while (offset < end) {
+        const slice = raw.slice(offset, end);
+        try {
+            const nestedCounter = parseCounterCompat(slice, version, domain);
+            const nestedCounterSize = domain === "bny"
+                ? nestedCounter.fullSizeB2
+                : nestedCounter.fullSize;
+            emitLine(lines, nestedCounter.qb64, `${counterCodeNameForVersion(nestedCounter.code, version)} counter`, options.indent, options);
+            offset += nestedCounterSize;
+            continue;
+        }
+        catch (error) {
+            if (!isRecoverableParseError(error)) {
+                throw error;
+            }
+        }
+        const matter = parseMatter(slice, domain);
+        const matterSize = domain === "bny" ? matter.fullSizeB2 : matter.fullSize;
+        const field = frame.body.native?.fields.find((candidate) => candidate.primitive.qb64 === matter.qb64);
+        const label = field ? nativeLabelName(field.label) : null;
+        const primitiveComment = matter instanceof Indexer
+            ? `Indexer ${matter.code}`
+            : matterCodeName(matter.code);
+        const comment = label ? `${label} (${primitiveComment})` : primitiveComment;
+        emitLine(lines, matter.qb64, comment, options.indent, options);
+        offset += matterSize;
+    }
+}
+function renderMessageBody(lines, frame, options) {
+    const rawBody = TEXT_DECODER.decode(frame.body.raw);
+    const isOpaqueCesrBody = frame.body.kind === "CESR"
+        && frame.body.ked === null;
+    if (isOpaqueCesrBody) {
+        emitLine(lines, rawBody, `OPAQUE CESR body (non-serder fallback, hex=${toHex(frame.body.raw)})`, 0, options);
+        return;
+    }
+    let body = rawBody;
+    if (options.pretty && frame.body.kind === "JSON") {
+        try {
+            body = JSON.stringify(JSON.parse(rawBody), null, 2);
+        }
+        catch (error) {
+            if (!(error instanceof SyntaxError)) {
+                throw error;
+            }
+            body = rawBody;
+        }
+    }
+    const info = [
+        `SERDER`,
+        frame.body.proto,
+        frame.body.kind,
+        frame.body.ilk ? `ilk=${frame.body.ilk}` : null,
+        frame.body.said ? `said=${frame.body.said}` : null,
+    ].filter(Boolean).join(" ");
+    emitLine(lines, body, info, 0, options);
+}
+function renderFrame(frame, index, options) {
+    const lines = [];
+    const version = frame.body.gvrsn ?? frame.body.pvrsn;
+    const domain = asDomain(frame.body.raw);
+    if (domain === "txt" || domain === "bny") {
+        renderNativeBody(lines, frame, options, version);
+    }
+    else {
+        renderMessageBody(lines, frame, options);
+    }
+    for (const group of frame.attachments) {
+        renderAttachmentGroupRaw(lines, group.raw, version, 0, options);
+    }
+    return { index, frame, lines };
+}
+function renderFrameChunk(lines, input, indent, options) {
+    const parsed = parseBytes(input);
+    const frames = [];
+    for (const event of parsed) {
+        if (event.type === "error") {
+            throw event.error;
+        }
+        frames.push(event.frame);
+    }
+    const rendered = renderAnnotatedFrames(frames, options);
+    for (const frame of rendered) {
+        for (const line of frame.lines) {
+            lines.push(`${spaces(indent)}${line}`);
+        }
+    }
+}
+function renderWrapperAwareStream(lines, input, inheritedVersion, indent, options) {
+    let offset = 0;
+    let activeVersion = inheritedVersion;
+    let usedWrapper = false;
+    while (offset < input.length) {
+        const slice = input.slice(offset);
+        const domain = asDomain(slice);
+        if (domain !== "txt" && domain !== "bny") {
+            if (!usedWrapper)
+                return false;
+            renderFrameChunk(lines, slice, indent, options);
+            return true;
+        }
+        const counter = parseCounterCompat(slice, activeVersion, domain);
+        const headerSize = domain === "bny" ? counter.fullSizeB2 : counter.fullSize;
+        const name = counterCodeName(counter.code);
+        if (name === "KERIACDCGenusVersion") {
+            emitLine(lines, counter.qb64, `${name} count=${counter.count}`, indent, options);
+            activeVersion = decodeVersionCounter(counter);
+            offset += headerSize;
+            usedWrapper = true;
+            continue;
+        }
+        if (!TOP_LEVEL_WRAPPER_GROUP_NAMES.has(name)) {
+            if (!usedWrapper)
+                return false;
+            renderFrameChunk(lines, slice, indent, options);
+            return true;
+        }
+        const unit = domain === "bny" ? 3 : 4;
+        const payloadSize = counter.count * unit;
+        const total = headerSize + payloadSize;
+        if (slice.length < total) {
+            throw new ShortageError(total, slice.length);
+        }
+        emitLine(lines, counter.qb64, `${name} count=${counter.count}`, indent, options);
+        const payload = slice.slice(headerSize, total);
+        const nestedHandled = renderWrapperAwareStream(lines, payload, activeVersion, indent + options.indent, options);
+        if (!nestedHandled) {
+            renderFrameChunk(lines, payload, indent + options.indent, options);
+        }
+        offset += total;
+        usedWrapper = true;
+    }
+    return usedWrapper;
+}
+/**
+ * Render stream-level wrapper groups (GenericGroup/BodyWithAttachmentGroup)
+ * so denot round-trips preserve wrapper counters. Returns null when input does
+ * not start in a wrapper-oriented domain and caller should use frame rendering.
+ */
+export function renderWrapperAnnotatedStream(input, options) {
+    const lines = [];
+    const rendered = renderWrapperAwareStream(lines, input, { major: 2, minor: 0 }, 0, options);
+    if (!rendered) {
+        return null;
+    }
+    return lines.join("\n");
+}
+/** Render parsed CESR frames into line-oriented, human-annotated text blocks. */
+export function renderAnnotatedFrames(frames, options) {
+    return frames.map((frame, index) => renderFrame(frame, index + 1, options));
+}

package/esm/cesr/src/bench/parser-benchmark.js

@@ -0,0 +1,134 @@
+import { createParser } from "../core/parser-engine.js";
+function asPositiveInteger(value, fallback) {
+    if (typeof value !== "number" || !Number.isFinite(value)) {
+        return fallback;
+    }
+    return Math.max(0, Math.trunc(value));
+}
+/**
+ * Normalize chunking to a legal parser-feed size.
+ *
+ * Boundary rules:
+ * - `<= 0` means full-stream feed for each iteration.
+ * - values larger than stream size collapse to full-stream feed.
+ */
+function normalizeChunkSize(chunkSize, totalBytes) {
+    const normalized = asPositiveInteger(chunkSize, 0);
+    if (normalized <= 0 || normalized >= totalBytes) {
+        return totalBytes;
+    }
+    return normalized;
+}
+/**
+ * Produce deterministic feed slices for one parse run.
+ *
+ * Invariant:
+ * - slices preserve original stream ordering.
+ * - slices are view-backed (`subarray`) to avoid benchmark-side copy noise.
+ */
+function buildChunks(input, chunkSize) {
+    if (chunkSize >= input.length) {
+        return [input];
+    }
+    const chunks = [];
+    for (let offset = 0; offset < input.length; offset += chunkSize) {
+        chunks.push(input.subarray(offset, Math.min(offset + chunkSize, input.length)));
+    }
+    return chunks;
+}
+/**
+ * Count parser frame/error events for one parser emission batch.
+ */
+function countFrames(events) {
+    let frameCount = 0;
+    let errorCount = 0;
+    for (const event of events) {
+        if (event.type === "frame") {
+            frameCount++;
+            continue;
+        }
+        errorCount++;
+    }
+    return { frameCount, errorCount };
+}
+/**
+ * Execute one complete parser pass over the provided stream.
+ *
+ * Boundary contract:
+ * - Always feeds configured chunks and then flushes once.
+ * - Returns event counts only; callers own timing and aggregation.
+ */
+export function parseCesrStreamOnce(input, options = {}) {
+    const chunkSize = normalizeChunkSize(options.chunkSize, input.length);
+    const parser = createParser(options.parserOptions);
+    let frameCount = 0;
+    let errorCount = 0;
+    for (const chunk of buildChunks(input, chunkSize)) {
+        const summary = countFrames(parser.feed(chunk));
+        frameCount += summary.frameCount;
+        errorCount += summary.errorCount;
+    }
+    const flushSummary = countFrames(parser.flush());
+    frameCount += flushSummary.frameCount;
+    errorCount += flushSummary.errorCount;
+    return { frameCount, errorCount };
+}
+/**
+ * Benchmark parser throughput across warmup + measured iterations.
+ *
+ * Invariants:
+ * - warmup runs are excluded from final metrics.
+ * - when `failOnParseError` is true, any parse error fails fast to prevent
+ *   silently benchmarking degraded correctness.
+ */
+export function benchmarkCesrParser(input, options = {}) {
+    if (input.length === 0) {
+        throw new Error("Benchmark input stream must not be empty");
+    }
+    const iterations = asPositiveInteger(options.iterations, 50);
+    if (iterations <= 0) {
+        throw new Error("iterations must be greater than 0");
+    }
+    const warmupIterations = asPositiveInteger(options.warmupIterations, 5);
+    const chunkSize = normalizeChunkSize(options.chunkSize, input.length);
+    const failOnParseError = options.failOnParseError ?? true;
+    for (let i = 0; i < warmupIterations; i++) {
+        parseCesrStreamOnce(input, {
+            chunkSize,
+            parserOptions: options.parserOptions,
+        });
+    }
+    let totalFrames = 0;
+    let totalErrors = 0;
+    const startMs = performance.now();
+    for (let i = 0; i < iterations; i++) {
+        const run = parseCesrStreamOnce(input, {
+            chunkSize,
+            parserOptions: options.parserOptions,
+        });
+        totalFrames += run.frameCount;
+        totalErrors += run.errorCount;
+        if (failOnParseError && run.errorCount > 0) {
+            throw new Error(`Benchmark run produced parse errors (run=${i + 1}, errorCount=${run.errorCount})`);
+        }
+    }
+    const elapsedMs = Math.max(performance.now() - startMs, Number.EPSILON);
+    const totalBytes = input.length * iterations;
+    const throughputBytesPerSec = (totalBytes * 1000) / elapsedMs;
+    const throughputMiBPerSec = throughputBytesPerSec / (1024 * 1024);
+    const framesPerSec = (totalFrames * 1000) / elapsedMs;
+    return {
+        iterations,
+        warmupIterations,
+        chunkSize,
+        bytesPerIteration: input.length,
+        totalBytes,
+        totalFrames,
+        totalErrors,
+        elapsedMs,
+        avgIterationMs: elapsedMs / iterations,
+        throughputBytesPerSec,
+        throughputMiBPerSec,
+        framesPerSec,
+    };
+}