@fraczak/k 8.0.0 → 8.1.0

package/codecs/README.md

@@ -105,9 +105,7 @@ as `k-parse.mjs`, keep that name. Every installed codec binary supports
 | `k-print` | `k-print.mjs` | Read a binary pattern+value stream and write the decoded JSON-like value. Use `--debug` to include the decoded pattern. |
 | `k-show` | `show.mjs` | Pass the wire stream through unchanged on stdout while printing the decoded value and filter to stderr. |
 | `k-json` | `json.mjs` | Convert JSON to/from the binary stream with `--parse` or `--print`. |
-| `k-djson` | `djson.mjs` | Convert JSON to/from a universal dynamic JSON singleton pattern with `--parse` or `--print`. |
-| `k-djson-btrie` | `djson-btrie.mjs` | Binary bit-trie dynamic JSON. |
-| `k-djson-ptrie` | `djson-ptrie.mjs` | Radix trie dynamic JSON. |
+| `k-djson` | `djson.mjs` | Dynamic binary bit-trie JSON codec. |
 | `k-int` | `int.mjs` | Convert decimal integers to/from the binary stream with `--parse` or `--print`. |
 | `k-ieee` | `ieee.mjs` | Convert float literals to/from the binary stream with `--parse` or `--print`. |
 | `k-unit` | `unit.mjs` | Write or validate the unit value `{}` with `--parse` or `--print`. |

package/codecs/djson.mjs

@@ -1,19 +1,24 @@
 #!/usr/bin/env node
 
 /**
- * Dynamic JSON (djson) codec with universal singleton pattern.
+ * Dynamic JSON (djson) binary bit-trie codec.
  *
- * Translates arbitrary JSON data to/from self-describing enveloped k
- * values using a single recursive AST pattern:
+ * Uses a unified binary bit-trie for both JSON arrays (list) and JSON
+ * objects (object):
  *
- *   JSON = <
- *     float64 number,
- *     str string,
- *     <{} true, {} false> bool,
- *     {} null,
- *     <{} nil, {JSON car, LIST cdr} cons> = LIST list,
- *     <{} nil, {{str key, JSON val} car, OBJ cdr} cons> = OBJ object
+ *   TRIE = <
+ *     {} nil,
+ *     {
+ *       < {} nil, JSON val > val,
+ *       TRIE 0,
+ *       TRIE 1
+ *     } trie
  *   >
+ *
+ * - In `list`, element at index i >= 0 is stored at the canonical binary
+ *   bit-path of i (0 -> [0], 1 -> [1], 2 -> [1,0], 3 -> [1,1], ...).
+ * - In `object`, entry with key k is stored at the prefix-free Unicode
+ *   character bit-path of k.
  */
 
 import {
@@ -26,22 +31,24 @@ import {
 } from "./runtime/codec-sdk.mjs";
 import { encodeNumberToValue, decodeValueToNumber } from "./ieee.mjs";
 import {
+  codePointToUnicodeValue,
+  unicodeValueToCodePoint,
   textToStringValue,
   stringValueToText
 } from "./runtime/unicode-string.mjs";
 
 export const doc = `
-Dynamic JSON (djson) codec with universal singleton pattern.
+Dynamic JSON (djson) binary bit-trie codec.
 
-Translates any JSON value to/from self-describing enveloped k values
-conforming to the universal recursive JSON AST type:
-  JSON = <
-    float64 number,
-    str string,
-    <{} true, {} false> bool,
-    {} null,
-    <{} nil, {JSON car, LIST cdr} cons> = LIST list,
-    <{} nil, {{str key, JSON val} car, OBJ cdr} cons> = OBJ object
+Represents both JSON arrays (list) and JSON objects (object) using a
+unified binary bit-trie:
+  TRIE = <
+    {} nil,
+    {
+      < {} nil, JSON val > val,
+      TRIE 0,
+      TRIE 1
+    } trie
   >
 `;
 
@@ -122,13 +129,21 @@ export const DJSON_PATTERN = patternFromFilter(`
   < {} true, {} false > bool,
   {} null,
 
-  < {} nil, { JSON car, LIST cdr } cons > = LIST list,
+  <
+    {} nil,
+    {
+      < {} nil, JSON val > val,
+      TRIE 0,
+      TRIE 1
+    } trie
+  > = TRIE list,
 
-  < {} nil, { { Str key, JSON val } car, OBJ cdr } cons > = OBJ object
+  TRIE object
 > = JSON
 `);
 
 const UNIT = Value.product({});
+const NIL = Value.variant("nil", UNIT);
 
 function requireProduct(value, where) {
   if (!isProduct(value)) {
@@ -137,37 +152,298 @@ function requireProduct(value, where) {
   return value.product;
 }
 
-function listToK(items) {
-  let result = Value.variant("nil", UNIT);
-  for (let i = items.length - 1; i >= 0; i--) {
-    result = Value.variant("cons", Value.product({
-      car: items[i],
-      cdr: result
-    }));
-  }
-  return result;
+function pushBitsMSB(out, width, n) {
+  for (let i = width - 1; i >= 0; i--) {
+    out.push((n >> i) & 1);
+  }
+}
+
+function readBitsMSB(bits, offset, width) {
+  let n = 0;
+  for (let i = 0; i < width; i++) {
+    n = (n << 1) | bits[offset + i];
+  }
+  return n;
+}
+
+export function indexToBitPath(index) {
+  if (index === 0) return [0];
+  const bits = [];
+  let n = index;
+  while (n > 0) {
+    bits.push(n & 1);
+    n = Math.floor(n / 2);
+  }
+  bits.reverse();
+  return bits;
+}
+
+export function bitPathToIndex(bits) {
+  if (bits.length === 1 && bits[0] === 0) return 0;
+  let n = 0;
+  for (const b of bits) {
+    n = n * 2 + b;
+  }
+  return n;
+}
+
+export function codePointToBitPath(cp, out = []) {
+  if (cp <= 0x7f) {
+    out.push(0);
+    pushBitsMSB(out, 7, cp);
+    return out;
+  }
+  if (cp <= 0x7ff) {
+    out.push(1, 0);
+    pushBitsMSB(out, 11, cp);
+    return out;
+  }
+  if (cp <= 0xffff) {
+    const hi = (cp >> 8) & 0xff;
+    const lo = cp & 0xff;
+    if (hi >= 0xe0 && hi <= 0xf8) {
+      out.push(1, 1, 1, 0);
+      if (hi <= 0xef) {
+        out.push(0);
+        pushBitsMSB(out, 4, hi - 0xe0);
+      } else if (hi <= 0xf7) {
+        out.push(1, 0);
+        pushBitsMSB(out, 3, hi - 0xf0);
+      } else {
+        out.push(1, 1);
+      }
+      pushBitsMSB(out, 8, lo);
+      return out;
+    }
+    out.push(1, 1, 0);
+    if (hi >= 0x08 && hi <= 0x0f) {
+      out.push(0, 0, 0);
+      pushBitsMSB(out, 3, hi - 0x08);
+    } else if (hi >= 0x10 && hi <= 0x7f) {
+      out.push(0, 0, 1);
+      pushBitsMSB(out, 7, hi - 0x10);
+    } else if (hi >= 0x80 && hi <= 0xcf) {
+      out.push(0, 1, 0);
+      pushBitsMSB(out, 7, hi - 0x80);
+    } else if (hi >= 0xd0 && hi <= 0xd7) {
+      out.push(0, 1, 1);
+      pushBitsMSB(out, 3, hi - 0xd0);
+    } else if (hi === 0xf9) {
+      out.push(1, 0, 0);
+    } else if (hi >= 0xfa && hi <= 0xfb) {
+      out.push(1, 0, 1);
+      pushBitsMSB(out, 1, hi - 0xfa);
+    } else if (hi >= 0xfc && hi <= 0xfd) {
+      out.push(1, 1, 0);
+      pushBitsMSB(out, 1, hi - 0xfc);
+    } else if (hi >= 0xfe && hi <= 0xff) {
+      out.push(1, 1, 1);
+      pushBitsMSB(out, 1, hi - 0xfe);
+    } else {
+      throw new Error(`Unsupported BMP hi byte: ${hi}`);
+    }
+    pushBitsMSB(out, 8, lo);
+    return out;
+  }
+  const mid = (cp >> 8) & 0xff;
+  const lo = cp & 0xff;
+  if (cp <= 0x1ffff) {
+    out.push(1, 1, 1, 1, 0);
+    pushBitsMSB(out, 8, mid);
+    pushBitsMSB(out, 8, lo);
+    return out;
+  }
+  const plane = cp >> 16;
+  out.push(1, 1, 1, 1, 1);
+  if (plane >= 2 && plane <= 3) {
+    out.push(0, 0);
+    pushBitsMSB(out, 1, plane - 2);
+  } else if (plane >= 4 && plane <= 7) {
+    out.push(0, 1);
+    pushBitsMSB(out, 2, plane - 4);
+  } else if (plane >= 8 && plane <= 15) {
+    out.push(1, 0);
+    pushBitsMSB(out, 3, plane - 8);
+  } else if (plane === 16) {
+    out.push(1, 1);
+  } else {
+    throw new Error(`Unsupported plane: ${plane}`);
+  }
+  pushBitsMSB(out, 8, mid);
+  pushBitsMSB(out, 8, lo);
+  return out;
 }
 
-function listFromK(value, where) {
-  const result = [];
-  let current = value;
-  while (true) {
-    if (!isVariant(current)) {
-      throw new Error(`${where}: expected list variant (nil or cons)`);
+export function stringToBitPath(str) {
+  const out = [];
+  for (const ch of str) {
+    codePointToBitPath(ch.codePointAt(0), out);
+  }
+  return out;
+}
+
+export function bitPathToString(bits) {
+  let i = 0;
+  let out = "";
+  while (i < bits.length) {
+    if (bits[i] === 0) {
+      const cp = readBitsMSB(bits, i + 1, 7);
+      out += String.fromCodePoint(cp);
+      i += 8;
+      continue;
     }
-    if (current.tag === "nil") {
-      requireProduct(current.value, `${where}.nil`);
-      return result;
+    if (bits[i + 1] === 0) {
+      const cp = readBitsMSB(bits, i + 2, 11);
+      out += String.fromCodePoint(cp);
+      i += 13;
+      continue;
     }
-    if (current.tag !== "cons") {
-      throw new Error(
-        `${where}: expected nil or cons, got ${current.tag}`
-      );
+    if (bits[i + 2] === 0) {
+      i += 3;
+      const tag = readBitsMSB(bits, i, 3);
+      i += 3;
+      let hi;
+      if (tag === 0) {
+        hi = 0x08 + readBitsMSB(bits, i, 3);
+        i += 3;
+      } else if (tag === 1) {
+        hi = 0x10 + readBitsMSB(bits, i, 7);
+        i += 7;
+      } else if (tag === 2) {
+        hi = 0x80 + readBitsMSB(bits, i, 7);
+        i += 7;
+      } else if (tag === 3) {
+        hi = 0xd0 + readBitsMSB(bits, i, 3);
+        i += 3;
+      } else if (tag === 4) {
+        hi = 0xf9;
+      } else if (tag === 5) {
+        hi = 0xfa + readBitsMSB(bits, i, 1);
+        i += 1;
+      } else if (tag === 6) {
+        hi = 0xfc + readBitsMSB(bits, i, 1);
+        i += 1;
+      } else {
+        hi = 0xfe + readBitsMSB(bits, i, 1);
+        i += 1;
+      }
+      const lo = readBitsMSB(bits, i, 8);
+      i += 8;
+      out += String.fromCodePoint((hi << 8) | lo);
+      continue;
+    }
+    if (bits[i + 3] === 0) {
+      i += 4;
+      let hi;
+      if (bits[i] === 0) {
+        hi = 0xe0 + readBitsMSB(bits, i + 1, 4);
+        i += 5;
+      } else if (bits[i + 1] === 0) {
+        hi = 0xf0 + readBitsMSB(bits, i + 2, 3);
+        i += 5;
+      } else {
+        hi = 0xf8;
+        i += 2;
+      }
+      const lo = readBitsMSB(bits, i, 8);
+      i += 8;
+      out += String.fromCodePoint((hi << 8) | lo);
+      continue;
+    }
+    if (bits[i + 4] === 0) {
+      i += 5;
+      const mid = readBitsMSB(bits, i, 8);
+      const lo = readBitsMSB(bits, i + 8, 8);
+      i += 16;
+      out += String.fromCodePoint((1 << 16) | (mid << 8) | lo);
+      continue;
+    }
+    i += 5;
+    let plane;
+    if (bits[i] === 0 && bits[i + 1] === 0) {
+      plane = 2 + readBitsMSB(bits, i + 2, 1);
+      i += 3;
+    } else if (bits[i] === 0 && bits[i + 1] === 1) {
+      plane = 4 + readBitsMSB(bits, i + 2, 2);
+      i += 4;
+    } else if (bits[i] === 1 && bits[i + 1] === 0) {
+      plane = 8 + readBitsMSB(bits, i + 2, 3);
+      i += 5;
+    } else {
+      plane = 16;
+      i += 2;
+    }
+    const mid = readBitsMSB(bits, i, 8);
+    const lo = readBitsMSB(bits, i + 8, 8);
+    i += 16;
+    out += String.fromCodePoint((plane << 16) | (mid << 8) | lo);
+  }
+  return out;
+}
+
+function makeMutableNode() {
+  return { val: null, b0: null, b1: null };
+}
+
+function insertMutableTrie(root, bits, kVal) {
+  let curr = root;
+  for (let i = 0; i < bits.length; i++) {
+    if (bits[i] === 0) {
+      if (!curr.b0) curr.b0 = makeMutableNode();
+      curr = curr.b0;
+    } else {
+      if (!curr.b1) curr.b1 = makeMutableNode();
+      curr = curr.b1;
     }
-    const pair = requireProduct(current.value, `${where}.cons`);
-    result.push(pair.car);
-    current = pair.cdr;
   }
+  curr.val = kVal;
+}
+
+function mutableNodeToK(node) {
+  if (!node) return NIL;
+  const valField = node.val !== null
+    ? Value.variant("val", node.val)
+    : NIL;
+  const b0 = mutableNodeToK(node.b0);
+  const b1 = mutableNodeToK(node.b1);
+  if (node.val === null && b0 === NIL && b1 === NIL) {
+    return NIL;
+  }
+  return Value.variant("trie", Value.product({
+    val: valField,
+    0: b0,
+    1: b1
+  }));
+}
+
+function collectTrieEntries(kNode, pathBits = [], entries = []) {
+  if (!isVariant(kNode)) {
+    throw new Error("djson: expected variant (nil or trie)");
+  }
+  if (kNode.tag === "nil") {
+    return entries;
+  }
+  if (kNode.tag !== "trie") {
+    throw new Error(`djson: unexpected tag ${kNode.tag}`);
+  }
+  const prod = requireProduct(kNode.value, "djson.trie");
+  if (!isVariant(prod.val)) {
+    throw new Error("djson.trie.val: expected variant");
+  }
+  if (prod.val.tag === "val") {
+    entries.push({ bits: pathBits.slice(), val: prod.val.value });
+  } else if (prod.val.tag !== "nil") {
+    throw new Error(
+      `djson.trie.val: unexpected tag ${prod.val.tag}`
+    );
+  }
+  pathBits.push(0);
+  collectTrieEntries(prod[0], pathBits, entries);
+  pathBits.pop();
+  pathBits.push(1);
+  collectTrieEntries(prod[1], pathBits, entries);
+  pathBits.pop();
+  return entries;
 }
 
 export function fromJsonValue(value) {
@@ -190,81 +466,96 @@ export function fromJsonValue(value) {
     return Value.variant("string", textToStringValue(value));
   }
   if (Array.isArray(value)) {
-    return Value.variant("list", listToK(value.map(fromJsonValue)));
+    if (value.length === 0) {
+      return Value.variant("list", NIL);
+    }
+    const root = makeMutableNode();
+    for (let i = 0; i < value.length; i++) {
+      insertMutableTrie(root, indexToBitPath(i), fromJsonValue(value[i]));
+    }
+    return Value.variant("list", mutableNodeToK(root));
   }
   if (value && typeof value === "object") {
-    const keys = Object.keys(value).sort();
-    const entries = keys.map((k) =>
-      Value.product({
-        key: textToStringValue(k),
-        val: fromJsonValue(value[k])
-      })
-    );
-    return Value.variant("object", listToK(entries));
+    const keys = Object.keys(value);
+    if (keys.length === 0) {
+      return Value.variant("object", NIL);
+    }
+    const root = makeMutableNode();
+    for (const k of keys) {
+      insertMutableTrie(
+        root,
+        stringToBitPath(k),
+        fromJsonValue(value[k])
+      );
+    }
+    return Value.variant("object", mutableNodeToK(root));
   }
   throw new Error(`Unsupported JSON value: ${value}`);
 }
 
 export function toJsonValue(value) {
   if (!isVariant(value)) {
-    throw new Error("djson.value: expected top-level Variant");
+    throw new Error("djson: expected top-level Variant");
   }
-
   if (value.tag === "null") {
     requireProduct(value.value, "djson.null");
     return null;
   }
-
   if (value.tag === "bool") {
     if (!isVariant(value.value)) {
       throw new Error("djson.bool: expected variant");
     }
-    if (value.value.tag === "true") {
-      requireProduct(value.value.value, "djson.bool.true");
-      return true;
-    }
-    if (value.value.tag === "false") {
-      requireProduct(value.value.value, "djson.bool.false");
-      return false;
-    }
+    if (value.value.tag === "true") return true;
+    if (value.value.tag === "false") return false;
     throw new Error(`djson.bool: invalid tag ${value.value.tag}`);
   }
-
   if (value.tag === "number") {
     return decodeValueToNumber(value.value);
   }
-
   if (value.tag === "string") {
     return stringValueToText(value.value);
   }
-
   if (value.tag === "list") {
-    const items = listFromK(value.value, "djson.list");
-    return items.map(toJsonValue);
+    const entries = collectTrieEntries(value.value);
+    entries.sort((a, b) =>
+      bitPathToIndex(a.bits) - bitPathToIndex(b.bits)
+    );
+    return entries.map((e) => toJsonValue(e.val));
   }
-
   if (value.tag === "object") {
-    const pairs = listFromK(value.value, "djson.object");
+    const entries = collectTrieEntries(value.value);
     const result = {};
-    for (const pairValue of pairs) {
-      const pair = requireProduct(pairValue, "djson.object.pair");
-      const key = stringValueToText(pair.key);
-      result[key] = toJsonValue(pair.val);
+    for (const e of entries) {
+      result[bitPathToString(e.bits)] = toJsonValue(e.val);
     }
     return result;
   }
-
-  throw new Error(`Unsupported djson tag: ${value.tag}`);
+  throw new Error(`djson: unsupported tag ${value.tag}`);
 }
 
-export function parse(text) {
-  const json = JSON.parse(text);
-  const value = fromJsonValue(json);
-  return withPattern(value, DJSON_PATTERN);
+export const DJSON_BTRIE_PATTERN = DJSON_PATTERN;
+export const pattern = DJSON_PATTERN;
+
+export function parse(rawText) {
+  const text = String(rawText).trim();
+  if (!text) throw new Error("Empty JSON input");
+  const parsed = JSON.parse(text);
+  const val = fromJsonValue(parsed);
+  return withPattern(val, DJSON_PATTERN);
 }
 
 export function print(value) {
   return JSON.stringify(toJsonValue(value));
 }
 
+export const codec = {
+  name: "djson",
+  doc,
+  pattern: DJSON_PATTERN,
+  parse,
+  print
+};
+
+export default codec;
+
 runCodecCLI(import.meta.url, { parse, print, doc });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@fraczak/k",
-    "version": "8.0.0",
+    "version": "8.1.0",
     "description": "k — a language for content-addressable schemas and data transformations over algebraic data types (products and tagged unions); includes runtime, contract derivation, REPL, and codec toolchain",
     "publishConfig": {
         "access": "public"
@@ -33,7 +33,6 @@
         "perf:poly:pkir": "BACKENDS=pkir node scripts/perf-poly.mjs",
         "perf:poly:llvm-pkir": "BACKENDS=llvm-pkir node scripts/perf-poly.mjs",
         "perf:poly:trace": "node scripts/perf-poly.mjs --trace",
-        "perf:djson": "node scripts/perf-djson.mjs",
         "perf:idempotency": "node scripts/benchmark-idempotency.mjs",
         "deps:graph": "node scripts/generate-dependency-graph.mjs",
         "compare": "node harness/cli.mjs",
@@ -60,8 +59,6 @@
         "k-wasm-compile": "backends/wasm/bin/k-wasm-compile.mjs",
         "k-wasm-run": "backends/wasm/bin/k-wasm-run.mjs",
         "k-djson": "codecs/djson.mjs",
-        "k-djson-btrie": "codecs/djson-btrie.mjs",
-        "k-djson-ptrie": "codecs/djson-ptrie.mjs",
         "k-ieee": "codecs/ieee.mjs",
         "k-int": "codecs/int.mjs",
         "k-json": "codecs/json.mjs",

package/scripts/build-repl-html.mjs

@@ -84,9 +84,7 @@ const verifiedExamples = [
   "ieee.k",
   "poly.k",
   "karatsuba-mult.k",
-  "djson.k",
-  "djson-btrie.k",
-  "djson-ptrie.k"
+  "djson.k"
 ];
 const examplesDir = path.join(projectRoot, "Examples");
 for (const filename of verifiedExamples) {
@@ -102,8 +100,6 @@ for (const filename of verifiedExamples) {
 const verifiedCodecs = [
   "json.mjs",
   "djson.mjs",
-  "djson-btrie.mjs",
-  "djson-ptrie.mjs",
   "utf8.mjs",
   "int.mjs",
   "unit.mjs",
@@ -1698,8 +1694,6 @@ const htmlContent = `<!DOCTYPE html>
               <option value="poly.k">poly.k</option>
               <option value="karatsuba-mult.k">karatsuba-mult.k</option>
               <option value="djson.k">djson.k</option>
-              <option value="djson-btrie.k">djson-btrie.k</option>
-              <option value="djson-ptrie.k">djson-ptrie.k</option>
             </optgroup>
             <optgroup label="Quick Expressions">
               <option value="expr:{} | ok">{} | ok</option>
@@ -1720,14 +1714,10 @@ const htmlContent = `<!DOCTYPE html>
               <option value="expr::codec load utf8.mjs">:codec load utf8.mjs</option>
               <option value="expr::codec load json.mjs">:codec load json.mjs</option>
               <option value="expr::codec load djson.mjs">:codec load djson.mjs</option>
-              <option value="expr::codec load djson-btrie.mjs">:codec load djson-btrie.mjs</option>
-              <option value="expr::codec load djson-ptrie.mjs">:codec load djson-ptrie.mjs</option>
               <option value="expr::codec load ieee.mjs">:codec load ieee.mjs</option>
               <option value="expr::codec load unit.mjs">:codec load unit.mjs</option>
               <option value="expr::input json.mjs {&quot;a&quot;:12,&quot;b&quot;:&quot;hello&quot;}">:input json.mjs {"a":12,"b":"hello"}</option>
               <option value="expr::input djson.mjs {&quot;a&quot;:12,&quot;b&quot;:&quot;hello&quot;}">:input djson.mjs {"a":12,"b":"hello"}</option>
-              <option value="expr::input djson-btrie.mjs {&quot;a&quot;:12,&quot;b&quot;:&quot;hello&quot;}">:input djson-btrie.mjs {"a":12,"b":"hello"}</option>
-              <option value="expr::input djson-ptrie.mjs {&quot;a&quot;:12,&quot;b&quot;:&quot;hello&quot;}">:input djson-ptrie.mjs {"a":12,"b":"hello"}</option>
               <option value="expr::input int.mjs [0,1,2]">:input int.mjs [0,1,2]</option>
             </optgroup>
           </select>

package/scripts/run-tests.mjs

@@ -43,6 +43,11 @@ const tests = [
   ["tests/test-hash-normalization.mjs", node, ["tests/test-hash-normalization.mjs"]],
   ["tests/test-structural-values.mjs", node, ["tests/test-structural-values.mjs"]],
   ["tests/test-codecs.mjs", node, ["tests/test-codecs.mjs"]],
+  [
+    "tests/test-djson-methods.mjs",
+    node,
+    ["tests/test-djson-methods.mjs"]
+  ],
   ["tests/test-deep-wire-value.mjs", node, ["tests/test-deep-wire-value.mjs"]],
   ["tests/test-k-object.mjs", node, ["tests/test-k-object.mjs"]],
   ["tests/test-kir.mjs", node, ["tests/test-kir.mjs"]],