@fraczak/k 6.3.2 → 6.4.0

package/codecs/README.md

@@ -79,6 +79,9 @@ notation for k values. That notation is documented in
 [`../DOCS/TEXTUAL_VALUES.md`](../DOCS/TEXTUAL_VALUES.md). It is a readable
 notation for k values, not a separate JSON container format.
 
+For the contributor-facing guide to writing a new external codec module, see
+[`../DOCS/CODECS.md`](../DOCS/CODECS.md).
+
 `k.mjs` is only the command-line adapter for that boundary. The operational
 runtime sees and preserves the pattern on the `Value` itself.
 

package/codecs/json.mjs

@@ -5,6 +5,9 @@ import { decodeWire, encodeToWire } from "./runtime/prefix-codec.mjs";
 import { isMainEntrypoint } from "./runtime/cli-entry.mjs";
 import { fromJsonValue, toJsonValue, patternFromJsonValue } from "./json-codec.mjs";
 
+const name = "json";
+const universal = true;
+
 function usage(stream = console.error) {
   stream(`Usage: ${argv[1]} --parse | --print`);
   stream("  --parse      Read JSON from stdin, write binary pattern+value stream.");
@@ -21,6 +24,14 @@ function readAll(stream) {
   });
 }
 
+function parse(text) {
+  return fromJsonValue(JSON.parse(text));
+}
+
+function print(value) {
+  return JSON.stringify(toJsonValue(value));
+}
+
 async function main() {
   const args = argv.slice(2);
   if (args.includes("-h") || args.includes("--help")) {
@@ -36,12 +47,13 @@ async function main() {
   const buf = await readAll(stdin);
 
   if (args[0] === "--parse") {
-    const json = JSON.parse(buf.toString("utf8"));
+    const text = buf.toString("utf8");
+    const json = JSON.parse(text);
     const value = fromJsonValue(json);
     stdout.write(encodeToWire(value, patternFromJsonValue(json)));
   } else {
     const { value } = decodeWire(buf);
-    stdout.write(`${JSON.stringify(toJsonValue(value))}\n`);
+    stdout.write(`${print(value)}\n`);
   }
 }
 
@@ -51,3 +63,5 @@ if (isMainEntrypoint(import.meta.url, argv[1])) {
     exit(1);
   });
 }
+
+export { name, universal, parse, print };

package/codecs/runtime/show-value.mjs

@@ -6,7 +6,7 @@ function pLabel(label) {
   return nameRE.test(label) ? ` ${label}` : ` ${JSON.stringify(label)}`;
 }
 
-function propertyListToFilter(propertyList) {
+function propertyListToFilter(propertyList, varPrefix = "X") {
   if (!propertyList || propertyList.length === 0) return "(...)";
 
   const refCount = new Array(propertyList.length).fill(0);
@@ -18,7 +18,7 @@ function propertyListToFilter(propertyList) {
   const varNames = new Map();
   let varCounter = 0;
   for (let i = 0; i < propertyList.length; i++) {
-    if (refCount[i] > 1) varNames.set(i, `X${varCounter++}`);
+    if (refCount[i] > 1) varNames.set(i, `${varPrefix}${varCounter++}`);
   }
 
   const defined = new Set();

package/conformance/helper-reuse/case.json

@@ -0,0 +1,8 @@
+{
+  "name": "helper reuse",
+  "program": "program.k",
+  "input": "input.kv",
+  "expected": "expected.kv",
+  "modes": ["source", "object", "kir", "retyped"],
+  "tags": ["runtime", "retyping", "call-sites"]
+}

package/conformance/helper-reuse/expected.kv

@@ -0,0 +1 @@
+{"left":"left","right":"right"}

package/conformance/helper-reuse/input.kv

@@ -0,0 +1 @@
+{"a":{"x":"left","z":{}},"b":{"x":"right","z":{}}}

package/conformance/helper-reuse/program.k

@@ -0,0 +1,2 @@
+pick = .x;
+{.a pick left, .b pick right}

package/conformance/identity/case.json

@@ -0,0 +1,8 @@
+{
+  "name": "identity",
+  "program": "program.k",
+  "input": "input.kv",
+  "expected": "expected.kv",
+  "modes": ["source", "object", "kir", "retyped"],
+  "tags": ["runtime", "identity"]
+}

package/conformance/identity/expected.kv

@@ -0,0 +1 @@
+{}

package/conformance/identity/input.kv

@@ -0,0 +1 @@
+{}

package/conformance/identity/program.k

@@ -0,0 +1 @@
+()

package/conformance/product-projection/case.json

@@ -0,0 +1,8 @@
+{
+  "name": "product projection",
+  "program": "program.k",
+  "input": "input.kv",
+  "expected": "expected.kv",
+  "modes": ["source", "object", "kir", "retyped"],
+  "tags": ["runtime", "projection", "product"]
+}

package/conformance/product-projection/expected.kv

@@ -0,0 +1 @@
+"left"

package/conformance/product-projection/input.kv

@@ -0,0 +1 @@
+{"x":"left","y":"right"}

package/conformance/product-projection/program.k

@@ -0,0 +1 @@
+.x

package/conformance/run.mjs

@@ -0,0 +1,149 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+import path from "node:path";
+import { argv, exit } from "node:process";
+import { fileURLToPath } from "node:url";
+import k from "../index.mjs";
+import { parseValue } from "../valueIO.mjs";
+import { compileObjectBuffer, decodeObject, objectToFunction } from "../object.mjs";
+import { objectToKIRP, retypeObjectRelation } from "../kir.mjs";
+import { validateKIRP } from "../objects/validate.mjs";
+import { exportPatternGraph } from "../codecs/runtime/codec.mjs";
+import { patternToPropertyList } from "../codecs/runtime/pattern-json.mjs";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const conformanceRoot = path.join(root, "conformance");
+const DEFAULT_MODES = ["source", "object", "kir", "retyped"];
+
+function helpText() {
+  return [
+    "Run k conformance fixtures.",
+    "",
+    `Usage: ${argv[1]} [options] [fixture-dir ...]`,
+    "",
+    "Options:",
+    "  --mode list    Comma-separated modes: source, object, kir, retyped.",
+    "  -h, --help     Show this help.",
+    "",
+    "With no fixture dirs, every directory under conformance/ with a case.json is run."
+  ].join("\n");
+}
+
+function usage(stream = console.error) {
+  stream(helpText());
+}
+
+function fixtureDirs(args) {
+  if (args.length > 0) return args.map((arg) => path.resolve(root, arg));
+  return fs.readdirSync(conformanceRoot, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => path.join(conformanceRoot, entry.name))
+    .filter((dir) => fs.existsSync(path.join(dir, "case.json")))
+    .sort();
+}
+
+function readFixture(dir) {
+  const spec = JSON.parse(fs.readFileSync(path.join(dir, "case.json"), "utf8"));
+  return {
+    dir,
+    name: spec.name || path.basename(dir),
+    modes: spec.modes || DEFAULT_MODES,
+    program: fs.readFileSync(path.join(dir, spec.program || "program.k"), "utf8"),
+    input: parseValue(fs.readFileSync(path.join(dir, spec.input || "input.kv"), "utf8")),
+    expected: parseValue(fs.readFileSync(path.join(dir, spec.expected || "expected.kv"), "utf8"))
+  };
+}
+
+function assertSameValue(actual, expected, label) {
+  const actualJson = JSON.stringify(actual.toJSON());
+  const expectedJson = JSON.stringify(expected.toJSON());
+  if (actualJson !== expectedJson) {
+    throw new Error(`${label}: expected ${expectedJson}, got ${actualJson}`);
+  }
+}
+
+function runSource(fixture) {
+  const result = k.compile(fixture.program)(fixture.input);
+  assertSameValue(result, fixture.expected, `${fixture.name}/source`);
+}
+
+function runObject(fixture) {
+  const object = decodeObject(compileObjectBuffer(fixture.program, {
+    source: path.relative(root, fixture.dir)
+  }));
+  const result = objectToFunction(object)(fixture.input);
+  assertSameValue(result, fixture.expected, `${fixture.name}/object`);
+}
+
+function runKIR(fixture) {
+  const object = decodeObject(compileObjectBuffer(fixture.program, {
+    source: path.relative(root, fixture.dir)
+  }));
+  const kir = validateKIRP(objectToKIRP(object));
+  if (kir.kind !== "executable") throw new Error(`${fixture.name}/kir: expected executable KIR`);
+  if (!kir.rels[kir.main]) throw new Error(`${fixture.name}/kir: main relation missing`);
+}
+
+function runRetyped(fixture) {
+  const object = decodeObject(compileObjectBuffer(fixture.program, {
+    source: path.relative(root, fixture.dir)
+  }));
+  const mainRel = object.rels[object.main];
+  const inputPatternId = mainRel.typePatternGraph.find(mainRel.def.patterns[0]);
+  const inputPattern = fixture.input.pattern || patternToPropertyList(exportPatternGraph(mainRel.typePatternGraph, inputPatternId));
+  const kir = validateKIRP(retypeObjectRelation(object, object.main, inputPattern));
+  if (kir.kind !== "executable") throw new Error(`${fixture.name}/retyped: expected executable KIR`);
+  if (!kir.rels[kir.main]) throw new Error(`${fixture.name}/retyped: main relation missing`);
+}
+
+function runMode(mode, fixture) {
+  if (mode === "source") return runSource(fixture);
+  if (mode === "object") return runObject(fixture);
+  if (mode === "kir") return runKIR(fixture);
+  if (mode === "retyped") return runRetyped(fixture);
+  throw new Error(`Unknown conformance mode '${mode}'`);
+}
+
+function parseArgs() {
+  const args = argv.slice(2);
+  if (args.includes("-h") || args.includes("--help")) {
+    usage(console.log);
+    exit(0);
+  }
+
+  let modes = DEFAULT_MODES;
+  const dirs = [];
+  while (args.length > 0) {
+    const arg = args.shift();
+    if (arg === "--mode") {
+      const value = args.shift();
+      if (!value) throw new Error("--mode requires a comma-separated list");
+      modes = value.split(",").map((mode) => mode.trim()).filter(Boolean);
+    } else if (arg.startsWith("--")) {
+      throw new Error(`Unknown option: ${arg}`);
+    } else {
+      dirs.push(arg);
+    }
+  }
+  return { modes, dirs };
+}
+
+try {
+  const { modes, dirs } = parseArgs();
+  const fixtures = fixtureDirs(dirs).map(readFixture);
+  let count = 0;
+  for (const fixture of fixtures) {
+    const fixtureModes = modes.filter((mode) => fixture.modes.includes(mode));
+    for (const mode of fixtureModes) {
+      runMode(mode, fixture);
+      count++;
+      console.log(`ok ${fixture.name}/${mode}`);
+    }
+  }
+  console.log(`OK ${count} checks`);
+} catch (error) {
+  console.error(error.stack || error.message || String(error));
+  usage();
+  exit(1);
+}

package/conformance/variant-construction/case.json

@@ -0,0 +1,8 @@
+{
+  "name": "variant construction",
+  "program": "program.k",
+  "input": "input.kv",
+  "expected": "expected.kv",
+  "modes": ["source", "object", "kir", "retyped"],
+  "tags": ["runtime", "variant"]
+}

package/conformance/variant-construction/expected.kv

@@ -0,0 +1 @@
+{"some":{}}

package/conformance/variant-construction/input.kv

@@ -0,0 +1 @@
+{}

package/conformance/variant-construction/program.k

@@ -0,0 +1 @@
+|some

package/harness/adapters/arm64.mjs

@@ -0,0 +1,105 @@
+import fs from "node:fs";
+import path from "node:path";
+import { BackendAdapter, BackendRunner } from "../backend.mjs";
+import { PersistentExecutable } from "../persistent.mjs";
+import { compileARM64ArtifactFromObject } from "../../backends/arm64/src/arm64.mjs";
+import { decodeWire } from "../../backend-api.mjs";
+
+class ARM64Runner extends BackendRunner {
+  constructor(backendId, servers = new Map()) {
+    super(backendId);
+    this.servers = servers;
+  }
+
+  getServer(exePath) {
+    let server = this.servers.get(exePath);
+    if (!server) {
+      server = new PersistentExecutable(exePath);
+      this.servers.set(exePath, server);
+    }
+    return server;
+  }
+
+  async runSingle(testCase, options = {}) {
+    const server = this.getServer(testCase.exePath);
+    if (options.trace) {
+      const res = await server.requestWithTrace(testCase.inputWire);
+      const decoded = decodeWire(res.payload);
+      return {
+        outputWire: res.payload,
+        outputVal: decoded.value,
+        outputPattern: decoded.pattern,
+        trace: res.trace,
+        profile: res.trace?.profile
+      };
+    } else {
+      const payload = await server.request(testCase.inputWire);
+      const decoded = decodeWire(payload);
+      return {
+        outputWire: payload,
+        outputVal: decoded.value,
+        outputPattern: decoded.pattern
+      };
+    }
+  }
+
+  async runBatch(compiledCases) {
+    for (const c of compiledCases) {
+      const server = this.getServer(c.exePath);
+      await server.request(c.inputWire);
+    }
+  }
+
+  async close() {
+    for (const server of this.servers.values()) {
+      server.close();
+    }
+    this.servers.clear();
+  }
+}
+
+export class ARM64Adapter extends BackendAdapter {
+  constructor(options = {}) {
+    const optLevel = options.optLevel || "-O2";
+    super("arm64", `Linux ARM64 ${optLevel} (persistent)`, options);
+    this.optLevel = optLevel;
+  }
+
+  isAvailable() {
+    return process.arch === "arm64" && process.platform === "linux";
+  }
+
+  async compile(testCase, options = {}) {
+    const cacheDir = options.cacheDir || path.join(process.cwd(), ".cache");
+    fs.mkdirSync(cacheDir, { recursive: true });
+    const exePath = path.join(cacheDir, `harness_arm64_${testCase.op || testCase.relName || "case"}`);
+
+    try {
+      compileARM64ArtifactFromObject(testCase.object, {
+        entry: testCase.relName || testCase.op || testCase.relationName,
+        inputPattern: testCase.inputPattern,
+        outputPattern: testCase.outputPattern,
+        outputPath: exePath,
+        optLevel: this.optLevel,
+        profile: Boolean(options.profile)
+      });
+
+      return {
+        status: "ok",
+        exePath,
+        inputWire: testCase.inputWire,
+        inputVal: testCase.inputVal,
+        outputPattern: testCase.outputPattern
+      };
+    } catch (error) {
+      return {
+        status: "failed",
+        error: error.stack || error.message || String(error)
+      };
+    }
+  }
+
+  async createRunner(compiledCases) {
+    return new ARM64Runner(this.id);
+  }
+}

package/harness/adapters/kvm.mjs

@@ -0,0 +1,144 @@
+import {
+  codes,
+  decodeWire,
+  encodeToWire,
+  executeKVM,
+  lowerToKVM,
+  specializeKVM
+} from "../../backend-api.mjs";
+import { BackendAdapter, BackendRunner } from "../backend.mjs";
+
+class KVMRunner extends BackendRunner {
+  constructor(backendId, state, options = {}) {
+    super(backendId);
+    this.state = state;
+    this.wire = options.wire !== false;
+  }
+
+  getContext(profile = null) {
+    return {
+      rels: this.state?.rels || {},
+      codes: this.state?.codes || {},
+      findCode: codes.find,
+      profile,
+      hashToName: this.state?.hashToName || null,
+      options: {
+        envelopeFree: true
+      }
+    };
+  }
+
+  async runSingle(testCase, options = {}) {
+    const profile = options.profile ? {} : null;
+    const context = this.getContext(profile);
+    const targetFunc = testCase.specKvmFunc || testCase.kvmFunc;
+
+    let inputVal = testCase.inputVal;
+    const t0 = process.hrtime.bigint();
+    let tDecode = t0;
+
+    if (this.wire && testCase.inputWire) {
+      const decoded = decodeWire(testCase.inputWire);
+      inputVal = decoded.value;
+      tDecode = process.hrtime.bigint();
+    }
+
+    const tEvalStart = process.hrtime.bigint();
+    const result = executeKVM(targetFunc, inputVal, context);
+    const tEvalEnd = process.hrtime.bigint();
+
+    let outputWire = null;
+    let tEncode = tEvalEnd;
+    if (this.wire && result !== undefined) {
+      const outPattern = testCase.outputPattern || targetFunc.outputPattern;
+      outputWire = encodeToWire(result, outPattern);
+      tEncode = process.hrtime.bigint();
+    }
+
+    const decodeNs = Number(tDecode - t0);
+    const evalNs = Number(tEvalEnd - tEvalStart);
+    const encodeNs = Number(tEncode - tEvalEnd);
+    const totalNs = Number(tEncode - t0);
+
+    const trace = {
+      ipcReadNs: 0,
+      decodeNs,
+      flatInNs: 0,
+      evalNs,
+      flatOutNs: 0,
+      encodeNs,
+      ipcWriteNs: 0,
+      totalNs
+    };
+
+    return {
+      outputVal: result,
+      outputWire,
+      trace,
+      profile
+    };
+  }
+
+  async runBatch(compiledCases) {
+    const context = this.getContext();
+    for (const c of compiledCases) {
+      const targetFunc = c.specKvmFunc || c.kvmFunc;
+      if (this.wire && c.inputWire) {
+        const decoded = decodeWire(c.inputWire);
+        const result = executeKVM(targetFunc, decoded.value, context);
+        if (result !== undefined) {
+          const outPattern = c.outputPattern || targetFunc.outputPattern;
+          encodeToWire(result, outPattern);
+        }
+      } else {
+        executeKVM(targetFunc, c.inputVal, context);
+      }
+    }
+  }
+}
+
+export class KVMAdapter extends BackendAdapter {
+  constructor(state = null, options = {}) {
+    const wire = options.wire !== false;
+    const id = options.id || (wire ? "kvm" : "kvm-pure");
+    const name = options.name || (wire ? "kVM Interpreter (wire codec)" : "kVM Interpreter (in-memory)");
+    super(id, name, options);
+    this.state = state;
+    this.wire = wire;
+  }
+
+  setState(state) {
+    this.state = state;
+  }
+
+  async compile(testCase, options = {}) {
+    try {
+      const kvmFunc = testCase.kvmFunc || lowerToKVM(testCase.relDef, testCase.relName || testCase.op);
+      let specKvmFunc = kvmFunc;
+      if (testCase.inputPattern) {
+        try {
+          specKvmFunc = specializeKVM(kvmFunc, testCase.inputPattern);
+        } catch {
+          specKvmFunc = kvmFunc;
+        }
+      }
+      return {
+        status: "ok",
+        kvmFunc,
+        specKvmFunc,
+        inputVal: testCase.inputVal,
+        inputWire: testCase.inputWire,
+        outputPattern: testCase.outputPattern
+      };
+    } catch (error) {
+      return {
+        status: "failed",
+        error: error.stack || error.message || String(error)
+      };
+    }
+  }
+
+  async createRunner(compiledCases) {
+    return new KVMRunner(this.id, this.state, { wire: this.wire });
+  }
+}

package/harness/adapters/llvm.mjs

@@ -0,0 +1,118 @@
+import fs from "node:fs";
+import path from "node:path";
+import { execSync } from "node:child_process";
+import { BackendAdapter, BackendRunner } from "../backend.mjs";
+import { PersistentExecutable } from "../persistent.mjs";
+import { compileObjectToExecutable } from "../../backends/llvm/src/executable.mjs";
+import { decodeWire } from "../../backend-api.mjs";
+import { decodeObject, encodeObject } from "../../object.mjs";
+
+function cloneObjectPayload(object) {
+  return decodeObject(encodeObject(object));
+}
+
+class LLVMRunner extends BackendRunner {
+  constructor(backendId, servers = new Map()) {
+    super(backendId);
+    this.servers = servers;
+  }
+
+  getServer(exePath) {
+    let server = this.servers.get(exePath);
+    if (!server) {
+      server = new PersistentExecutable(exePath);
+      this.servers.set(exePath, server);
+    }
+    return server;
+  }
+
+  async runSingle(testCase, options = {}) {
+    const server = this.getServer(testCase.exePath);
+    if (options.trace) {
+      const res = await server.requestWithTrace(testCase.inputWire);
+      const decoded = decodeWire(res.payload);
+      return {
+        outputWire: res.payload,
+        outputVal: decoded.value,
+        outputPattern: decoded.pattern,
+        trace: res.trace,
+        profile: res.trace?.profile
+      };
+    } else {
+      const payload = await server.request(testCase.inputWire);
+      const decoded = decodeWire(payload);
+      return {
+        outputWire: payload,
+        outputVal: decoded.value,
+        outputPattern: decoded.pattern
+      };
+    }
+  }
+
+  async runBatch(compiledCases) {
+    for (const c of compiledCases) {
+      const server = this.getServer(c.exePath);
+      await server.request(c.inputWire);
+    }
+  }
+
+  async close() {
+    for (const server of this.servers.values()) {
+      server.close();
+    }
+    this.servers.clear();
+  }
+}
+
+export class LLVMAdapter extends BackendAdapter {
+  constructor(options = {}) {
+    const clangOpt = options.clangOpt || "-O3";
+    super("llvm", `LLVM Executable (persistent, ${clangOpt})`, options);
+    this.clangOpt = clangOpt;
+    this._available = null;
+  }
+
+  isAvailable() {
+    if (this._available !== null) return this._available;
+    try {
+      execSync("which clang", { stdio: "ignore" });
+      this._available = true;
+    } catch {
+      this._available = false;
+    }
+    return this._available;
+  }
+
+  async compile(testCase, options = {}) {
+    const cacheDir = options.cacheDir || path.join(process.cwd(), ".cache");
+    fs.mkdirSync(cacheDir, { recursive: true });
+    const exePath = path.join(cacheDir, `harness_llvm_${testCase.op || testCase.relName || "case"}`);
+
+    try {
+      const relation = testCase.relName || testCase.op || testCase.relationName || testCase.object.main;
+      compileObjectToExecutable(cloneObjectPayload(testCase.object), exePath, {
+        relation,
+        inputPattern: testCase.inputPattern,
+        runtimeMode: options.runtimeMode || "fast",
+        clangOpt: this.clangOpt
+      });
+
+      return {
+        status: "ok",
+        exePath,
+        inputWire: testCase.inputWire,
+        inputVal: testCase.inputVal,
+        outputPattern: testCase.outputPattern
+      };
+    } catch (error) {
+      return {
+        status: "failed",
+        error: error.stack || error.message || String(error)
+      };
+    }
+  }
+
+  async createRunner(compiledCases) {
+    return new LLVMRunner(this.id);
+  }
+}