@fraczak/k 7.0.2 → 7.2.0

package/backends/llvm/bin/k-llvm-compile.mjs

@@ -9,26 +9,44 @@ import {
 } from "../src/cli.mjs";
 import {
   compileObjectToLLVM,
-  compileLLVMArtifactFromKVM
+  compileLLVMArtifactFromKVM,
+  compileLLVMArtifactFromPKIR
 } from "../src/llvm.mjs";
 import { inputPatternForObjectRelation } from "../src/executable.mjs";
 
-function usage(stream = console.error) {
+function helpText() {
   const prog = argv[1] || "k-llvm-compile.mjs";
-  stream(`Usage: node ${prog} [options] [source-snippet | input-file [output.ll]]`);
-  stream("Compile a k .ko/.klib object, source, or .kvm input into prototype LLVM IR.");
-  stream("");
-  stream("Arguments:");
-  stream("  source-snippet  Inline k source, in the same style as k.mjs.");
-  stream("  input-file      Source .k, .ko, .klib, or .kvm file. Reads UTF-8 source from stdin when omitted.");
-  stream("  output.ll       Output LLVM IR path. Writes to stdout when omitted.");
-  stream("");
-  stream("Options:");
-  stream("  -o, --output path  Specify output file path explicitly.");
-  stream("  --lib file         Load one .klib dependency before compiling.");
-  stream("  --export spec      Export a library alias into source scope. May be repeated.");
-  stream("                     spec is 'name' or 'libname:localname'.");
-  stream("  -h, --help         Show this help.");
+  return [
+    "Compile a k .ko/.klib object, source, .kvm, or .pkir into LLVM IR.",
+    "",
+    `Usage: ${prog} [options]`,
+    "       [source-snippet | input-file [output.ll]]",
+    "",
+    "Arguments:",
+    "  source-snippet   Inline k source snippet (in the style of k.mjs).",
+    "  input-file       Source .k, .ko, .klib, .kvm, or .pkir file.",
+    "                   Reads UTF-8 source from stdin when omitted.",
+    "  output.ll        Output LLVM IR path (default: stdout).",
+    "",
+    "Options:",
+    "  -o, --output path  Specify output file path explicitly.",
+    "  --pkir             Lower via Polymorphic K IR (PKIR).",
+    "  --retype rel       Relation to compile (default: object main).",
+    "  --lib file         Load one .klib or .k library before compiling.",
+    "  --export spec      Export library alias into scope. Repeatable.",
+    "                     spec is 'name' or 'libname:localname'.",
+    "  -h, --help         Show this help.",
+    "",
+    "Examples:",
+    `  ${prog} 'x = |x; x x' program.ll`,
+    `  ${prog} program.k -o program.ll`,
+    `  ${prog} --pkir program.k -o program.ll`,
+    `  ${prog} --lib core.klib --export add program.k program.ll`
+  ].join("\n");
+}
+
+function usage(stream = console.error) {
+  stream(helpText());
 }
 
 try {
@@ -39,29 +57,49 @@ try {
   }
 
   let explicitOutput = null;
+  let relation = null;
   const remainingArgs = [];
   while (args.length > 0) {
     const arg = args[0];
     if (arg === "-o" || arg === "--output") {
       args.shift();
       explicitOutput = args.shift();
-      if (!explicitOutput) throw new Error("-o/--output requires a file argument");
+      if (!explicitOutput) {
+        throw new Error("-o/--output requires a file argument");
+      }
+    } else if (arg === "--retype") {
+      args.shift();
+      relation = args.shift();
+      if (!relation) {
+        throw new Error("--retype requires a relation name");
+      }
     } else {
       remainingArgs.push(args.shift());
     }
   }
 
-  const { libraries, exportSpecs } = parseCompileOptions(remainingArgs);
-  const input = resolveProgramInput(remainingArgs, { allowStdinSource: true });
+  const { libraries, exportSpecs, usePKIR } =
+    parseCompileOptions(remainingArgs);
+  const input = resolveProgramInput(remainingArgs, {
+    allowStdinSource: true
+  });
   const positionalOutput = remainingArgs.shift();
   if (remainingArgs.length > 0) throw new Error("Too many arguments");
 
   const outputPath = explicitOutput || positionalOutput || null;
 
   let llvm;
-  if (input.kind === "kvm") {
+  if (input.kind === "pkir") {
+    const pkir = JSON.parse(fs.readFileSync(input.path, "utf8"));
+    const entry = relation || pkir.entry || pkir.name || "__main__";
+    llvm = compileLLVMArtifactFromPKIR(pkir, {
+      relation: entry,
+      inputPattern: pkir.inputPattern,
+      outputPattern: pkir.outputPattern
+    });
+  } else if (input.kind === "kvm") {
     const kvm = JSON.parse(fs.readFileSync(input.path, "utf8"));
-    const entry = kvm.entry || kvm.relation || "__main__";
+    const entry = relation || kvm.entry || kvm.relation || "__main__";
     const mainFunc = kvm.functions?.[entry];
     const artifact = compileLLVMArtifactFromKVM(kvm, {
       relation: entry,
@@ -76,9 +114,11 @@ try {
       stdin
     });
 
+    const targetRel = relation || object.main;
     const res = compileObjectToLLVM(object, {
-      relation: object.main,
-      inputPattern: inputPatternForObjectRelation(object, object.main)
+      relation: targetRel,
+      inputPattern: inputPatternForObjectRelation(object, targetRel),
+      usePKIR
     });
     llvm = res.llvm;
   }
@@ -90,6 +130,5 @@ try {
   }
 } catch (error) {
   console.error(error.stack || error.message || String(error));
-  usage();
   exit(1);
 }

package/backends/llvm/bin/k-llvm.mjs

@@ -0,0 +1,292 @@
+#!/usr/bin/env node
+
+import crypto from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { argv, exit, stdin, stdout } from "node:process";
+import { fileURLToPath } from "node:url";
+
+import { decodeWire } from "@fraczak/k/backend-api.mjs";
+import {
+  compileProgramInputToObject,
+  parseCompileOptions,
+  readAll,
+  resolveProgramInput
+} from "../src/cli.mjs";
+import {
+  compileKVMToExecutable,
+  compileObjectToExecutable,
+  compilePKIRToExecutable
+} from "../src/executable.mjs";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+
+function helpText() {
+  const prog = argv[1] || "k-llvm.mjs";
+  const defaultCache = path.join(os.tmpdir(), "k-llvm-cache");
+  return [
+    "Compile k source, .ko, .kvm, or .pkir to native machine code via",
+    "LLVM and execute it over a binary pattern+value wire stream.",
+    "",
+    `Usage: ${prog} [options]`,
+    "       ( source-snippet | input-file ) [ input-wire-file ]",
+    "",
+    "Arguments:",
+    "  source-snippet   Inline k source snippet (in the style of k.mjs).",
+    "  input-file       Source .k, .ko, .klib, .kvm, or .pkir file.",
+    "  input-wire-file  Optional binary wire input (default: stdin).",
+    "",
+    "Options:",
+    "  --lib file        Load one .klib or .k library before compiling.",
+    "  --export spec     Export library alias into scope. Repeatable.",
+    "                    spec is 'name' or 'libname:localname'.",
+    "  --pkir            Lower via Polymorphic K IR (PKIR).",
+    "  --retype rel      Relation to specialize (default: object main).",
+    "  --cache-dir path  Directory to store compiled native executables.",
+    "  --no-cache        Compile in a temporary directory and discard.",
+    "  --opt flag        Clang optimization level (-O0, -O1, -O2, -O3).",
+    "  -h, --help        Show this help.",
+    "",
+    "Environment Variables:",
+    `  K_LLVM_CACHE_DIR  Cache directory (default: ${defaultCache}).`,
+    "  K_LLVM_CLANG_OPT  Default Clang optimization flag (default: -O1).",
+    "  CLANG, CC         Path to Clang compiler binary (default: clang).",
+    "  K_TRACE, TRACE    When '1', output phase tracing to stderr.",
+    "",
+    "Examples:",
+    `  echo '"abc"' | k-parse | ${prog} '{.1 0}' | k-print`,
+    `  echo '{"x":1,"y":2}' | k-json --parse | \\`,
+    `    ${prog} --lib Examples/poly.k --export concat \\`,
+    `      's={()xs,()ys}concat; s' | k-json --print`
+  ].join("\n");
+}
+
+function usage(stream = console.error) {
+  stream(helpText());
+}
+
+let cachedFingerprint = null;
+function getBackendFingerprint() {
+  if (cachedFingerprint != null) return cachedFingerprint;
+  const hash = crypto.createHash("sha256");
+  for (const relPath of [
+    "src/llvm.mjs",
+    "src/kvm2llvm.mjs",
+    "src/executable.mjs",
+    "runtime/krt.c",
+    "runtime/krt.h"
+  ]) {
+    const full = path.join(root, relPath);
+    if (fs.existsSync(full)) {
+      hash.update(relPath);
+      hash.update("\0");
+      hash.update(fs.readFileSync(full));
+      hash.update("\0");
+    }
+  }
+  cachedFingerprint = hash.digest("hex").slice(0, 16);
+  return cachedFingerprint;
+}
+
+function computeCacheKey({
+  fingerprint,
+  inputKey,
+  relation,
+  inputPattern,
+  usePKIR,
+  clangOpt
+}) {
+  const hash = crypto.createHash("sha256");
+  hash.update("k-llvm-runner-v1\0");
+  hash.update(fingerprint);
+  hash.update("\0");
+  hash.update(String(relation || ""));
+  hash.update("\0");
+  hash.update(JSON.stringify(inputPattern || null));
+  hash.update("\0");
+  hash.update(usePKIR ? "pkir\0" : "monomorphic\0");
+  hash.update(String(clangOpt || ""));
+  hash.update("\0");
+  hash.update(inputKey);
+  return hash.digest("hex");
+}
+
+function runNativeExecutable(exePath, inputBuffer) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(exePath, [], {
+      stdio: ["pipe", "pipe", "inherit"]
+    });
+    const chunks = [];
+    child.stdout.on("data", (chunk) => chunks.push(chunk));
+    child.on("error", reject);
+    child.on("close", (status) => {
+      if (status !== 0) {
+        reject(new Error(`${exePath} exited with status ${status}`));
+        return;
+      }
+      resolve(Buffer.concat(chunks));
+    });
+    child.stdin.end(inputBuffer);
+  });
+}
+
+async function main() {
+  const rawArgs = argv.slice(2);
+  if (rawArgs.includes("-h") || rawArgs.includes("--help")) {
+    usage(console.log);
+    return;
+  }
+
+  let cacheDir = process.env.K_LLVM_CACHE_DIR ||
+    path.join(os.tmpdir(), "k-llvm-cache");
+  let useCache = true;
+  let relation = null;
+  let clangOpt = process.env.K_LLVM_CLANG_OPT || "-O1";
+
+  const remainingArgs = [];
+  while (rawArgs.length > 0) {
+    const arg = rawArgs[0];
+    if (arg === "--cache-dir") {
+      rawArgs.shift();
+      cacheDir = rawArgs.shift();
+      if (!cacheDir) {
+        throw new Error("--cache-dir requires a directory path");
+      }
+    } else if (arg === "--no-cache") {
+      rawArgs.shift();
+      useCache = false;
+    } else if (arg === "--retype") {
+      rawArgs.shift();
+      relation = rawArgs.shift();
+      if (!relation) {
+        throw new Error("--retype requires a relation name");
+      }
+    } else if (arg === "--opt") {
+      rawArgs.shift();
+      clangOpt = rawArgs.shift();
+      if (!clangOpt) {
+        throw new Error("--opt requires a flag (e.g. -O1)");
+      }
+    } else {
+      remainingArgs.push(rawArgs.shift());
+    }
+  }
+
+  const { libraries, exportSpecs, usePKIR } =
+    parseCompileOptions(remainingArgs);
+  const programInput = resolveProgramInput(remainingArgs);
+  const inputWirePath = remainingArgs.shift();
+  if (remainingArgs.length > 0) {
+    throw new Error(`Unexpected argument: ${remainingArgs[0]}`);
+  }
+
+  const inputBuffer = await readAll(
+    inputWirePath == null ? stdin : fs.createReadStream(inputWirePath)
+  );
+  if (inputBuffer.length === 0) {
+    throw new Error("No input wire data received on stdin");
+  }
+
+  const { pattern: inputEnvelopePattern } = decodeWire(inputBuffer);
+
+  let exePath;
+  let tempExeDir = null;
+
+  if (useCache) {
+    fs.mkdirSync(cacheDir, { recursive: true });
+    const fingerprint = getBackendFingerprint();
+    const inputKey = JSON.stringify({
+      programInput,
+      libraries: libraries.map((lib) => Object.keys(lib.rels || {})),
+      exportSpecs
+    });
+    const key = computeCacheKey({
+      fingerprint,
+      inputKey,
+      relation,
+      inputPattern: inputEnvelopePattern,
+      usePKIR,
+      clangOpt
+    });
+    exePath = path.join(cacheDir, `${key}.exe`);
+    if (!fs.existsSync(exePath)) {
+      const tmpPath = path.join(cacheDir, `${key}.${process.pid}.tmp`);
+      if (programInput.kind === "pkir") {
+        const pkir = JSON.parse(
+          fs.readFileSync(programInput.path, "utf8")
+        );
+        compilePKIRToExecutable(pkir, tmpPath, {
+          inputPattern: inputEnvelopePattern,
+          clangOpt
+        });
+      } else if (programInput.kind === "kvm") {
+        const kvm = JSON.parse(
+          fs.readFileSync(programInput.path, "utf8")
+        );
+        compileKVMToExecutable(kvm, tmpPath, {
+          inputPattern: inputEnvelopePattern,
+          clangOpt,
+          usePKIR
+        });
+      } else {
+        const object = await compileProgramInputToObject(programInput, {
+          libraries,
+          exportSpecs,
+          stdin
+        });
+        compileObjectToExecutable(object, tmpPath, {
+          relation: relation || object.main,
+          inputPattern: inputEnvelopePattern,
+          clangOpt,
+          usePKIR
+        });
+      }
+      fs.renameSync(tmpPath, exePath);
+    }
+  } else {
+    tempExeDir = fs.mkdtempSync(path.join(os.tmpdir(), "k-llvm-run-"));
+    exePath = path.join(tempExeDir, "prog.exe");
+    if (programInput.kind === "pkir") {
+      const pkir = JSON.parse(fs.readFileSync(programInput.path, "utf8"));
+      compilePKIRToExecutable(pkir, exePath, {
+        inputPattern: inputEnvelopePattern,
+        clangOpt
+      });
+    } else if (programInput.kind === "kvm") {
+      const kvm = JSON.parse(fs.readFileSync(programInput.path, "utf8"));
+      compileKVMToExecutable(kvm, exePath, {
+        inputPattern: inputEnvelopePattern,
+        clangOpt,
+        usePKIR
+      });
+    } else {
+      const object = await compileProgramInputToObject(programInput, {
+        libraries,
+        exportSpecs,
+        stdin
+      });
+      compileObjectToExecutable(object, exePath, {
+        relation: relation || object.main,
+        inputPattern: inputEnvelopePattern,
+        clangOpt,
+        usePKIR
+      });
+    }
+  }
+
+  try {
+    const outputBuffer = await runNativeExecutable(exePath, inputBuffer);
+    stdout.write(outputBuffer);
+  } finally {
+    if (tempExeDir) {
+      fs.rmSync(tempExeDir, { recursive: true, force: true });
+    }
+  }
+}
+
+main().catch((error) => {
+  console.error(error.stack || error.message || String(error));
+  exit(1);
+});

package/backends/llvm/runtime/krt.c

@@ -1914,6 +1914,14 @@ unsigned char *k_encode_wire_as(k_pattern *pattern, k_value *value, size_t *out_
   return bytes;
 }
 
+unsigned char *k_encode_wire(k_value *value, size_t *out_length) {
+  k_pattern *pattern = derive_pattern(value);
+  if (pattern == NULL) return NULL;
+  unsigned char *bytes = k_encode_wire_as(pattern, value, out_length);
+  free_pattern(pattern);
+  return bytes;
+}
+
 int k_write_wire_as(FILE *out, k_pattern *pattern, k_value *value) {
   size_t length = 0;
   unsigned char *bytes = k_encode_wire_as(pattern, value, &length);

package/backends/llvm/runtime/krt.h

@@ -110,6 +110,7 @@ void k_wire_prefix_free(k_wire_prefix prefix);
 k_value *k_decode_wire_value_with_prefix(const unsigned char *bytes, size_t length, k_wire_prefix *prefix, k_pattern *pattern, k_rt *rt);
 unsigned char *k_encode_wire_as_with_prefix(k_wire_prefix *prefix, k_pattern *pattern, k_value *value, size_t *out_length);
 unsigned char *k_encode_wire_as(k_pattern *pattern, k_value *value, size_t *out_length);
+unsigned char *k_encode_wire(k_value *value, size_t *out_length);
 int k_write_wire(FILE *out, k_value *value);
 int k_write_wire_as(FILE *out, k_pattern *pattern, k_value *value);
 

package/backends/llvm/src/cli.mjs

@@ -1,20 +1,73 @@
 import fs from "node:fs";
 import path from "node:path";
-import { compileLibrary, compileObjectBuffer, decodeObject, loadLibrary } from "@fraczak/k/object.mjs";
+import {
+  compileLibrary,
+  compileObjectBuffer,
+  decodeObject,
+  loadLibrary
+} from "@fraczak/k/object.mjs";
 
 export function readAll(stream) {
   return new Promise((resolve, reject) => {
     const chunks = [];
-    stream.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
+    stream.on("data", (chunk) => {
+      chunks.push(
+        Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
+      );
+    });
     stream.on("end", () => resolve(Buffer.concat(chunks)));
     stream.on("error", reject);
   });
 }
 
+export function loadLibraryFromPath(libPath) {
+  const ext = path.extname(libPath).toLowerCase();
+  if (ext === ".klib" || ext === ".ko") {
+    return loadLibrary(decodeObject(fs.readFileSync(libPath)));
+  }
+  if (ext === ".k") {
+    let depLibs = [];
+    const baseName = path.basename(libPath);
+    if (baseName === "poly.k") {
+      const dir = path.dirname(libPath);
+      const arithPath = path.join(dir, "arithmetics.k");
+      if (fs.existsSync(arithPath)) {
+        const arithLib = loadLibrary(
+          compileLibrary(fs.readFileSync(arithPath, "utf8"), {
+            source: arithPath
+          })
+        );
+        depLibs.push(arithLib);
+      }
+    }
+    const rawSource = fs.readFileSync(libPath, "utf8");
+    if (depLibs.length > 0) {
+      const neededExports = [
+        "0", "int", "inc", "dec", "nat", "zero_int?", "neg?", "GT"
+      ];
+      const preamble = buildExportPreamble(neededExports, depLibs);
+      const compiled = compileLibrary(preamble + rawSource, {
+        source: libPath,
+        libraries: depLibs
+      });
+      const mergedLib = loadLibrary(compiled);
+      for (const dep of depLibs) {
+        for (const [k, v] of Object.entries(dep.relAlias || {})) {
+          if (!mergedLib.relAlias[k]) mergedLib.relAlias[k] = v;
+        }
+      }
+      return mergedLib;
+    }
+    return loadLibrary(compileLibrary(rawSource, { source: libPath }));
+  }
+  throw new Error(`Unsupported library extension: ${ext}`);
+}
+
 export function parseCompileOptions(args) {
   const libraries = [];
   const exportSpecs = [];
   let loadedLibrary = false;
+  let usePKIR = false;
 
   while (args.length > 0) {
     if (args[0] === "--") {
@@ -22,22 +75,26 @@ export function parseCompileOptions(args) {
       break;
     }
     if (args[0] === "--lib") {
-      if (loadedLibrary) throw new Error("--lib may only be specified once");
+      if (loadedLibrary) {
+        throw new Error("--lib may only be specified once");
+      }
       args.shift();
       const libPath = args.shift();
       if (!libPath) throw new Error("--lib requires a file argument");
-      const ext = path.extname(libPath).toLowerCase();
-      if (ext === ".k") {
-        libraries.push(loadLibrary(compileLibrary(fs.readFileSync(libPath, "utf8"), { source: libPath })));
-      } else {
-        libraries.push(loadLibrary(decodeObject(fs.readFileSync(libPath))));
-      }
+      libraries.push(loadLibraryFromPath(libPath));
       loadedLibrary = true;
     } else if (args[0] === "--export") {
       args.shift();
       const spec = args.shift();
-      if (!spec) throw new Error("--export requires a spec argument (name or libname:localname)");
+      if (!spec) {
+        throw new Error(
+          "--export requires a spec argument (name or libname:localname)"
+        );
+      }
       exportSpecs.push(spec);
+    } else if (args[0] === "--pkir") {
+      args.shift();
+      usePKIR = true;
     } else if (args[0].startsWith("--")) {
       break;
     } else {
@@ -45,7 +102,7 @@ export function parseCompileOptions(args) {
     }
   }
 
-  return { libraries, exportSpecs };
+  return { libraries, exportSpecs, usePKIR };
 }
 
 export function fileInputType(filePath) {
@@ -53,6 +110,7 @@ export function fileInputType(filePath) {
   if (ext === ".ko") return "ko";
   if (ext === ".kvm") return "kvm";
   if (ext === ".klib") return "klib";
+  if (ext === ".pkir") return "pkir";
   return "source-file";
 }
 
@@ -69,11 +127,15 @@ export function resolveProgramInput(args, { allowStdinSource = false } = {}) {
     return { kind, path: inputArg, sourceName: inputArg };
   }
 
-  if ([".k", ".ko", ".kvm", ".klib"].includes(ext)) {
+  if ([".k", ".ko", ".kvm", ".klib", ".pkir"].includes(ext)) {
     throw new Error(`Input file not found: ${inputArg}`);
   }
 
-  return { kind: "inline-source", source: inputArg, sourceName: "<inline>" };
+  return {
+    kind: "inline-source",
+    source: inputArg,
+    sourceName: "<inline>"
+  };
 }
 
 export function buildExportPreamble(exports = [], libraries = []) {
@@ -94,9 +156,15 @@ export function buildExportPreamble(exports = [], libraries = []) {
   }
   const lines = [];
   for (const spec of exports) {
-    const [libName, localName] = spec.includes(":") ? spec.split(":", 2) : [spec, spec];
+    const [libName, localName] = spec.includes(":")
+      ? spec.split(":", 2)
+      : [spec, spec];
     const hash = aliasMap[libName];
-    if (!hash) throw new Error(`--export: '${libName}' not found in loaded libraries`);
+    if (!hash) {
+      throw new Error(
+        `--export: '${libName}' not found in loaded libraries`
+      );
+    }
     const body = hash.startsWith("@") ? hash.slice(1) : hash;
     lines.push(`${localName} = @${body};`);
   }
@@ -116,7 +184,9 @@ export async function compileProgramInputToObject(
     });
     object = decodeObject(objBuffer);
   } else if (input.kind === "stdin-source") {
-    if (!stdin) throw new Error("stdin source requested without a stdin stream");
+    if (!stdin) {
+      throw new Error("stdin source requested without a stdin stream");
+    }
     const source = (await readAll(stdin)).toString("utf8");
     const preamble = buildExportPreamble(exportSpecs, libraries);
     const objBuffer = compileObjectBuffer(preamble + source, {
@@ -135,7 +205,15 @@ export async function compileProgramInputToObject(
   } else if (input.kind === "ko" || input.kind === "klib") {
     object = decodeObject(fs.readFileSync(input.path));
   } else if (input.kind === "kvm") {
-    throw new Error("Input is already lowered to .kvm bytecode; use compileLLVMArtifactFromKVM or compileKVMToExecutable.");
+    throw new Error(
+      "Input is already lowered to .kvm bytecode; use " +
+      "compileLLVMArtifactFromKVM or compileKVMToExecutable."
+    );
+  } else if (input.kind === "pkir") {
+    throw new Error(
+      "Input is already lowered to .pkir IR; use " +
+      "compileLLVMArtifactFromPKIR or compilePKIRToExecutable."
+    );
   } else {
     throw new Error(`Unsupported input kind: ${input.kind}`);
   }