lecodes-cli 0.19.0 → 0.20.0

package/src/dev/devServer.ts

@@ -0,0 +1,333 @@
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"
+import { createReadStream, statSync } from "node:fs"
+import { join } from "node:path"
+import type { Manifest } from "../manifest"
+import { resolveLocalPath } from "../localFiles"
+import { collectLocalEntries } from "../compile/collectLocal"
+import { applyLocalShaders, isShaderArtifactName, shadersCacheDir } from "../compile/shaders"
+import { ambiguousEntryWarning, compileProject } from "../compile/projectCompile"
+import { applyProjectFonts } from "../compile/fonts"
+import { applyAppAssetIcons } from "../compile/assetIcons"
+import { renderDevScreen, renderPrelude } from "./clientTemplates"
+import { upgradeWebSocket, type WsConnection } from "./wsServer"
+import { toKey } from "../util"
+import { makeStackDemangler } from "../compile/nativeStack"
+
+/*
+ * The `lecodes dev` LAN server: serves the project to devices and hot-reloads it on change.
+ *
+ * All routes live under a capability-token base path (/t/<token>):
+ *   GET /t/<tok>/dev.js          the bootstrap bundle the QR points at (placeholder screen + prelude)
+ *   GET /t/<tok>/bundle.js?t=…   the current project bundle + reload prelude (?t=poll for no-WS hosts)
+ *   GET /t/<tok>/assets/<path>   files streamed from the working tree (the bundle's asset URLs)
+ *   GET /t/<tok>/shaders/<name>  locally compiled shader artifacts (.lecodes/shaders — matc)
+ *   GET /t/<tok>/version         {"v":N} — polling fallback
+ *   POST /t/<tok>/log            log/error backchannel for polling clients
+ *   WS  /t/<tok>/ws              reload push + log/error backchannel
+ *   GET /t/<tok>/web/[file]      the Web Lite browser runner (viewer-lite dist-embed, `dev --web`):
+ *                                it loads bundle.js?t=web — the bundle WITHOUT the native prelude;
+ *                                the page owns reload (ws / version polling) and the log backchannel
+ *
+ * Compile is eager: the command calls recompile() on every file change; a successful compile bumps
+ * `v` and broadcasts {type:"reload", v} to connected sockets (polling clients see /version move).
+ * A failed compile keeps the last good bundle — devices stay on it until the next good compile.
+ */
+
+export type DevServerOptions = {
+  root: string
+  manifest: Manifest | null
+  name: string
+  /** The advertised origin baked into bundles + the prelude, e.g. "http://192.168.1.5:4499". */
+  origin: string
+  /** Bind address (default 0.0.0.0). */
+  bind?: string
+  port: number
+  token: string
+  entryOverride?: string
+  /** The Web Lite runner folder to serve under /web/ (src/dev/webRunner.ts); absent = no browser route. */
+  webDir?: string | null
+  /** Diagnostics (compile errors, device logs); defaults to stderr-style silence-safe noop. */
+  log?: (msg: string) => void
+}
+
+export type RecompileResult =
+  | { ok: true, changed: boolean, v: number, warnings: string[] }
+  | { ok: false, error: string }
+
+export type DevServer = {
+  url: string
+  /** The browser runner page (`dev --web`), or null when no runner is vendored. */
+  webUrl: string | null
+  /** The token base path ("/t/<token>"). */
+  base: string
+  /** The actually-bound port (differs from options.port when 0 was passed, e.g. in tests). */
+  port: number
+  v: () => number
+  clientCount: () => number
+  recompile: () => Promise<RecompileResult>
+  close: () => void
+}
+
+const MIME: Record<string, string> = {
+  png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", webp: "image/webp", gif: "image/gif",
+  svg: "image/svg+xml", ktx: "application/octet-stream", ktx2: "application/octet-stream",
+  glb: "model/gltf-binary", bin: "application/octet-stream",
+  mp3: "audio/mpeg", wav: "audio/wav", ogg: "audio/ogg", m4a: "audio/mp4",
+  mp4: "video/mp4", webm: "video/webm",
+  ttf: "font/ttf", otf: "font/otf", woff: "font/woff", woff2: "font/woff2",
+  json: "application/json", txt: "text/plain", js: "text/javascript", mjs: "text/javascript",
+  css: "text/css", html: "text/html", wasm: "application/wasm", map: "application/json",
+}
+
+const contentType = (path: string): string => {
+  const i = path.lastIndexOf(".")
+  return (i >= 0 && MIME[path.slice(i + 1).toLowerCase()]) || "application/octet-stream"
+}
+
+const sendText = (res: ServerResponse, status: number, type: string, body: string) => {
+  res.writeHead(status, { "content-type": type, "cache-control": "no-store" })
+  res.end(body)
+}
+
+const sendJson = (res: ServerResponse, status: number, body: unknown) =>
+  sendText(res, status, "application/json", JSON.stringify(body))
+
+const readBody = (req: IncomingMessage): Promise<string> =>
+  new Promise((resolve) => {
+    let body = ""
+    req.on("data", (c: Buffer) => { if (body.length < 256 * 1024) body += c.toString("utf8") })
+    req.on("end", () => resolve(body))
+    req.on("error", () => resolve(body))
+  })
+
+export const startDevServer = async (options: DevServerOptions): Promise<DevServer> => {
+  const { root, manifest, name, origin, token, entryOverride } = options
+  const log = options.log ?? (() => {})
+  const base = `/t/${token}`
+
+  // ---- compile state (the last good bundle; devices never see a broken one) ----------------------
+  let v = 0
+  let bundleJs: string | null = null
+  let compileChain: Promise<unknown> = Promise.resolve()
+
+  // The header's `// last-updated:` timestamp differs on every compile — ignore it when deciding
+  // whether anything really changed (a no-op save must not reload devices).
+  const stripVolatile = (js: string) => js.replace(/^\/\/ last-updated: .*$/m, "")
+
+  const compileOnce = async (): Promise<RecompileResult> => {
+    try {
+      const { entries, warnings } = collectLocalEntries(root, manifest, `${base}/assets`, { localShaders: true })
+      // Custom shaders compile LOCALLY via matc (downloaded on first use) into .lecodes/shaders,
+      // served under /shaders/ below — no push needed. Degrades to a warning when matc is
+      // unavailable; a broken .mat throws and fails this compile like any code error.
+      warnings.push(...(await applyLocalShaders(entries, { root, srcPrefix: `${base}/shaders` })).warnings)
+      // font() macro: vendor registry faces into assets/fonts/ (served from the working tree like
+      // any asset — LAN/offline friendly) and emit the `// fonts:` boot header.
+      const fontsHeader = await applyProjectFonts(entries, {
+        mode: "vendor",
+        root,
+        vendorDir: root,
+        vendorPrefix: "",
+        publicUrl: origin,
+        makeResourceEntry: (path, absPath) => ({ path, type: "resource", fileSrc: `${base}/assets` + encodeURI(path), absPath }),
+      })
+      // assetIcon() macro: resolve from the user-level cache (~/.lecodes/cache/icons) and inline —
+      // one registry fetch per icon ever, so hot recompiles stay LAN/offline friendly.
+      const iconsHeader = await applyAppAssetIcons(entries)
+      if (!entryOverride) {
+        const ambiguous = ambiguousEntryWarning(entries)
+        if (ambiguous) warnings.push(ambiguous)
+      }
+      const js = await compileProject({ entries, name, publicUrl: origin, entryOverride, header: true, fontsHeader, iconsHeader })
+      const changed = bundleJs === null || stripVolatile(js) !== stripVolatile(bundleJs)
+      if (changed) {
+        bundleJs = js
+        v++
+        broadcast(JSON.stringify({ type: "reload", v }))
+      }
+      return { ok: true, changed, v, warnings }
+    } catch (e) {
+      return { ok: false, error: e instanceof Error ? e.message : String(e) }
+    }
+  }
+
+  // Serialized: a recompile requested mid-compile runs after it (compileOnce never rejects).
+  const recompile = (): Promise<RecompileResult> => {
+    const run = compileChain.then(compileOnce)
+    compileChain = run
+    return run
+  }
+
+  // dev.js: the placeholder project, compiled once per server start.
+  const devJs = await compileProject({
+    entries: [{ path: "/.dev-entry.ts", type: "text", text: renderDevScreen(name) }],
+    name: "LeCodes Dev",
+    publicUrl: origin,
+    entryOverride: "/.dev-entry.ts",
+    header: true,
+  })
+
+  // ---- websocket clients -------------------------------------------------------------------------
+  const clients = new Set<WsConnection>()
+  const broadcast = (text: string) => {
+    for (const client of clients) client.send(text)
+  }
+
+  // A device reports stacks against the bundle it evaluated ("module.js:5:51631" on the creator-pkg
+  // hosts) — rebuild the mapper whenever the bundle changes so those frames print as `main.ts:92:23`.
+  let demangleFor: { js: string, map: ((s: string) => string) | null } | null = null
+  const demangle = (text: string): string => {
+    if (bundleJs === null) return text
+    if (demangleFor?.js !== bundleJs) demangleFor = { js: bundleJs, map: makeStackDemangler(bundleJs) }
+    return demangleFor.map ? demangleFor.map(text) : text
+  }
+
+  const printDeviceMessage = (raw: string) => {
+    let m: { type?: string, level?: string, text?: string, v?: number } = {}
+    try { m = JSON.parse(raw) } catch { return }
+    if (m.type === "log") log(`[device:${m.level ?? "log"}] ${demangle(m.text ?? "")}`)
+    else if (m.type === "error") log(`[device:error] ${demangle(m.text ?? "")}`)
+  }
+
+  // ---- http --------------------------------------------------------------------------------------
+  // "web" = the browser runner: no prelude at all (the page brings its own reload + log channel).
+  const transportOf = (url: URL): "ws" | "poll" | "web" => {
+    const t = url.searchParams.get("t")
+    return t === "poll" ? "poll" : t === "web" ? "web" : "ws"
+  }
+  const withPrelude = (js: string, url: URL, v: number) => {
+    const transport = transportOf(url)
+    return transport === "web" ? js : js + "\n" + renderPrelude({ origin, base, transport, v })
+  }
+  const webDir = options.webDir ?? null
+
+  const handle = async (req: IncomingMessage, res: ServerResponse) => {
+    const url = new URL(req.url ?? "/", origin)
+    const path = url.pathname
+    // Browsers ask for it on every page load (the /web/ runner) — keep the console clean.
+    if (path === "/favicon.ico") { res.writeHead(204); res.end(); return }
+    if (!path.startsWith(base + "/")) return sendText(res, 404, "text/plain", "Not found")
+    const route = path.slice(base.length)
+
+    if (route === "/dev.js") {
+      return sendText(res, 200, "text/javascript", withPrelude(devJs, url, -1))
+    }
+    if (route === "/bundle.js") {
+      if (bundleJs === null) return sendText(res, 503, "text/plain", "No successful compile yet — check the terminal.")
+      res.setHeader("x-lecodes-v", String(v))   // the browser runner learns the version it runs
+      return sendText(res, 200, "text/javascript", withPrelude(bundleJs, url, v))
+    }
+    // The browser runner: a static folder with relative asset paths, so it needs the trailing slash.
+    if (route === "/web") {
+      res.writeHead(302, { location: `${base}/web/${url.search}` })
+      res.end()
+      return
+    }
+    if (route.startsWith("/web/")) {
+      if (!webDir) return sendText(res, 404, "text/plain", "No Web Lite runner vendored with this CLI (lecodes-viewer-lite dist-embed).")
+      const rel = route === "/web/" ? "embed.html" : route.slice("/web/".length)
+      if (rel.split("/").some((seg) => seg === ".." || seg === "" || seg === ".")) return sendText(res, 400, "text/plain", "Bad path")
+      const abs = join(webDir, ...rel.split("/"))
+      try {
+        if (!statSync(abs).isFile()) throw new Error("not a file")
+      } catch {
+        return sendText(res, 404, "text/plain", "Not found")
+      }
+      // The entry files (html, embed.js) must revalidate — a CLI update swaps them under the same URL.
+      const hashed = rel.startsWith("assets/")
+      res.writeHead(200, { "content-type": contentType(rel), "cache-control": hashed ? "max-age=31536000, immutable" : "no-store" })
+      createReadStream(abs).pipe(res)
+      return
+    }
+    if (route === "/version") {
+      return sendJson(res, 200, { v })
+    }
+    if (route === "/log" && req.method === "POST") {
+      printDeviceMessage(await readBody(req))
+      return sendJson(res, 200, { ok: true })
+    }
+    if (route.startsWith("/shaders/")) {
+      const name = route.slice("/shaders/".length)
+      if (!isShaderArtifactName(name)) return sendText(res, 400, "text/plain", "Bad path")
+      const abs = join(shadersCacheDir(root), name)
+      try {
+        if (!statSync(abs).isFile()) throw new Error("not a file")
+      } catch {
+        return sendText(res, 404, "text/plain", "Not found")
+      }
+      res.writeHead(200, { "content-type": "application/octet-stream", "cache-control": "no-store" })
+      createReadStream(abs).pipe(res)
+      return
+    }
+    if (route.startsWith("/assets/")) {
+      let key: string
+      try {
+        key = toKey(decodeURIComponent(route.slice("/assets".length)))
+      } catch {
+        return sendText(res, 400, "text/plain", "Bad path")
+      }
+      const abs = resolveLocalPath(root, key)
+      if (!abs || !statSync(abs).isFile()) return sendText(res, 404, "text/plain", "Not found")
+      res.writeHead(200, { "content-type": contentType(key), "cache-control": "no-store" })
+      createReadStream(abs).pipe(res)
+      return
+    }
+    return sendText(res, 404, "text/plain", "Not found")
+  }
+
+  const server: Server = createServer((req, res) => {
+    handle(req, res).catch((e) => {
+      log(`request failed: ${e instanceof Error ? e.message : String(e)}`)
+      if (!res.headersSent) sendText(res, 500, "text/plain", "Internal error")
+    })
+  })
+
+  server.on("upgrade", (req, socket) => {
+    const url = new URL(req.url ?? "/", origin)
+    if (url.pathname !== base + "/ws") {
+      socket.destroy()
+      return
+    }
+    const conn = upgradeWebSocket(req, socket, {
+      onMessage: (text) => {
+        let m: { type?: string, v?: number } = {}
+        try { m = JSON.parse(text) } catch { /* not ours */ }
+        if (m.type === "hello") {
+          // A (re)connecting world announces what it runs; nudge it forward if it's stale.
+          if (typeof m.v === "number" && m.v !== v && bundleJs !== null) {
+            conn?.send(JSON.stringify({ type: "reload", v }))
+          }
+        } else {
+          printDeviceMessage(text)
+        }
+      },
+      onClose: () => { if (conn) clients.delete(conn) },
+    })
+    if (conn) clients.add(conn)
+  })
+
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject)
+    server.listen(options.port, options.bind ?? "0.0.0.0", () => {
+      server.removeListener("error", reject)
+      resolve()
+    })
+  })
+
+  const address = server.address()
+  const boundPort = typeof address === "object" && address !== null ? address.port : options.port
+
+  return {
+    url: `${origin}${base}/dev.js`,
+    webUrl: webDir ? `${origin}${base}/web/` : null,
+    base,
+    port: boundPort,
+    v: () => v,
+    clientCount: () => clients.size,
+    recompile,
+    close: () => {
+      for (const client of clients) client.close()
+      server.close()
+    },
+  }
+}

package/src/dev/webRunner.ts

@@ -0,0 +1,22 @@
+import { existsSync } from "node:fs"
+import { createRequire } from "node:module"
+import { dirname, join } from "node:path"
+import { distRoot } from "../distRoot"
+
+/**
+ * Where the browser runner for `lecodes dev --web` lives: viewer-lite's built `dist-embed`
+ * (embed.html + embed.js + assets/ — the Web Lite host as a static folder). Two shapes:
+ *   - vendored next to the CLI as runtime/web (scripts/vendor-runtime.ts — the npm package and
+ *     the standalone binary),
+ *   - the lecodes-viewer-lite package's own dist-embed (monorepo dev after `bun run build:embed`).
+ * `null` = no runner available (the dev server then serves everything but /web/).
+ */
+export const resolveWebRunnerDir = (): string | null => {
+  const candidates = [join(distRoot, "runtime", "web")]
+  try {
+    const pkg = createRequire(import.meta.url).resolve("lecodes-viewer-lite/package.json")
+    candidates.push(join(dirname(pkg), "dist-embed"))
+  } catch { /* no package: standalone */ }
+  for (const dir of candidates) if (existsSync(join(dir, "embed.html")) && existsSync(join(dir, "embed.js"))) return dir
+  return null
+}

package/src/dev/wsServer.ts

@@ -0,0 +1,144 @@
+import { createHash } from "node:crypto"
+import type { IncomingMessage } from "node:http"
+import type { Duplex } from "node:stream"
+
+/*
+ * A minimal RFC 6455 WebSocket server for the `lecodes dev` reload channel — dependency-free, like
+ * the rest of the CLI. Scope: server side only, text frames (the protocol is small JSON messages),
+ * ping→pong, close handshake. No fragmentation support (continuation frames are dropped) and no
+ * extensions — fine for OkHttp/NSURLSession/WinHTTP clients sending short messages.
+ */
+
+const WS_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
+
+/** Sec-WebSocket-Accept for a client's Sec-WebSocket-Key (RFC 6455 §4.2.2). */
+export const acceptKey = (key: string): string =>
+  createHash("sha1").update(key + WS_GUID).digest("base64")
+
+/** A server→client text frame (FIN, unmasked — servers MUST NOT mask). */
+export const encodeTextFrame = (text: string): Buffer => {
+  const payload = Buffer.from(text, "utf8")
+  const len = payload.length
+  let header: Buffer
+  if (len < 126) {
+    header = Buffer.from([ 0x81, len ])
+  } else if (len < 65536) {
+    header = Buffer.alloc(4)
+    header[0] = 0x81
+    header[1] = 126
+    header.writeUInt16BE(len, 2)
+  } else {
+    header = Buffer.alloc(10)
+    header[0] = 0x81
+    header[1] = 127
+    header.writeBigUInt64BE(BigInt(len), 2)
+  }
+  return Buffer.concat([ header, payload ])
+}
+
+const encodeControlFrame = (opcode: number, payload: Buffer = Buffer.alloc(0)): Buffer =>
+  Buffer.concat([ Buffer.from([ 0x80 | opcode, Math.min(payload.length, 125) ]), payload.subarray(0, 125) ])
+
+export const OPCODE = { CONTINUATION: 0x0, TEXT: 0x1, BINARY: 0x2, CLOSE: 0x8, PING: 0x9, PONG: 0xa } as const
+
+export type WsFrame = { opcode: number, payload: Buffer }
+
+/** Stateful frame decoder: feed raw socket chunks, get complete (unmasked) frames back. */
+export const createFrameDecoder = () => {
+  let buf: Buffer = Buffer.alloc(0)
+  return (chunk: Buffer): WsFrame[] => {
+    buf = buf.length === 0 ? chunk : Buffer.concat([ buf, chunk ])
+    const frames: WsFrame[] = []
+    while (buf.length >= 2) {
+      const opcode = buf[0] & 0x0f
+      const masked = (buf[1] & 0x80) !== 0
+      let len = buf[1] & 0x7f
+      let off = 2
+      if (len === 126) {
+        if (buf.length < 4) break
+        len = buf.readUInt16BE(2)
+        off = 4
+      } else if (len === 127) {
+        if (buf.length < 10) break
+        len = Number(buf.readBigUInt64BE(2))
+        off = 10
+      }
+      const maskLen = masked ? 4 : 0
+      if (buf.length < off + maskLen + len) break
+      let payload = buf.subarray(off + maskLen, off + maskLen + len)
+      if (masked) {
+        const mask = buf.subarray(off, off + 4)
+        const unmasked = Buffer.alloc(len)
+        for (let i = 0; i < len; i++) unmasked[i] = payload[i] ^ mask[i & 3]
+        payload = unmasked
+      }
+      buf = buf.subarray(off + maskLen + len)
+      frames.push({ opcode, payload })
+    }
+    return frames
+  }
+}
+
+export type WsConnection = {
+  send: (text: string) => void
+  close: () => void
+}
+
+export type WsHandlers = {
+  onMessage: (text: string) => void
+  /** Fired exactly once, however the connection ends. */
+  onClose: () => void
+}
+
+/**
+ * Complete the upgrade handshake on an `http.Server` `'upgrade'` event and speak WebSocket on the
+ * raw socket. Returns null (socket destroyed) if the request isn't a valid upgrade.
+ */
+export const upgradeWebSocket = (req: IncomingMessage, socket: Duplex, handlers: WsHandlers): WsConnection | null => {
+  const key = req.headers["sec-websocket-key"]
+  if (typeof key !== "string" || (req.headers.upgrade ?? "").toLowerCase() !== "websocket") {
+    socket.destroy()
+    return null
+  }
+  socket.write(
+    "HTTP/1.1 101 Switching Protocols\r\n" +
+    "Upgrade: websocket\r\n" +
+    "Connection: Upgrade\r\n" +
+    `Sec-WebSocket-Accept: ${acceptKey(key)}\r\n\r\n`,
+  )
+
+  let open = true
+  const finish = () => {
+    if (!open) return
+    open = false
+    handlers.onClose()
+  }
+  const decode = createFrameDecoder()
+  socket.on("data", (chunk: Buffer) => {
+    for (const frame of decode(chunk)) {
+      if (frame.opcode === OPCODE.TEXT) {
+        handlers.onMessage(frame.payload.toString("utf8"))
+      } else if (frame.opcode === OPCODE.PING) {
+        if (open) socket.write(encodeControlFrame(OPCODE.PONG, frame.payload))
+      } else if (frame.opcode === OPCODE.CLOSE) {
+        if (open) socket.write(encodeControlFrame(OPCODE.CLOSE))
+        socket.end()
+        finish()
+      }
+    }
+  })
+  socket.on("close", finish)
+  socket.on("error", () => socket.destroy())
+
+  return {
+    send: (text: string) => {
+      if (open) socket.write(encodeTextFrame(text))
+    },
+    close: () => {
+      if (!open) return
+      socket.write(encodeControlFrame(OPCODE.CLOSE))
+      socket.end()
+      finish()
+    },
+  }
+}

package/src/distRoot.ts

@@ -0,0 +1,20 @@
+import { existsSync } from "node:fs"
+import { dirname, join } from "node:path"
+import { fileURLToPath } from "node:url"
+
+/* Root of the CLI's on-disk companion files (runtime/ with the vendored SDK source + type bundle).
+ *
+ * Two distribution shapes:
+ *  - npm package / monorepo dev: this module lives in dist/ (or src/), companions at ../runtime —
+ *    resolve relative to import.meta.url as before.
+ *  - standalone binary (`bun build --compile`, see scripts/build-bin.ts): import.meta.url points into
+ *    the virtual bunfs archive which does NOT exist on disk, so the companions ship in the release
+ *    archive NEXT TO the executable and are resolved via process.execPath. */
+export const distRoot = (() => {
+  const candidates = [
+    join(dirname(fileURLToPath(import.meta.url)), ".."), // package root (dist/.. or src/..)
+    dirname(process.execPath),                           // standalone binary: next to the exe
+  ]
+  for (const c of candidates) if (existsSync(join(c, "runtime"))) return c
+  return candidates[0]
+})()