dsh-hyperframes 0.3.2 → 0.4.0

package/skills/media-use/scripts/lib/specs.mjs

@@ -1,132 +0,0 @@
-// Machine-capability probe for the spec-gated local-model fallback.
-//
-// Local models are USER-INSTALLED and local-use-only — media-use recommends,
-// spec-checks, and assists install, but never bundles or runs them as a service.
-// This probe answers "what tier can this machine actually run?" so selection can
-// offer a medium/large local model, or fall back to recommending the CLI path.
-//
-// `osMod` and `exec` are injectable for tests. `exec(cmd)` returns the command's
-// stdout as a string, or throws / returns null on failure.
-
-import os from "node:os";
-import { statfsSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { execSync } from "node:child_process";
-
-function defaultExec(cmd) {
-  return execSync(cmd, { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 4000 });
-}
-
-// Available (not just total) RAM — the real budget for "will this model fit
-// alongside the OS + open apps". On unified-memory Macs the model shares system
-// RAM, so a big model on a busy machine OOMs/thrashes even if total RAM looks
-// ample. macOS: vm_stat free + inactive + speculative + purgeable pages (all
-// reclaimable). Linux: /proc/meminfo MemAvailable. Fallback: total (optimistic
-// but stable when the probe is unavailable, e.g. in tests).
-function availableRamMB(platform, exec, totalMB) {
-  try {
-    if (platform === "darwin") {
-      const out = String(exec("vm_stat"));
-      const pageSize = parseInt((out.match(/page size of (\d+)/) || [])[1] || "16384", 10);
-      const pages = (name) => {
-        const m = out.match(new RegExp(`${name}:\\s+(\\d+)`));
-        return m ? parseInt(m[1], 10) : 0;
-      };
-      const reclaimable =
-        pages("Pages free") +
-        pages("Pages inactive") +
-        pages("Pages speculative") +
-        pages("Pages purgeable");
-      const mb = Math.round((reclaimable * pageSize) / (1024 * 1024));
-      if (mb > 0) return mb;
-    } else if (platform === "linux") {
-      const out = String(exec("cat /proc/meminfo"));
-      const m = out.match(/MemAvailable:\s+(\d+)\s+kB/);
-      if (m) return Math.round(parseInt(m[1], 10) / 1024);
-    }
-  } catch {
-    // probe unavailable — fall through to the total-RAM estimate
-  }
-  return totalMB;
-}
-
-function detectGpu(platform, arch, ramMB, exec) {
-  // Apple Silicon: Metal GPU with unified memory — VRAM tracks system RAM.
-  if (platform === "darwin" && arch === "arm64") {
-    return { present: true, kind: "apple", vramMB: ramMB };
-  }
-  // NVIDIA: query total VRAM. Any failure (no driver, no GPU) -> no GPU.
-  try {
-    const out = exec("nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits");
-    const mb = parseInt(String(out).trim().split(/\r?\n/)[0], 10);
-    if (Number.isFinite(mb) && mb > 0) return { present: true, kind: "nvidia", vramMB: mb };
-  } catch {
-    // fall through — no usable GPU
-  }
-  return { present: false, kind: null, vramMB: 0 };
-}
-
-export function probeSpecs({ osMod = os, exec = defaultExec } = {}) {
-  const platform = osMod.platform();
-  const arch = osMod.arch();
-  const cpuCores = osMod.cpus().length;
-  const ramMB = Math.round(osMod.totalmem() / (1024 * 1024));
-  return {
-    platform,
-    arch,
-    cpuCores,
-    ramMB,
-    availableRamMB: availableRamMB(platform, exec, ramMB),
-    appleSilicon: platform === "darwin" && arch === "arm64",
-    gpu: detectGpu(platform, arch, ramMB, exec),
-  };
-}
-
-// Where Hugging Face actually puts downloaded weights. A free-space check
-// against cwd measures the wrong filesystem, so the disk question has to be
-// asked about this directory. Precedence follows huggingface_hub's own order.
-export function weightsCacheDir({ env = process.env, osMod = os } = {}) {
-  if (env.HF_HUB_CACHE) return env.HF_HUB_CACHE;
-  if (env.HUGGINGFACE_HUB_CACHE) return env.HUGGINGFACE_HUB_CACHE;
-  if (env.HF_HOME) return join(env.HF_HOME, "hub");
-  return join(osMod.homedir(), ".cache", "huggingface", "hub");
-}
-
-// Free space on the filesystem that will hold the weights. The cache dir
-// usually does not exist until the first download and statfs throws on a
-// missing path, so walk up to the deepest ancestor that does exist. Returns
-// null when even the root cannot be read, so callers can say "unknown" instead
-// of implying zero and scaring someone off a download that would have worked.
-export function freeSpaceMB(dir, statfsFn = statfsSync) {
-  let path = dir;
-  for (;;) {
-    try {
-      const { bavail, bsize } = statfsFn(path);
-      return (bavail * bsize) / 1e6;
-    } catch {
-      const parent = dirname(path);
-      if (parent === path) return null;
-      path = parent;
-    }
-  }
-}
-
-// One line the user reads BEFORE agreeing to a pull that can be tens of GB.
-// Always names the size and where it lands. When it will not fit we say so
-// plainly rather than withholding the tier: a machine that could free up space
-// should still be told the tier exists. `statfsFn` / `env` / `osMod` are
-// injectable for tests.
-export function describeDownload(
-  sizeMB,
-  { statfsFn = statfsSync, env = process.env, osMod = os } = {},
-) {
-  const dir = weightsCacheDir({ env, osMod });
-  const gb = (mb) => (mb / 1000).toFixed(1);
-  const head = `downloads ~${gb(sizeMB)}GB of weights to ${dir}`;
-  const free = freeSpaceMB(dir, statfsFn);
-  if (free == null) return `${head} (free space unknown)`;
-  if (free < sizeMB) {
-    return `${head}, but only ${gb(free)}GB is free there, so it will NOT fit as-is`;
-  }
-  return `${head} (${gb(free)}GB free there)`;
-}

package/skills/media-use/scripts/lib/specs.test.mjs

@@ -1,157 +0,0 @@
-import { strict as assert } from "node:assert";
-import { test } from "node:test";
-import { describeDownload, freeSpaceMB, probeSpecs, weightsCacheDir } from "./specs.mjs";
-
-// Fake os module + exec so the probe is deterministic across CI machines.
-const fakeOs = (over = {}) => ({
-  platform: () => over.platform ?? "linux",
-  arch: () => over.arch ?? "x64",
-  cpus: () => Array.from({ length: over.cores ?? 8 }),
-  totalmem: () => (over.ramMB ?? 16384) * 1024 * 1024,
-});
-
-test("probeSpecs reports structured caps", () => {
-  const s = probeSpecs({ osMod: fakeOs({ cores: 12, ramMB: 32768 }), exec: () => null });
-  assert.equal(s.cpuCores, 12);
-  assert.equal(s.ramMB, 32768);
-  assert.equal(s.platform, "linux");
-  assert.equal(s.gpu.present, false);
-  // probe unavailable (exec returns null) -> availableRamMB falls back to total
-  assert.equal(s.availableRamMB, 32768);
-});
-
-test("availableRamMB is read from /proc/meminfo on Linux", () => {
-  const exec = (cmd) =>
-    cmd.includes("meminfo") ? "MemTotal: 33554432 kB\nMemAvailable: 8388608 kB\n" : null;
-  const s = probeSpecs({ osMod: fakeOs({ platform: "linux", ramMB: 32768 }), exec });
-  assert.equal(s.availableRamMB, 8192, "8388608 kB -> 8192 MB");
-});
-
-test("availableRamMB is summed from reclaimable vm_stat pages on macOS", () => {
-  const vmStat =
-    "Mach Virtual Memory Statistics: (page size of 16384 bytes)\n" +
-    "Pages free:                    100000.\n" +
-    "Pages inactive:                200000.\n" +
-    "Pages speculative:              50000.\n" +
-    "Pages purgeable:                 10000.\n";
-  const exec = (cmd) => (cmd.includes("vm_stat") ? vmStat : null);
-  const s = probeSpecs({
-    osMod: fakeOs({ platform: "darwin", arch: "arm64", ramMB: 24576 }),
-    exec,
-  });
-  // (100000+200000+50000+10000) pages * 16384 B / 1MiB = 5625 MB
-  assert.equal(s.availableRamMB, 5625);
-});
-
-test("Apple Silicon is detected as a unified-memory GPU", () => {
-  const s = probeSpecs({
-    osMod: fakeOs({ platform: "darwin", arch: "arm64", ramMB: 24576 }),
-    exec: () => null,
-  });
-  assert.equal(s.appleSilicon, true);
-  assert.equal(s.gpu.present, true);
-  assert.equal(s.gpu.kind, "apple");
-  // unified memory: VRAM tracks system RAM
-  assert.equal(s.gpu.vramMB, 24576);
-});
-
-test("NVIDIA GPU is detected via nvidia-smi VRAM query", () => {
-  const exec = (cmd) => (cmd.includes("nvidia-smi") ? "24564" : null);
-  const s = probeSpecs({ osMod: fakeOs({ platform: "linux" }), exec });
-  assert.equal(s.gpu.present, true);
-  assert.equal(s.gpu.kind, "nvidia");
-  assert.equal(s.gpu.vramMB, 24564);
-});
-
-test("no GPU when nvidia-smi is absent / fails", () => {
-  const s = probeSpecs({
-    osMod: fakeOs({ platform: "linux" }),
-    exec: () => {
-      throw new Error("command not found");
-    },
-  });
-  assert.equal(s.gpu.present, false);
-  assert.equal(s.gpu.vramMB, 0);
-});
-
-// --- download disclosure: what the user is agreeing to before the pull ---
-
-const fakeHome = { homedir: () => "/home/tester" };
-// statfs reports blocks, not bytes: bavail * bsize. 1 MB blocks keep the sums
-// readable, and mirror the real struct's shape.
-const fakeStatfs = (freeMB, existsOnly) => (path) => {
-  if (existsOnly && path !== existsOnly) throw new Error(`ENOENT: ${path}`);
-  return { bavail: freeMB, bsize: 1e6 };
-};
-
-test("weightsCacheDir follows huggingface_hub's precedence", () => {
-  assert.equal(
-    weightsCacheDir({ env: {}, osMod: fakeHome }),
-    "/home/tester/.cache/huggingface/hub",
-  );
-  assert.equal(weightsCacheDir({ env: { HF_HOME: "/data/hf" }, osMod: fakeHome }), "/data/hf/hub");
-  assert.equal(
-    weightsCacheDir({ env: { HF_HOME: "/data/hf", HUGGINGFACE_HUB_CACHE: "/c" }, osMod: fakeHome }),
-    "/c",
-    "HUGGINGFACE_HUB_CACHE outranks HF_HOME",
-  );
-  assert.equal(
-    weightsCacheDir({ env: { HF_HUB_CACHE: "/a", HUGGINGFACE_HUB_CACHE: "/c" }, osMod: fakeHome }),
-    "/a",
-    "HF_HUB_CACHE wins outright",
-  );
-});
-
-test("freeSpaceMB walks up to the deepest existing ancestor", () => {
-  // the cache dir does not exist until the first download, and statfs throws
-  // on a missing path - so the answer has to come from an ancestor
-  const statfs = fakeStatfs(4096, "/home");
-  assert.equal(freeSpaceMB("/home/tester/.cache/huggingface/hub", statfs), 4096);
-});
-
-test("freeSpaceMB reports null rather than zero when nothing can be read", () => {
-  assert.equal(
-    freeSpaceMB("/home/tester/.cache", () => {
-      throw new Error("EACCES");
-    }),
-    null,
-    "unknown must not be reported as no-space",
-  );
-});
-
-test("describeDownload names the size and where it lands", () => {
-  const msg = describeDownload(87500, {
-    statfsFn: fakeStatfs(200000),
-    env: {},
-    osMod: fakeHome,
-  });
-  assert.match(msg, /~87\.5GB/);
-  assert.match(msg, /\/home\/tester\/\.cache\/huggingface\/hub/);
-  assert.match(msg, /200\.0GB free/);
-  assert.equal(/NOT fit/.test(msg), false, "it fits, so no warning");
-});
-
-test("describeDownload says plainly when the weights will not fit", () => {
-  const msg = describeDownload(87500, {
-    statfsFn: fakeStatfs(14000),
-    env: {},
-    osMod: fakeHome,
-  });
-  assert.match(msg, /only 14\.0GB is free there/);
-  assert.match(msg, /will NOT fit as-is/);
-  // the tier is still described, not withheld: a machine that could free up
-  // space should know the tier exists
-  assert.match(msg, /~87\.5GB/);
-});
-
-test("describeDownload admits when free space is unknown", () => {
-  const msg = describeDownload(87500, {
-    statfsFn: () => {
-      throw new Error("EACCES");
-    },
-    env: {},
-    osMod: fakeHome,
-  });
-  assert.match(msg, /free space unknown/);
-  assert.equal(/NOT fit/.test(msg), false, "unknown is not a refusal");
-});

package/skills/media-use/scripts/lib/stats.mjs

@@ -1,117 +0,0 @@
-import { statSync } from "node:fs";
-import { readGlobalManifest } from "./cache.mjs";
-import { readManifest } from "./manifest.mjs";
-import { readMisses } from "./misses.mjs";
-
-const TOP_MISSES = 5;
-
-function emptyReport() {
-  return {
-    total_resolves: 0,
-    by_type: {},
-    by_source: {},
-    by_provider: {},
-    by_via: {},
-    misses: 0,
-    hit_rate: null,
-    top_missed_intents: {},
-    global_cache_assets: 0,
-    global_cache_disk_bytes: 0,
-    cross_project_reuse: 0,
-  };
-}
-
-function increment(map, key) {
-  if (!key) return;
-  map[key] = (map[key] || 0) + 1;
-}
-
-function timestampOf(record) {
-  return record?.ts || record?.timestamp || record?.created_at || record?.createdAt || null;
-}
-
-function inWindow(record, cutoff) {
-  if (!cutoff) return true;
-  const ts = timestampOf(record);
-  // Older manifest records may not carry a timestamp; keep them in the report
-  // because --days can only window records/misses that carry a ts/timestamp.
-  if (!ts) return true;
-  const time = Date.parse(ts);
-  return Number.isNaN(time) ? true : time >= cutoff;
-}
-
-function sourceOf(record) {
-  return record?._source || record?.source || record?.provenance?.source || "unknown";
-}
-
-function normalizeIntent(intent) {
-  return String(intent ?? "")
-    .trim()
-    .toLowerCase()
-    .replace(/\s+/g, " ");
-}
-
-function topMissedIntents(misses) {
-  const grouped = {};
-  for (const miss of misses) {
-    const type = miss?.type || "unknown";
-    const intent = normalizeIntent(miss?.intent);
-    if (!intent) continue;
-    grouped[type] ||= {};
-    grouped[type][intent] = (grouped[type][intent] || 0) + 1;
-  }
-  const out = {};
-  for (const [type, intents] of Object.entries(grouped)) {
-    out[type] = Object.entries(intents)
-      .map(([intent, count]) => ({ intent, count }))
-      .sort((a, b) => b.count - a.count || a.intent.localeCompare(b.intent))
-      .slice(0, TOP_MISSES);
-  }
-  return out;
-}
-
-function diskBytes(records) {
-  let total = 0;
-  for (const record of records) {
-    const p = record?.cached_path || record?.path;
-    if (!p) continue;
-    try {
-      total += statSync(p).size;
-    } catch {
-      // cache entries can outlive files; stats skips missing files
-    }
-  }
-  return total;
-}
-
-export function buildStats({ projectDir, days, now = Date.now() } = {}) {
-  // Only a positive finite --days windows the report; null / NaN / <= 0 mean
-  // "all time" rather than silently excluding everything (a negative cutoff
-  // would land in the future and drop every record). The reads below are each
-  // best-effort (they return [] / skip on IO errors), so there is no top-level
-  // catch masking a real logic bug as an all-zero "no usage" report.
-  const n = Number(days);
-  const cutoff = Number.isFinite(n) && n > 0 ? Number(now) - n * 24 * 60 * 60 * 1000 : null;
-  const records = (projectDir ? readManifest(projectDir) : []).filter((r) => inWindow(r, cutoff));
-  const misses = readMisses().filter((miss) => inWindow(miss, cutoff));
-  const globalRecords = readGlobalManifest();
-  const report = emptyReport();
-
-  report.total_resolves = records.length;
-  report.misses = misses.length;
-  for (const record of records) {
-    increment(report.by_type, record?.type || "unknown");
-    increment(report.by_source, sourceOf(record));
-    increment(report.by_provider, record?.provenance?.provider);
-    increment(report.by_via, record?.provenance?.via);
-  }
-
-  const attempts = report.total_resolves + report.misses;
-  report.hit_rate = attempts === 0 ? null : report.total_resolves / attempts;
-  report.top_missed_intents = topMissedIntents(misses);
-  report.global_cache_assets = globalRecords.length;
-  report.global_cache_disk_bytes = diskBytes(globalRecords);
-  report.cross_project_reuse = globalRecords.filter((r) => r?.provenance?.reused_by).length;
-
-  return report;
-}

package/skills/media-use/scripts/lib/stats.test.mjs

@@ -1,165 +0,0 @@
-import { strict as assert } from "node:assert";
-import { test } from "node:test";
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { buildStats } from "./stats.mjs";
-
-function sandbox() {
-  const root = mkdtempSync(join(tmpdir(), "mu-stats-"));
-  const home = join(root, "home");
-  const projectDir = join(root, "project");
-  mkdirSync(home, { recursive: true });
-  mkdirSync(projectDir, { recursive: true });
-  process.env.HOME = home;
-  return { root, home, projectDir };
-}
-
-function restoreEnv(saved) {
-  for (const k of Object.keys(process.env)) if (!(k in saved)) delete process.env[k];
-  Object.assign(process.env, saved);
-}
-
-function seedManifest(dir, records) {
-  mkdirSync(join(dir, ".media"), { recursive: true });
-  writeFileSync(
-    join(dir, ".media/manifest.jsonl"),
-    records.map((record) => JSON.stringify(record)).join("\n") + "\n",
-  );
-}
-
-function isoDaysAgo(days) {
-  return new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString();
-}
-
-test("buildStats aggregates resolves, misses, providers, sources, and reuse", () => {
-  const savedEnv = { ...process.env };
-  const { root, home, projectDir } = sandbox();
-  try {
-    seedManifest(projectDir, [
-      {
-        id: "bgm_001",
-        type: "bgm",
-        source: "search",
-        ts: isoDaysAgo(0),
-        provenance: { provider: "heygen.audio.sounds", prompt: "upbeat", via: "url" },
-      },
-      {
-        id: "image_001",
-        type: "image",
-        _source: "reused-explicit",
-        timestamp: isoDaysAgo(0),
-        provenance: { provider: "local", reused_by: "agent" },
-      },
-      {
-        id: "bgm_002",
-        type: "bgm",
-        source: "generated",
-        provenance: { provider: "codex" },
-      },
-    ]);
-    const cachedFile = join(home, ".media/mu-v1-cache/asset.wav");
-    mkdirSync(join(cachedFile, ".."), { recursive: true });
-    writeFileSync(cachedFile, "12345");
-    seedManifest(home, [
-      {
-        id: "bgm_009",
-        type: "bgm",
-        reusable: true,
-        cached_path: cachedFile,
-        provenance: { provider: "heygen.audio.sounds", reused_by: "agent" },
-      },
-    ]);
-    writeFileSync(
-      join(home, ".media/misses.jsonl"),
-      JSON.stringify({
-        ts: isoDaysAgo(0),
-        type: "bgm",
-        intent: "dark cinematic riser",
-      }) + "\n",
-      { flag: "a" },
-    );
-
-    const stats = buildStats({ projectDir });
-    assert.equal(stats.total_resolves, 3);
-    assert.deepEqual(stats.by_type, { bgm: 2, image: 1 });
-    assert.deepEqual(stats.by_source, { search: 1, "reused-explicit": 1, generated: 1 });
-    assert.equal(stats.by_provider["heygen.audio.sounds"], 1);
-    assert.equal(stats.by_via.url, 1);
-    assert.equal(stats.misses, 1);
-    assert.equal(stats.hit_rate, 0.75);
-    assert.deepEqual(stats.top_missed_intents.bgm[0], {
-      intent: "dark cinematic riser",
-      count: 1,
-    });
-    assert.equal(stats.global_cache_assets, 1);
-    assert.equal(stats.global_cache_disk_bytes, 5);
-    assert.equal(stats.cross_project_reuse, 1);
-  } finally {
-    restoreEnv(savedEnv);
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-test("buildStats returns a zeroed report when nothing exists", () => {
-  const savedEnv = { ...process.env };
-  const { root, projectDir } = sandbox();
-  try {
-    const stats = buildStats({ projectDir });
-    assert.equal(stats.total_resolves, 0);
-    assert.deepEqual(stats.by_type, {});
-    assert.deepEqual(stats.by_source, {});
-    assert.deepEqual(stats.by_provider, {});
-    assert.deepEqual(stats.by_via, {});
-    assert.equal(stats.misses, 0);
-    assert.equal(stats.hit_rate, null);
-    assert.deepEqual(stats.top_missed_intents, {});
-    assert.equal(stats.global_cache_assets, 0);
-    assert.equal(stats.global_cache_disk_bytes, 0);
-    assert.equal(stats.cross_project_reuse, 0);
-  } finally {
-    restoreEnv(savedEnv);
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-test("buildStats returns JSON-safe output", () => {
-  const savedEnv = { ...process.env };
-  const { root, projectDir } = sandbox();
-  try {
-    const stats = buildStats({ projectDir });
-    assert.deepEqual(JSON.parse(JSON.stringify(stats)), stats);
-  } finally {
-    restoreEnv(savedEnv);
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-test("buildStats applies days window to timestamped records and misses", () => {
-  const savedEnv = { ...process.env };
-  const { root, home, projectDir } = sandbox();
-  try {
-    seedManifest(projectDir, [
-      { id: "bgm_old", type: "bgm", ts: isoDaysAgo(100), provenance: { provider: "old" } },
-      { id: "bgm_new", type: "bgm", ts: isoDaysAgo(1), provenance: { provider: "new" } },
-      { id: "image_untimed", type: "image", provenance: { provider: "none" } },
-    ]);
-    mkdirSync(join(home, ".media"), { recursive: true });
-    writeFileSync(
-      join(home, ".media/misses.jsonl"),
-      [
-        JSON.stringify({ ts: isoDaysAgo(100), type: "bgm", intent: "old miss" }),
-        JSON.stringify({ ts: isoDaysAgo(1), type: "bgm", intent: "new miss" }),
-      ].join("\n"),
-    );
-
-    const stats = buildStats({ projectDir, days: 7 });
-    assert.equal(stats.total_resolves, 2);
-    assert.deepEqual(stats.by_type, { bgm: 1, image: 1 });
-    assert.equal(stats.misses, 1);
-    assert.equal(stats.top_missed_intents.bgm[0].intent, "new miss");
-  } finally {
-    restoreEnv(savedEnv);
-    rmSync(root, { recursive: true, force: true });
-  }
-});