hyperframes 0.8.58 → 0.8.60

package/dist/skills/registry/localSemantic.js

@@ -0,0 +1,548 @@
+import { createRequire as __hf_createRequire } from "node:module";
+import { fileURLToPath as __hf_fileURLToPath } from "node:url";
+import { dirname as __hf_dirname } from "node:path";
+var require = __hf_createRequire(import.meta.url);
+var __filename = __hf_fileURLToPath(import.meta.url);
+var __dirname = __hf_dirname(__filename);
+
+// src/registry/localSemantic.ts
+import { existsSync as existsSync4, mkdirSync as mkdirSync4, readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
+import { homedir as homedir4 } from "os";
+import { join as join4 } from "path";
+
+// src/registry/localEmbedder.ts
+import { readFileSync as readFileSync4 } from "fs";
+
+// src/utils/optionalPackages.ts
+import { spawn } from "child_process";
+import { randomUUID } from "crypto";
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  writeFileSync
+} from "fs";
+import { createRequire } from "module";
+import { homedir } from "os";
+import { basename, dirname, join } from "path";
+
+// src/utils/npxCommand.ts
+function buildCmdShimCommand(name, args, platform) {
+  if (platform === "win32") {
+    return { command: "cmd.exe", args: ["/d", "/s", "/c", `${name}.cmd`, ...args] };
+  }
+  return { command: name, args: [...args] };
+}
+function buildNpmCommand(args, platform = process.platform) {
+  return buildCmdShimCommand("npm", args, platform);
+}
+
+// src/utils/optionalPackages.ts
+var OPTIONAL_PACKAGES = {
+  "onnxruntime-node": "1.21.1",
+  "@google/genai": "1.52.0"
+};
+var CACHE_DIR = join(homedir(), ".cache", "hyperframes", "optional");
+function optionalPackageDir(name, cacheDir = CACHE_DIR) {
+  return join(cacheDir, `${name.replace("/", "__")}@${OPTIONAL_PACKAGES[name]}`);
+}
+async function loadOptionalPackage(name, feature, deps = defaultDeps) {
+  const dir = optionalPackageDir(name, deps.cacheDir);
+  const installed = deps.loadInstalled(dir, name);
+  if (installed !== null) return installed;
+  const version = OPTIONAL_PACKAGES[name];
+  deps.log(`installing ${name} for ${feature}, once`);
+  try {
+    await deps.install(dir, name, version);
+  } catch (err) {
+    const output = err.message.trim();
+    const advice = NETWORK_FAILURE.test(output) ? "Check your network connection, then retry" : "Retry";
+    throw new Error(
+      `${feature} needs ${name}, and installing it failed (${output.split("\n")[0]}). ${advice}, or install it yourself: npm install ${name}@${version} --prefix "${dir}"`
+    );
+  }
+  const loaded = deps.loadInstalled(dir, name);
+  if (loaded === null) {
+    throw new Error(`${name}@${version} installed into ${dir} but could not be loaded.`);
+  }
+  return loaded;
+}
+var NETWORK_FAILURE = /ENOTFOUND|ETIMEDOUT|EAI_AGAIN|ECONNRESET|ECONNREFUSED|network/i;
+function manifestPath(dir, name) {
+  return join(dir, "node_modules", name, "package.json");
+}
+function isInstalled(dir, name) {
+  return existsSync(manifestPath(dir, name));
+}
+function loadInstalled(dir, name) {
+  if (!isInstalled(dir, name)) return null;
+  return createRequire(join(dir, "package.json"))(name);
+}
+function runNpm(args) {
+  const npm = buildNpmCommand(args);
+  const child = spawn(npm.command, npm.args, { stdio: ["ignore", "pipe", "pipe"] });
+  return new Promise((resolve, reject) => {
+    let output = "";
+    child.stdout.on("data", (chunk) => output += chunk);
+    child.stderr.on("data", (chunk) => output += chunk);
+    child.on("error", reject);
+    child.on(
+      "close",
+      (code) => code === 0 ? resolve() : reject(new Error(output || `npm exited with code ${code}`))
+    );
+  });
+}
+function isProcessAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    return err.code !== "ESRCH";
+  }
+}
+function sweepStaleStaging(dir) {
+  const prefix = `${basename(dir)}.tmp-`;
+  const parent = dirname(dir);
+  if (!existsSync(parent)) return;
+  for (const entry of readdirSync(parent)) {
+    if (!entry.startsWith(prefix)) continue;
+    const pid = /^(\d+)(?:-|$)/.exec(entry.slice(prefix.length))?.[1];
+    if (pid === void 0 || isProcessAlive(Number(pid))) continue;
+    const stale = join(parent, entry);
+    try {
+      rmSync(stale, { recursive: true, force: true });
+    } catch (err) {
+      console.error(`could not remove stale install dir ${stale}: ${err.message}`);
+    }
+  }
+}
+async function install(dir, name, version, run = runNpm) {
+  sweepStaleStaging(dir);
+  const staging = `${dir}.tmp-${process.pid}-${randomUUID().slice(0, 8)}`;
+  mkdirSync(staging, { recursive: true });
+  writeFileSync(join(staging, "package.json"), "{}");
+  try {
+    await run([
+      "install",
+      `${name}@${version}`,
+      "--prefix",
+      staging,
+      "--no-audit",
+      "--no-fund",
+      "--loglevel=error",
+      // A first-use install is interactive, not a CI resolve: fail fast and name the
+      // manual command instead of sitting through npm's default multi-minute backoff.
+      "--fetch-retries=0",
+      "--fetch-timeout=20000"
+    ]);
+    if (isInstalled(dir, name)) return;
+    rmSync(dir, { recursive: true, force: true });
+    renameSync(staging, dir);
+  } catch (err) {
+    if (!isInstalled(dir, name)) throw err;
+  } finally {
+    rmSync(staging, { recursive: true, force: true });
+  }
+}
+var defaultDeps = {
+  cacheDir: CACHE_DIR,
+  loadInstalled,
+  install,
+  log: (line) => console.error(line)
+};
+
+// src/registry/localModel.ts
+import { createHash } from "crypto";
+import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync3, unlinkSync as unlinkSync2 } from "fs";
+import { homedir as homedir3 } from "os";
+import { join as join3 } from "path";
+
+// src/utils/download.ts
+import { randomUUID as randomUUID2 } from "crypto";
+import { createWriteStream, renameSync as renameSync2, unlinkSync } from "fs";
+import { get as httpsGet } from "https";
+import { Transform } from "stream";
+import { pipeline } from "stream/promises";
+
+// src/telemetry/config.ts
+import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, renameSync as renameSync3, rmSync as rmSync2, writeFileSync as writeFileSync2 } from "fs";
+import { join as join2 } from "path";
+import { homedir as homedir2 } from "os";
+import { randomUUID as randomUUID3 } from "crypto";
+var CONFIG_DIR = join2(homedir2(), ".hyperframes");
+var CONFIG_FILE = join2(CONFIG_DIR, "config.json");
+var STATE_FILE = join2(CONFIG_DIR, "install-state.json");
+var LEGACY_STATE_FILE = join2(homedir2(), ".local", "state", "hyperframes", "install-state.json");
+
+// src/registry/localModel.ts
+var LOCAL_MODEL_ID = "bge-small-en-v1.5";
+var LOCAL_MODEL_DIMENSIONS = 384;
+var QUERY_INSTRUCTION = "Represent this sentence for searching relevant passages: ";
+var MODELS_DIR = join3(homedir3(), ".hyperframes", "models");
+var MODEL_REPO = "Xenova/bge-small-en-v1.5";
+var LOCAL_MODEL_REVISION = "ea104dacec62c0de699686887e3f920caeb4f3e3";
+var LOCAL_MODEL_ARTIFACTS = [
+  {
+    url: `https://huggingface.co/${MODEL_REPO}/resolve/${LOCAL_MODEL_REVISION}/onnx/model_quantized.onnx`,
+    dest: () => localModelPath(),
+    bytes: 34014426,
+    sha256: "6c9c6101a956d62dfb5e7190c538226c0c5bb9cb27b651234b6df063ee7dbfe4"
+  },
+  {
+    url: `https://huggingface.co/${MODEL_REPO}/resolve/${LOCAL_MODEL_REVISION}/tokenizer.json`,
+    dest: () => localTokenizerPath(),
+    bytes: 711396,
+    sha256: "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66"
+  }
+];
+function localModelPath() {
+  return join3(MODELS_DIR, `${LOCAL_MODEL_ID}.onnx`);
+}
+function localTokenizerPath() {
+  return join3(MODELS_DIR, `${LOCAL_MODEL_ID}.tokenizer.json`);
+}
+function modelFileIsValid(file) {
+  const path = file.dest();
+  if (!existsSync3(path)) return false;
+  try {
+    return createHash("sha256").update(readFileSync3(path)).digest("hex") === file.sha256;
+  } catch {
+    return false;
+  }
+}
+function isLocalModelReady() {
+  return LOCAL_MODEL_ARTIFACTS.every(modelFileIsValid);
+}
+
+// src/registry/wordpiece.ts
+function configFromTokenizerJson(raw) {
+  const parsed = JSON.parse(raw);
+  const model = parsed.model;
+  if (!model?.vocab) throw new Error("tokenizer.json has no WordPiece vocab");
+  return {
+    vocab: model.vocab,
+    unkToken: model.unk_token ?? "[UNK]",
+    continuingSubwordPrefix: model.continuing_subword_prefix ?? "##",
+    maxInputCharsPerWord: model.max_input_chars_per_word ?? 100,
+    clsToken: "[CLS]",
+    sepToken: "[SEP]"
+  };
+}
+function normalize(text) {
+  let out = "";
+  for (const char of text) {
+    const code = char.codePointAt(0);
+    if (code === 0 || code === 65533) continue;
+    if (isControl(char, code)) continue;
+    if (isWhitespace(char, code)) {
+      out += " ";
+      continue;
+    }
+    out += isChinese(code) ? ` ${char} ` : char;
+  }
+  return out.toLowerCase().normalize("NFD").replace(new RegExp("\\p{Mn}", "gu"), "");
+}
+function preTokenize(normalized) {
+  const words = [];
+  for (const chunk of normalized.split(/\s+/)) {
+    if (!chunk) continue;
+    let current = "";
+    for (const char of chunk) {
+      if (isPunctuation(char)) {
+        if (current) words.push(current);
+        words.push(char);
+        current = "";
+      } else {
+        current += char;
+      }
+    }
+    if (current) words.push(current);
+  }
+  return words;
+}
+function wordPiece(word, config) {
+  if ([...word].length > config.maxInputCharsPerWord) return [config.unkToken];
+  const pieces = [];
+  let start = 0;
+  while (start < word.length) {
+    let end = word.length;
+    let match = null;
+    while (start < end) {
+      const candidate = start === 0 ? word.slice(start, end) : config.continuingSubwordPrefix + word.slice(start, end);
+      if (candidate in config.vocab) {
+        match = candidate;
+        break;
+      }
+      end -= 1;
+    }
+    if (match === null) return [config.unkToken];
+    pieces.push(match);
+    start = end;
+  }
+  return pieces;
+}
+function encode(text, config) {
+  const tokens = [config.clsToken];
+  for (const word of preTokenize(normalize(text))) tokens.push(...wordPiece(word, config));
+  tokens.push(config.sepToken);
+  const unk = config.vocab[config.unkToken];
+  const ids = tokens.map((token) => config.vocab[token] ?? unk);
+  return {
+    ids,
+    attentionMask: ids.map(() => 1),
+    tokenTypeIds: ids.map(() => 0)
+  };
+}
+function isControl(char, code) {
+  if (char === "	" || char === "\n" || char === "\r") return false;
+  return code < 32 || code === 127 || new RegExp("\\p{Cc}|\\p{Cf}", "u").test(char);
+}
+function isWhitespace(char, code) {
+  return char === " " || char === "	" || char === "\n" || char === "\r" || code === 11 || code === 12 || new RegExp("\\p{Zs}", "u").test(char);
+}
+function isPunctuation(char) {
+  const code = char.codePointAt(0);
+  const asciiSymbol = code >= 33 && code <= 47 || code >= 58 && code <= 64 || code >= 91 && code <= 96 || code >= 123 && code <= 126;
+  return asciiSymbol || new RegExp("\\p{P}", "u").test(char);
+}
+function isChinese(code) {
+  return code >= 19968 && code <= 40959 || code >= 13312 && code <= 19903 || code >= 131072 && code <= 173791 || code >= 173824 && code <= 177983 || code >= 177984 && code <= 178207 || code >= 178208 && code <= 183983 || code >= 63744 && code <= 64255 || code >= 194560 && code <= 195103;
+}
+
+// src/registry/localEmbedder.ts
+var MAX_TOKENS = 512;
+var FEATURE = "on-device search";
+async function loadLocalEmbedder() {
+  const ort = await loadOptionalPackage("onnxruntime-node", FEATURE);
+  const config = configFromTokenizerJson(readFileSync4(localTokenizerPath(), "utf-8"));
+  const session = await ort.InferenceSession.create(localModelPath());
+  return {
+    async embed(texts, options) {
+      if (texts.length === 0) return [];
+      const prefix = options?.isQuery ? QUERY_INSTRUCTION : "";
+      const encodings = texts.map((text) => truncate(encode(prefix + text, config)));
+      const width = Math.max(...encodings.map((e) => e.ids.length));
+      const batch = encodings.length;
+      const ids = new BigInt64Array(batch * width);
+      const mask = new BigInt64Array(batch * width);
+      const types = new BigInt64Array(batch * width);
+      encodings.forEach((encoding, row) => {
+        encoding.ids.forEach((id, column) => {
+          const at = row * width + column;
+          ids[at] = BigInt(id);
+          mask[at] = 1n;
+          types[at] = 0n;
+        });
+      });
+      const dims = [batch, width];
+      const output = await session.run({
+        input_ids: new ort.Tensor("int64", ids, dims),
+        attention_mask: new ort.Tensor("int64", mask, dims),
+        token_type_ids: new ort.Tensor("int64", types, dims)
+      });
+      const hidden = output["last_hidden_state"];
+      if (!hidden) throw new Error("model returned no last_hidden_state");
+      const data = hidden.data;
+      const hiddenSize = hidden.dims[2];
+      if (hiddenSize !== LOCAL_MODEL_DIMENSIONS) {
+        throw new Error(
+          `model produced ${hiddenSize} dimensions, expected ${LOCAL_MODEL_DIMENSIONS}`
+        );
+      }
+      return encodings.map((_, row) => {
+        const start = row * width * hiddenSize;
+        return normalize2(Array.from(data.subarray(start, start + hiddenSize)));
+      });
+    }
+  };
+}
+function truncate(encoding) {
+  if (encoding.ids.length <= MAX_TOKENS) return encoding;
+  const ids = [
+    ...encoding.ids.slice(0, MAX_TOKENS - 1),
+    encoding.ids[encoding.ids.length - 1]
+  ];
+  return { ids, attentionMask: ids.map(() => 1), tokenTypeIds: ids.map(() => 0) };
+}
+function normalize2(vector) {
+  const norm = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
+  return norm === 0 ? vector : vector.map((value) => value / norm);
+}
+function cosine(a, b) {
+  let sum = 0;
+  for (let i = 0; i < a.length; i += 1) sum += a[i] * b[i];
+  return sum;
+}
+
+// src/registry/localSemantic.ts
+function isMediaVectorRow(value) {
+  if (!value || typeof value !== "object") return false;
+  const row = value;
+  return typeof row.id === "string" && typeof row.kind === "string" && typeof row.title === "string" && typeof row.description === "string" && Array.isArray(row.tags) && row.tags.every((tag) => typeof tag === "string") && typeof row.file === "string" && (row.duration === void 0 || typeof row.duration === "number" && Number.isFinite(row.duration) && row.duration >= 0) && (row.dimensions === void 0 || typeof row.dimensions === "object" && row.dimensions !== null && Number.isInteger(row.dimensions.width) && Number.isInteger(row.dimensions.height) && row.dimensions.width > 0 && row.dimensions.height > 0);
+}
+var CATALOG_ARTIFACT_TIMEOUT_MS = 3e4;
+function localVectorDirectory() {
+  return process.env["HYPERFRAMES_CATALOG_ARTIFACT_DIR"] ?? join4(homedir4(), ".hyperframes", "catalog");
+}
+function vectorPairAgrees(fetched, artifactBasename) {
+  const meta = fetched.find(([file]) => file === `${artifactBasename}.json`)?.[1];
+  const bin = fetched.find(([file]) => file === `${artifactBasename}.bin`)?.[1];
+  if (!meta || !bin) return false;
+  try {
+    const parsed = JSON.parse(meta.toString("utf-8"));
+    if (parsed.dimensions !== LOCAL_MODEL_DIMENSIONS) return false;
+    if (artifactBasename === "media-vectors" && (!parsed.rows || parsed.rows.length !== (parsed.names?.length ?? -1) || parsed.rows.some(
+      (row, index) => !isMediaVectorRow(row) || row.id !== parsed.names?.[index]
+    ))) {
+      return false;
+    }
+    return bin.byteLength === (parsed.names?.length ?? -1) * LOCAL_MODEL_DIMENSIONS * 4;
+  } catch {
+    return false;
+  }
+}
+function vectorRevisionAgrees(fetched, expectedRevision, artifactBasename = "local-vectors") {
+  if (expectedRevision === void 0) return true;
+  const meta = fetched.find(([file]) => file === `${artifactBasename}.json`)?.[1];
+  if (!meta) return false;
+  try {
+    const parsed = JSON.parse(meta.toString("utf-8"));
+    return parsed.revision === expectedRevision;
+  } catch {
+    return false;
+  }
+}
+async function fetchLocalVectors(registryBaseUrl, options = {}) {
+  const directory = options.directory ?? localVectorDirectory();
+  const artifactBasename = options.artifactBasename ?? "local-vectors";
+  const base = registryBaseUrl.replace(/\/+$/, "");
+  try {
+    mkdirSync4(directory, { recursive: true, mode: 448 });
+    const fetched = [];
+    for (const file of [`${artifactBasename}.json`, `${artifactBasename}.bin`]) {
+      const response = await fetch(`${base}/catalog-artifact/${file}`, {
+        signal: AbortSignal.timeout(CATALOG_ARTIFACT_TIMEOUT_MS)
+      });
+      if (!response.ok) return false;
+      fetched.push([file, Buffer.from(await response.arrayBuffer())]);
+    }
+    if (!vectorPairAgrees(fetched, artifactBasename) || !vectorRevisionAgrees(fetched, options.expectedRevision, artifactBasename)) {
+      return false;
+    }
+    for (const [file, bytes] of fetched) {
+      writeFileSync3(join4(directory, file), bytes, { mode: 384 });
+    }
+    return artifactBasename === "media-vectors" ? hasMediaVectors(directory) : hasLocalVectors(directory);
+  } catch {
+    return false;
+  }
+}
+function hasLocalVectors(directory = localVectorDirectory()) {
+  return existsSync4(join4(directory, "local-vectors.bin")) && existsSync4(join4(directory, "local-vectors.json"));
+}
+function hasMediaVectors(directory = localVectorDirectory()) {
+  return existsSync4(join4(directory, "media-vectors.bin")) && existsSync4(join4(directory, "media-vectors.json"));
+}
+function loadLocalVectors(directory = localVectorDirectory()) {
+  const meta = JSON.parse(
+    readFileSync5(join4(directory, "local-vectors.json"), "utf-8")
+  );
+  const buffer = readFileSync5(join4(directory, "local-vectors.bin"));
+  const vectors = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
+  const expected = meta.names.length * meta.dimensions;
+  if (vectors.length !== expected) {
+    throw new Error(`local vectors hold ${vectors.length} floats, expected ${expected}`);
+  }
+  if (meta.dimensions !== LOCAL_MODEL_DIMENSIONS) {
+    throw new Error(
+      `local vectors are ${meta.dimensions}-dimension, model produces ${LOCAL_MODEL_DIMENSIONS}`
+    );
+  }
+  return { names: meta.names, dimensions: meta.dimensions, vectors };
+}
+function loadMediaVectors(directory = localVectorDirectory()) {
+  const meta = JSON.parse(
+    readFileSync5(join4(directory, "media-vectors.json"), "utf-8")
+  );
+  const buffer = readFileSync5(join4(directory, "media-vectors.bin"));
+  const vectors = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
+  const rows = meta.rows ?? [];
+  if (rows.length !== meta.names.length) {
+    throw new Error(`media vectors hold ${rows.length} rows, expected ${meta.names.length}`);
+  }
+  if (rows.some((row, index) => !isMediaVectorRow(row) || row.id !== meta.names[index])) {
+    throw new Error("media vector row order does not match media vector names");
+  }
+  if (vectors.length !== rows.length * meta.dimensions) {
+    throw new Error(
+      `media vectors hold ${vectors.length} floats, expected ${rows.length * meta.dimensions}`
+    );
+  }
+  if (meta.dimensions !== LOCAL_MODEL_DIMENSIONS) {
+    throw new Error(
+      `media vectors are ${meta.dimensions}-dimension, model produces ${LOCAL_MODEL_DIMENSIONS}`
+    );
+  }
+  return { names: meta.names, dimensions: meta.dimensions, vectors, rows };
+}
+function localVectorNames(directory = localVectorDirectory()) {
+  if (!hasLocalVectors(directory)) return [];
+  try {
+    const meta = JSON.parse(readFileSync5(join4(directory, "local-vectors.json"), "utf-8"));
+    return meta.names ?? [];
+  } catch {
+    return [];
+  }
+}
+function cachedLocalVectorRevision(directory = localVectorDirectory()) {
+  if (!hasLocalVectors(directory)) return void 0;
+  try {
+    const meta = JSON.parse(readFileSync5(join4(directory, "local-vectors.json"), "utf-8"));
+    return typeof meta.revision === "string" ? meta.revision : void 0;
+  } catch {
+    return void 0;
+  }
+}
+async function localSemanticRanking(query, directory = localVectorDirectory()) {
+  if (!isLocalModelReady() || !hasLocalVectors(directory)) return null;
+  const set = loadLocalVectors(directory);
+  const embedder = await loadLocalEmbedder();
+  const [queryVector] = await embedder.embed([query], { isQuery: true });
+  if (!queryVector) return null;
+  const scored = set.names.map((name, row) => {
+    const start = row * set.dimensions;
+    return {
+      name,
+      score: cosine(queryVector, Array.from(set.vectors.subarray(start, start + set.dimensions)))
+    };
+  });
+  scored.sort((a, b) => b.score - a.score || b.name.localeCompare(a.name));
+  return scored;
+}
+async function mediaSemanticRanking(query, directory = localVectorDirectory()) {
+  if (!isLocalModelReady() || !hasMediaVectors(directory)) return null;
+  const set = loadMediaVectors(directory);
+  const embedder = await loadLocalEmbedder();
+  const [queryVector] = await embedder.embed([query], { isQuery: true });
+  if (!queryVector) return null;
+  return set.rows.map((row, index) => ({
+    row,
+    score: cosine(
+      queryVector,
+      Array.from(set.vectors.subarray(index * set.dimensions, (index + 1) * set.dimensions))
+    )
+  })).sort((a, b) => b.score - a.score || b.row.id.localeCompare(a.row.id));
+}
+export {
+  cachedLocalVectorRevision,
+  fetchLocalVectors,
+  hasLocalVectors,
+  isMediaVectorRow,
+  localSemanticRanking,
+  localVectorNames,
+  mediaSemanticRanking,
+  vectorPairAgrees
+};