@wei840222/qmd 2026.8.23

package/scripts/package-smoke.mjs

@@ -0,0 +1,205 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+const root = fileURLToPath(new URL("..", import.meta.url));
+const pkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8"));
+
+class PackageSmokeError extends Error {
+  constructor(message, exitCode = 1) {
+    super(message);
+    this.exitCode = exitCode;
+  }
+}
+
+function run(label, command, args, options = {}) {
+  console.log(`==> ${label}`);
+  const { quiet, ...spawnOptions } = options;
+  const result = spawnSync(command, args, {
+    cwd: root,
+    stdio: quiet ? "pipe" : "inherit",
+    shell: process.platform === "win32",
+    ...spawnOptions,
+  });
+  if (result.status !== 0) {
+    if (quiet) {
+      if (result.stdout) process.stderr.write(result.stdout);
+      if (result.stderr) process.stderr.write(result.stderr);
+    }
+    throw new PackageSmokeError(`Package smoke failed: ${label}`, result.status ?? 1);
+  }
+}
+
+function capture(label, command, args, options = {}) {
+  console.log(`==> ${label}`);
+  const result = spawnSync(command, args, {
+    cwd: root,
+    encoding: "utf8",
+    shell: process.platform === "win32",
+    ...options,
+  });
+  if (result.status !== 0) {
+    if (result.stdout) process.stderr.write(result.stdout);
+    if (result.stderr) process.stderr.write(result.stderr);
+    throw new PackageSmokeError(`Package smoke failed: ${label}`, result.status ?? 1);
+  }
+  return result.stdout;
+}
+
+function assertPath(path, label = path) {
+  const full = join(root, path);
+  if (!existsSync(full)) {
+    throw new PackageSmokeError(`Package smoke failed: missing ${label} (${path})`);
+  }
+  return full;
+}
+
+function assertMissing(path, label = path) {
+  if (existsSync(join(root, path))) {
+    throw new PackageSmokeError(`Package smoke failed: unexpected ${label} (${path})`);
+  }
+}
+
+function main() {
+run("build compiled package", process.execPath, ["scripts/build.mjs"]);
+run("AST grammar runtime packages", process.execPath, ["scripts/check-package-grammars.mjs"]);
+
+for (const entry of pkg.files ?? []) {
+  assertPath(entry.replace(/\/$/, ""), `package.json files[] entry ${entry}`);
+}
+
+for (const [name, binPath] of Object.entries(pkg.bin ?? {})) {
+  const full = assertPath(binPath, `bin ${name}`);
+  const mode = statSync(full).mode;
+  if ((mode & 0o111) === 0) {
+    throw new PackageSmokeError(`Package smoke failed: bin ${name} is not executable (${binPath})`);
+  }
+}
+
+assertPath("dist/index.js", "compiled main export");
+assertPath("dist/index.d.ts", "compiled type export");
+assertPath("dist/cli/qmd.js", "compiled CLI");
+assertPath("dist/search/jieba-loader.js", "compiled jieba capability loader");
+assertPath("dist/search/cjk-analyzer.js", "compiled CJK analyzer");
+assertPath("dist/search/zh-dict.txt", "compiled Chinese dictionary asset");
+assertMissing("dist/search/zh-tw-dictionary.txt", "renamed Chinese dictionary asset");
+assertMissing("dist/search/zh-tw-tech-dictionary.js", "legacy zh-TW dictionary module");
+assertPath("THIRD_PARTY_NOTICES.md", "third-party notices");
+
+run("compiled CLI under Node", process.execPath, ["dist/cli/qmd.js", "--help"], { quiet: true });
+run("package wrapper", "sh", ["bin/qmd", "--help"], { quiet: true });
+
+if (process.env.QMD_SKIP_BUN_SMOKE === "1") {
+  console.log("==> compiled CLI under Bun (skipped by QMD_SKIP_BUN_SMOKE=1)");
+} else {
+  run("compiled CLI under Bun", "bun", ["dist/cli/qmd.js", "--help"], { quiet: true });
+}
+
+const packageSmokeRoot = process.env.QMD_PACKAGE_SMOKE_TMPDIR || join(root, ".tmp");
+mkdirSync(packageSmokeRoot, { recursive: true });
+const packageSmokeDir = mkdtempSync(join(packageSmokeRoot, "qmd-package-smoke-"));
+try {
+  const packOutput = capture("pack npm tarball", "npm", [
+    "pack",
+    "--ignore-scripts",
+    "--json",
+    "--pack-destination",
+    packageSmokeDir,
+  ]);
+  const packResult = JSON.parse(packOutput);
+  const tarballName = packResult[0]?.filename;
+  if (typeof tarballName !== "string") {
+    throw new PackageSmokeError("Package smoke failed: npm pack did not report a tarball filename");
+  }
+
+  const consumerDir = join(packageSmokeDir, "consumer");
+  mkdirSync(consumerDir);
+  writeFileSync(
+    join(consumerDir, "package.json"),
+    `${JSON.stringify({ private: true, type: "module" }, null, 2)}\n`,
+  );
+  run(
+    "install packed tarball with Bun",
+    "bun",
+    ["add", "--ignore-scripts", join(packageSmokeDir, tarballName)],
+    { cwd: consumerDir, quiet: true },
+  );
+
+  const installedScopeAndName = pkg.name.split("/");
+  const installedRoot = join(consumerDir, "node_modules", ...installedScopeAndName);
+  const loaderUrl = pathToFileURL(
+    join(installedRoot, "dist", "search", "jieba-loader.js"),
+  ).href;
+  const analyzerUrl = pathToFileURL(
+    join(installedRoot, "dist", "search", "cjk-analyzer.js"),
+  ).href;
+  for (const path of [
+    "THIRD_PARTY_NOTICES.md",
+    "dist/search/zh-dict.txt",
+  ]) {
+    if (!existsSync(join(installedRoot, path))) {
+      throw new Error(`Packed package is missing ${path}`);
+    }
+  }
+  const jiebaSmoke = `
+    const { loadJiebaCapability } = await import(process.env.QMD_JIEBA_LOADER_URL);
+    const capability = await loadJiebaCapability();
+    if (!capability.available) throw new Error(capability.diagnostic.code);
+    const actual = capability.cut("我們使用記憶體快取資料", false);
+    const expected = ["我們", "使用", "記憶體", "快取", "資料"];
+    if (JSON.stringify(actual) !== JSON.stringify(expected)) {
+      throw new Error("Unexpected jieba golden tokens");
+    }
+    const { analyzeCjk } = await import(process.env.QMD_CJK_ANALYZER_URL);
+    const analyzed = await analyzeCjk("相依性雜湊佇列");
+    if (analyzed.word !== "相依性 雜湊 佇列") throw new Error("Unexpected analyzer dictionary tokens");
+    if (JSON.stringify(capability.cut("CC霜", false)) !== JSON.stringify(["CC霜"])) {
+      throw new Error("Bundled Chinese dictionary is incomplete");
+    }
+  `;
+  const smokeEnv = {
+    ...process.env,
+    QMD_JIEBA_LOADER_URL: loaderUrl,
+    QMD_CJK_ANALYZER_URL: analyzerUrl,
+  };
+
+  run(
+    "packed jieba capability under Node",
+    process.execPath,
+    ["--input-type=module", "--eval", jiebaSmoke],
+    { cwd: consumerDir, env: smokeEnv, quiet: true },
+  );
+  if (process.env.QMD_SKIP_BUN_SMOKE !== "1") {
+    run(
+      "packed jieba capability under Bun",
+      "bun",
+      ["--eval", jiebaSmoke],
+      { cwd: consumerDir, env: smokeEnv, quiet: true },
+    );
+  }
+} finally {
+  rmSync(packageSmokeDir, { recursive: true, force: true });
+}
+}
+
+try {
+  main();
+} catch (error) {
+  if (error instanceof PackageSmokeError) {
+    console.error(error.message);
+    process.exitCode = error.exitCode;
+  } else {
+    console.error(error);
+    process.exitCode = 1;
+  }
+}

package/scripts/sync-zh-dict.mjs

@@ -0,0 +1,187 @@
+#!/usr/bin/env node
+import { createHash } from "node:crypto";
+import { createRequire } from "node:module";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { readFileSync, writeFileSync } from "node:fs";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "..");
+const sourcesPath = join(root, "src", "search", "zh-dict.sources.json");
+const outputPath = join(root, "src", "search", "zh-dict.txt");
+const updatePins = process.argv.includes("--update-pins");
+const require = createRequire(import.meta.url);
+
+function fail(message) {
+  throw new Error(`Invalid Chinese dictionary source: ${message}`);
+}
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, "utf8"));
+}
+
+function sha256(bytes) {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+
+function gitBlobSha(bytes) {
+  return createHash("sha1")
+    .update(Buffer.from(`blob ${bytes.byteLength}\0`))
+    .update(bytes)
+    .digest("hex");
+}
+
+function compareTerms(left, right) {
+  return left < right ? -1 : left > right ? 1 : 0;
+}
+
+function assertHash(value, label, length) {
+  if (typeof value !== "string" || !new RegExp(`^[a-f0-9]{${length}}$`, "u").test(value)) {
+    fail(`${label} must be a lowercase hexadecimal hash`);
+  }
+}
+
+function assertRemoteSource(source, label) {
+  if (!source || typeof source !== "object") fail(`${label} is missing`);
+  if (typeof source.repository !== "string" || typeof source.branch !== "string"
+    || typeof source.commit !== "string" || typeof source.path !== "string") {
+    fail(`${label} has invalid repository metadata`);
+  }
+  const url = new URL(source.repository);
+  if (url.hostname !== "github.com" || url.pathname.split("/").filter(Boolean).length !== 2) {
+    fail(`${label} repository must be a canonical GitHub URL`);
+  }
+  assertHash(source.commit, `${label}.commit`, 40);
+  assertHash(source.gitBlobSha, `${label}.gitBlobSha`, 40);
+  assertHash(source.sha256, `${label}.sha256`, 64);
+}
+
+function rawUrl(source) {
+  const [owner, repository] = new URL(source.repository).pathname.split("/").filter(Boolean);
+  return `https://raw.githubusercontent.com/${owner}/${repository}/${source.commit}/${source.path}`;
+}
+
+function apiUrl(source) {
+  const [owner, repository] = new URL(source.repository).pathname.split("/").filter(Boolean);
+  return `https://api.github.com/repos/${owner}/${repository}/commits/${encodeURIComponent(source.branch)}`;
+}
+
+async function downloadPinnedSource(source, label) {
+  const response = await fetch(rawUrl(source), { redirect: "error" });
+  if (!response.ok) throw new Error(`${label} download failed with HTTP ${response.status}.`);
+  const bytes = Buffer.from(await response.arrayBuffer());
+  if (sha256(bytes) !== source.sha256 || gitBlobSha(bytes) !== source.gitBlobSha) {
+    throw new Error(`${label} hash mismatch.`);
+  }
+  return bytes;
+}
+
+async function updateRemotePin(source, label) {
+  const response = await fetch(apiUrl(source), {
+    headers: { Accept: "application/vnd.github+json" },
+    redirect: "error",
+  });
+  if (!response.ok) throw new Error(`${label} commit lookup failed with HTTP ${response.status}.`);
+  const payload = await response.json();
+  if (!payload || typeof payload.sha !== "string" || !/^[a-f0-9]{40}$/u.test(payload.sha)) {
+    throw new Error(`${label} commit lookup returned an invalid SHA.`);
+  }
+  source.commit = payload.sha;
+  const responseWithNewPin = await fetch(rawUrl(source), { redirect: "error" });
+  if (!responseWithNewPin.ok) throw new Error(`${label} download failed with HTTP ${responseWithNewPin.status}.`);
+  const bytes = Buffer.from(await responseWithNewPin.arrayBuffer());
+  source.sha256 = sha256(bytes);
+  source.gitBlobSha = gitBlobSha(bytes);
+  return bytes;
+}
+
+function loadNodeRsDictionary(source) {
+  if (!source || typeof source !== "object") fail("nodeRs is missing");
+  if (typeof source.package !== "string" || typeof source.version !== "string" || typeof source.path !== "string") {
+    fail("nodeRs has invalid package metadata");
+  }
+  assertHash(source.gitBlobSha, "nodeRs.gitBlobSha", 40);
+  assertHash(source.sha256, "nodeRs.sha256", 64);
+  const packageJsonPath = require.resolve(`${source.package}/package.json`);
+  const packageJson = readJson(packageJsonPath);
+  if (packageJson.version !== source.version) {
+    throw new Error(`Expected ${source.package}@${source.version}, found ${packageJson.version}.`);
+  }
+  const bytes = readFileSync(join(dirname(packageJsonPath), source.path));
+  if (sha256(bytes) !== source.sha256 || gitBlobSha(bytes) !== source.gitBlobSha) {
+    throw new Error("@node-rs/jieba dictionary hash mismatch.");
+  }
+  return bytes;
+}
+
+function parseDictionary(bytes, label) {
+  const entries = new Map();
+  const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+  for (const [index, line] of text.split("\n").entries()) {
+    if (!line) continue;
+    const match = /^(\S+) (\d+) (\S+)$/u.exec(line);
+    if (!match) fail(`${label} line ${index + 1} is not a jieba dictionary entry`);
+    const [, term, frequency, tag] = match;
+    if (term.normalize("NFC") !== term) fail(`${label} line ${index + 1} is not NFC normalized`);
+    entries.set(term, `${term} ${frequency} ${tag}`);
+  }
+  if (entries.size === 0) fail(`${label} has no dictionary entries`);
+  return entries;
+}
+
+function selectTechnicalTerms(bytes, settings) {
+  if (!settings || typeof settings !== "object" || !Array.isArray(settings.domains)
+    || !Number.isSafeInteger(settings.frequency) || settings.frequency <= 0
+    || typeof settings.tag !== "string" || !settings.tag || /\s/u.test(settings.tag)) {
+    fail("technicalTerms has invalid selection settings");
+  }
+  const domains = settings.domains;
+  if (!domains.every(domain => typeof domain === "string" && domain)) {
+    fail("technicalTerms.domains must contain non-empty strings");
+  }
+  const ruleset = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
+  if (!Array.isArray(ruleset.spelling_rules)) fail("zhtwMcp ruleset is missing spelling_rules");
+  const terms = new Set();
+  for (const rule of ruleset.spelling_rules) {
+    const domain = /@domain ([^。；]+)/u.exec(rule?.context ?? "")?.[1]?.trim();
+    if (
+      rule?.type !== "cross_strait"
+      || !domain
+      || !domains.some(candidate => domain.includes(candidate))
+      || !Array.isArray(rule.to)
+    ) continue;
+    for (const term of rule.to) {
+      if (typeof term === "string" && term.length >= 2 && !/\s/u.test(term) && term.normalize("NFC") === term) {
+        terms.add(term);
+      }
+    }
+  }
+  return [...terms].sort(compareTerms).map(term => `${term} ${settings.frequency} ${settings.tag}`);
+}
+
+const sources = readJson(sourcesPath);
+if (sources.schemaVersion !== 1 || !sources.output || typeof sources.output !== "object") {
+  fail("unsupported source schema");
+}
+assertRemoteSource(sources.apclab, "apclab");
+assertRemoteSource(sources.zhtwMcp, "zhtwMcp");
+
+const nodeRsBytes = loadNodeRsDictionary(sources.nodeRs);
+const apclabBytes = updatePins
+  ? await updateRemotePin(sources.apclab, "APCLab dictionary")
+  : await downloadPinnedSource(sources.apclab, "APCLab dictionary");
+const zhtwMcpBytes = updatePins
+  ? await updateRemotePin(sources.zhtwMcp, "zhtw-mcp ruleset")
+  : await downloadPinnedSource(sources.zhtwMcp, "zhtw-mcp ruleset");
+
+const entries = parseDictionary(nodeRsBytes, "@node-rs/jieba dictionary");
+for (const [term, entry] of parseDictionary(apclabBytes, "APCLab dictionary")) entries.set(term, entry);
+for (const entry of selectTechnicalTerms(zhtwMcpBytes, sources.technicalTerms)) {
+  entries.set(entry.split(" ", 1)[0], entry);
+}
+
+const dictionaryText = `${[...entries.keys()].sort(compareTerms).map(term => entries.get(term)).join("\n")}\n`;
+const dictionaryBytes = Buffer.from(dictionaryText, "utf8");
+sources.output.sha256 = sha256(dictionaryBytes);
+writeFileSync(outputPath, dictionaryBytes);
+writeFileSync(sourcesPath, `${JSON.stringify(sources, null, 2)}\n`);
+console.log(`Generated ${entries.size} Chinese dictionary entries at ${outputPath}.`);

package/scripts/test-all.mjs

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = fileURLToPath(new URL("..", import.meta.url));
+
+// Mirror bin/qmd's darwin Metal residency mitigation for test subprocesses.
+// libggml-metal asserts on a non-empty residency set during its static
+// destructor (ggml-org/llama.cpp#22593, fix open as #22595) and dumps a
+// multi-kB backtrace at process exit even when tests pass. The env var must
+// be set BEFORE the subprocess starts because libggml-metal reads it via
+// libc getenv at module-load time. Opt out with QMD_METAL_KEEP_RESIDENCY=1.
+const darwinMetalEnv =
+  process.platform === "darwin" && process.env.QMD_METAL_KEEP_RESIDENCY !== "1"
+    ? { GGML_METAL_NO_RESIDENCY: "1" }
+    : {};
+
+const cleanEnv = { ...process.env };
+delete cleanEnv.OPENAI_BASE_URL;
+delete cleanEnv.OPENAI_API_KEY;
+delete cleanEnv.QMD_EMBED_MODEL;
+delete cleanEnv.QMD_GENERATE_MODEL;
+delete cleanEnv.QMD_RERANK_MODEL;
+
+function run(label, command, args, options = {}) {
+  console.log(`==> ${label}`);
+  const { env: extraEnv, ...spawnOptions } = options;
+  const result = spawnSync(command, args, {
+    cwd: root,
+    stdio: "inherit",
+    shell: process.platform === "win32",
+    env: { ...cleanEnv, ...darwinMetalEnv, ...(extraEnv ?? {}) },
+    ...spawnOptions,
+  });
+  if (result.status !== 0) {
+    console.error(`Test task failed: ${label}`);
+    process.exit(result.status ?? 1);
+  }
+}
+
+run("TypeScript build typecheck", process.execPath, [join(root, "node_modules", "typescript", "bin", "tsc"), "-p", "tsconfig.build.json", "--noEmit"]);
+run("Vitest suite under Node", process.execPath, [join(root, "node_modules", "vitest", "vitest.mjs"), "run", "--reporter=verbose", "--testTimeout", "60000", "test/"], { env: { CI: "true" } });
+run("Bun test suite", "bun", ["test", "--timeout", "60000", "--preload", "./src/test-preload.ts", "test/"], { env: { CI: "true" } });
+run("Package smoke", process.execPath, ["scripts/package-smoke.mjs"]);