diffwiki-core 0.4.0 → 0.5.0-rc.202607230329.fb70592

package/dist/bm25.cjs

@@ -0,0 +1,85 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/bm25.ts
+var bm25_exports = {};
+__export(bm25_exports, {
+  rankDocs: () => rankDocs,
+  tokenize: () => tokenize
+});
+module.exports = __toCommonJS(bm25_exports);
+var K1 = 1.5;
+var B = 0.75;
+var FIELD_WEIGHTS = { title: 3, tags: 2, collection: 1.5, body: 1 };
+function tokenize(text) {
+  return text.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 0);
+}
+function termFreqs(tokens) {
+  const freqs = /* @__PURE__ */ new Map();
+  for (const t of tokens) freqs.set(t, (freqs.get(t) ?? 0) + 1);
+  return freqs;
+}
+function bm25FieldScore(queryTerms, fieldTokens, avgFieldLen, idf) {
+  const tf = termFreqs(fieldTokens);
+  const dl = fieldTokens.length;
+  const avgDl = avgFieldLen || 1;
+  let score = 0;
+  for (const term of queryTerms) {
+    const freq = tf.get(term) ?? 0;
+    if (freq === 0) continue;
+    const termIdf = idf.get(term) ?? 0;
+    const numerator = freq * (K1 + 1);
+    const denominator = freq + K1 * (1 - B + B * (dl / avgDl));
+    score += termIdf * (numerator / denominator);
+  }
+  return score;
+}
+function rankDocs(term, docs) {
+  const queryTerms = tokenize(term);
+  if (queryTerms.length === 0) return docs.map((d) => ({ ...d }));
+  const N = docs.length;
+  const docTokenSets = docs.map(
+    (d) => /* @__PURE__ */ new Set([...d.fields.title, ...d.fields.tags, ...d.fields.collection, ...d.fields.body])
+  );
+  const idf = /* @__PURE__ */ new Map();
+  for (const qt of queryTerms) {
+    let docFreq = 0;
+    for (const set of docTokenSets) if (set.has(qt)) docFreq++;
+    idf.set(qt, Math.log((N - docFreq + 0.5) / (docFreq + 0.5) + 1));
+  }
+  const avg = (sel) => docs.reduce((s, d) => s + sel(d), 0) / (N || 1);
+  const avgLens = {
+    title: avg((d) => d.fields.title.length),
+    tags: avg((d) => d.fields.tags.length),
+    collection: avg((d) => d.fields.collection.length),
+    body: avg((d) => d.fields.body.length)
+  };
+  const scored = [];
+  for (const d of docs) {
+    const total = bm25FieldScore(queryTerms, d.fields.title, avgLens.title, idf) * FIELD_WEIGHTS.title + bm25FieldScore(queryTerms, d.fields.tags, avgLens.tags, idf) * FIELD_WEIGHTS.tags + bm25FieldScore(queryTerms, d.fields.collection, avgLens.collection, idf) * FIELD_WEIGHTS.collection + bm25FieldScore(queryTerms, d.fields.body, avgLens.body, idf) * FIELD_WEIGHTS.body;
+    if (total > 0) scored.push({ ...d, score: total });
+  }
+  scored.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
+  return scored;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  rankDocs,
+  tokenize
+});

package/dist/bm25.d.cts

@@ -0,0 +1,48 @@
+/**
+ * Pure, runtime-agnostic BM25 ranking.
+ *
+ * This module has ZERO Node/filesystem imports, so the exact same ranking runs on
+ * the server (native search over files) and in the browser (the static site, over
+ * a pre-built index). It is exposed browser-side via the `diffwiki-core/bm25`
+ * subpath export so the client bundle never pulls in Node-only core code.
+ *
+ * @module
+ */
+/** Tokenize a string: lowercase, split on non-alphanumeric, drop empties. */
+declare function tokenize(text: string): string[];
+/** Pre-tokenized, weighted fields of a rankable document. */
+interface RankableFields {
+    title: string[];
+    tags: string[];
+    collection: string[];
+    body: string[];
+}
+/**
+ * A document in the serialisable search index: routing metadata (for linking to
+ * the article) plus the pre-tokenized fields BM25 ranks. The static exporter
+ * writes `SearchDoc[]` to `search-index.json`; the browser ranks it with
+ * {@link rankDocs}.
+ */
+interface SearchDoc {
+    collection: string;
+    slug: string;
+    title: string;
+    tags: string[];
+    fields: RankableFields;
+}
+/**
+ * Rank pre-tokenized docs against a query with BM25 over the four weighted fields
+ * (title ×3, tags ×2, collection ×1.5, body ×1). Pure — no I/O.
+ *
+ * An empty/blank query returns every doc unranked (browse-all), preserving the
+ * native search's behaviour. Otherwise only matching docs (score > 0) are
+ * returned, highest score first. Each returned doc keeps its original shape plus
+ * a `score`.
+ */
+declare function rankDocs<T extends {
+    fields: RankableFields;
+}>(term: string, docs: T[]): Array<T & {
+    score?: number;
+}>;
+
+export { type RankableFields, type SearchDoc, rankDocs, tokenize };

package/dist/bm25.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Pure, runtime-agnostic BM25 ranking.
+ *
+ * This module has ZERO Node/filesystem imports, so the exact same ranking runs on
+ * the server (native search over files) and in the browser (the static site, over
+ * a pre-built index). It is exposed browser-side via the `diffwiki-core/bm25`
+ * subpath export so the client bundle never pulls in Node-only core code.
+ *
+ * @module
+ */
+/** Tokenize a string: lowercase, split on non-alphanumeric, drop empties. */
+declare function tokenize(text: string): string[];
+/** Pre-tokenized, weighted fields of a rankable document. */
+interface RankableFields {
+    title: string[];
+    tags: string[];
+    collection: string[];
+    body: string[];
+}
+/**
+ * A document in the serialisable search index: routing metadata (for linking to
+ * the article) plus the pre-tokenized fields BM25 ranks. The static exporter
+ * writes `SearchDoc[]` to `search-index.json`; the browser ranks it with
+ * {@link rankDocs}.
+ */
+interface SearchDoc {
+    collection: string;
+    slug: string;
+    title: string;
+    tags: string[];
+    fields: RankableFields;
+}
+/**
+ * Rank pre-tokenized docs against a query with BM25 over the four weighted fields
+ * (title ×3, tags ×2, collection ×1.5, body ×1). Pure — no I/O.
+ *
+ * An empty/blank query returns every doc unranked (browse-all), preserving the
+ * native search's behaviour. Otherwise only matching docs (score > 0) are
+ * returned, highest score first. Each returned doc keeps its original shape plus
+ * a `score`.
+ */
+declare function rankDocs<T extends {
+    fields: RankableFields;
+}>(term: string, docs: T[]): Array<T & {
+    score?: number;
+}>;
+
+export { type RankableFields, type SearchDoc, rankDocs, tokenize };

package/dist/bm25.js

@@ -0,0 +1,8 @@
+import {
+  rankDocs,
+  tokenize
+} from "./chunk-FTUF6IXS.js";
+export {
+  rankDocs,
+  tokenize
+};

package/dist/chunk-EEETAETA.js

@@ -0,0 +1,145 @@
+// src/plugins/sdk.ts
+import { execFile } from "child_process";
+import { promisify } from "util";
+import { existsSync, readFileSync } from "fs";
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+var execFileAsync = promisify(execFile);
+async function runProcess(file, args, opts) {
+  const { stdout, stderr } = await execFileAsync(file, args, {
+    env: opts?.env,
+    timeout: opts?.timeout,
+    // Large: qmd emits multi-MB JSON search payloads that overflow the default 1 MB.
+    maxBuffer: 50 * 1024 * 1024
+  });
+  return { stdout, stderr };
+}
+async function runProcessJson(file, args, opts) {
+  const { stdout } = await runProcess(file, args, opts);
+  try {
+    return JSON.parse(stdout);
+  } catch {
+    throw new Error(`${file} ${args.join(" ")} did not return valid JSON: ${stdout.slice(0, 200)}`);
+  }
+}
+async function commandExists(name) {
+  try {
+    await execFileAsync(name, ["--version"]);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function resolveNpmBin(pkg, command, fromUrl) {
+  try {
+    let dir = dirname(fileURLToPath(fromUrl));
+    for (; ; ) {
+      const pkgJson = join(dir, "node_modules", pkg, "package.json");
+      if (existsSync(pkgJson)) {
+        const raw = JSON.parse(readFileSync(pkgJson, "utf8"));
+        const rel = typeof raw.bin === "string" ? raw.bin : raw.bin?.[command];
+        if (!rel) return null;
+        const binPath = join(dirname(pkgJson), rel);
+        return existsSync(binPath) ? binPath : null;
+      }
+      const parent = dirname(dir);
+      if (parent === dir) return null;
+      dir = parent;
+    }
+  } catch {
+    return null;
+  }
+}
+async function resolveDependency(dep, fromUrl) {
+  if (dep.npm) {
+    const local = resolveNpmBin(dep.npm, dep.command, fromUrl);
+    if (local) return { dependency: dep, command: [local], source: "npm" };
+  }
+  if (await commandExists(dep.command)) {
+    return { dependency: dep, command: [dep.command], source: "path" };
+  }
+  return { dependency: dep, command: null, source: "missing" };
+}
+function definePlugin(def) {
+  const { name, kind, version, capabilities, dependencies, handlers } = def;
+  const describeResult = {
+    name,
+    kind,
+    version,
+    capabilities: {
+      searchTypes: capabilities.searchTypes,
+      collectionFilter: capabilities.collectionFilter,
+      events: capabilities.events
+    },
+    dependencies: (dependencies ?? []).map((d) => ({ command: d.command, npm: d.npm, hint: d.hint }))
+  };
+  async function dispatch(op, params, log) {
+    switch (op) {
+      case "describe":
+        return describeResult;
+      case "search":
+        return handlers.search(params, log);
+      case "index":
+        if (handlers.index) return handlers.index(params, log);
+        return { indexed: 0 };
+      case "setup":
+        if (handlers.setup) return handlers.setup(params, log);
+        return { ready: true };
+      case "event":
+        if (handlers.onEvent) return handlers.onEvent(params, log);
+        return { accepted: true };
+      case "health":
+        if (handlers.health) return handlers.health(params, log);
+        return { ready: true, diagnostics: [] };
+      default:
+        throw new Error(`unknown op: ${op}`);
+    }
+  }
+  function run() {
+    const log = (level, message) => {
+      process.stderr.write(JSON.stringify({ log: { level, message } }) + "\n");
+    };
+    let buffer = "";
+    process.stdin.setEncoding("utf8");
+    process.stdin.on("data", (chunk) => {
+      buffer += chunk;
+    });
+    process.stdin.on("end", () => {
+      const line = buffer.trim();
+      if (!line) {
+        process.stderr.write(`${name}: no input received on stdin
+`);
+        process.exit(1);
+        return;
+      }
+      let req;
+      try {
+        req = JSON.parse(line);
+      } catch {
+        process.stderr.write(`${name}: malformed JSON on stdin: ${line.slice(0, 200)}
+`);
+        process.exit(1);
+        return;
+      }
+      dispatch(req.op, req.params ?? {}, log).then((result) => {
+        const response = { ok: true, result };
+        process.stdout.write(JSON.stringify(response) + "\n");
+        process.exit(0);
+      }).catch((err) => {
+        const message = err instanceof Error ? err.message : String(err);
+        const response = { ok: false, error: { message } };
+        process.stdout.write(JSON.stringify(response) + "\n");
+        process.exit(1);
+      });
+    });
+  }
+  return { definition: def, run };
+}
+
+export {
+  runProcess,
+  runProcessJson,
+  commandExists,
+  resolveDependency,
+  definePlugin
+};

package/dist/chunk-FTUF6IXS.js

@@ -0,0 +1,60 @@
+// src/bm25.ts
+var K1 = 1.5;
+var B = 0.75;
+var FIELD_WEIGHTS = { title: 3, tags: 2, collection: 1.5, body: 1 };
+function tokenize(text) {
+  return text.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 0);
+}
+function termFreqs(tokens) {
+  const freqs = /* @__PURE__ */ new Map();
+  for (const t of tokens) freqs.set(t, (freqs.get(t) ?? 0) + 1);
+  return freqs;
+}
+function bm25FieldScore(queryTerms, fieldTokens, avgFieldLen, idf) {
+  const tf = termFreqs(fieldTokens);
+  const dl = fieldTokens.length;
+  const avgDl = avgFieldLen || 1;
+  let score = 0;
+  for (const term of queryTerms) {
+    const freq = tf.get(term) ?? 0;
+    if (freq === 0) continue;
+    const termIdf = idf.get(term) ?? 0;
+    const numerator = freq * (K1 + 1);
+    const denominator = freq + K1 * (1 - B + B * (dl / avgDl));
+    score += termIdf * (numerator / denominator);
+  }
+  return score;
+}
+function rankDocs(term, docs) {
+  const queryTerms = tokenize(term);
+  if (queryTerms.length === 0) return docs.map((d) => ({ ...d }));
+  const N = docs.length;
+  const docTokenSets = docs.map(
+    (d) => /* @__PURE__ */ new Set([...d.fields.title, ...d.fields.tags, ...d.fields.collection, ...d.fields.body])
+  );
+  const idf = /* @__PURE__ */ new Map();
+  for (const qt of queryTerms) {
+    let docFreq = 0;
+    for (const set of docTokenSets) if (set.has(qt)) docFreq++;
+    idf.set(qt, Math.log((N - docFreq + 0.5) / (docFreq + 0.5) + 1));
+  }
+  const avg = (sel) => docs.reduce((s, d) => s + sel(d), 0) / (N || 1);
+  const avgLens = {
+    title: avg((d) => d.fields.title.length),
+    tags: avg((d) => d.fields.tags.length),
+    collection: avg((d) => d.fields.collection.length),
+    body: avg((d) => d.fields.body.length)
+  };
+  const scored = [];
+  for (const d of docs) {
+    const total = bm25FieldScore(queryTerms, d.fields.title, avgLens.title, idf) * FIELD_WEIGHTS.title + bm25FieldScore(queryTerms, d.fields.tags, avgLens.tags, idf) * FIELD_WEIGHTS.tags + bm25FieldScore(queryTerms, d.fields.collection, avgLens.collection, idf) * FIELD_WEIGHTS.collection + bm25FieldScore(queryTerms, d.fields.body, avgLens.body, idf) * FIELD_WEIGHTS.body;
+    if (total > 0) scored.push({ ...d, score: total });
+  }
+  scored.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
+  return scored;
+}
+
+export {
+  tokenize,
+  rankDocs
+};