@sorane/search 0.2.0

package/assets/search.mjs

@@ -0,0 +1,239 @@
+// Browser-side search (sorane SSG).
+//
+// FTS mount:  <div data-search data-mode="fts" data-index=".../assets/search-index.json">
+// Hybrid:     + data-model-base, data-lib-base
+
+const MODEL_ID = "ruri-v3-30m";
+const QUERY_PREFIX = "検索クエリ: ";
+const TOP_K = 10;
+
+function tokenizeQuery(query) {
+  const segs = query
+    .split(/[぀-ゟ]+|[\s、。・，．:：;；!！?？()（）「」『』【】\[\]]+/)
+    .map((s) => s.trim())
+    .filter((s) => s.length >= 2);
+  if (segs.length === 0) {
+    const flat = query.trim();
+    return flat.length >= 1 ? [flat] : [];
+  }
+  return segs;
+}
+
+function decodeVectors(b64, dim) {
+  const binary = atob(b64);
+  const buf = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) buf[i] = binary.charCodeAt(i);
+  const i8 = new Int8Array(buf.buffer);
+  if (i8.length % dim !== 0) throw new Error(`vector length ${i8.length} is not a multiple of dim ${dim}`);
+  return i8;
+}
+
+function topK(query, vectors, dim, k, allow) {
+  const n = vectors.length / dim;
+  const heap = [];
+  for (let i = 0; i < n; i++) {
+    if (allow && !allow(i)) continue;
+    let s = 0;
+    const off = i * dim;
+    for (let j = 0; j < dim; j++) s += query[j] * vectors[off + j];
+    if (heap.length < k) {
+      heap.push({ index: i, score: s });
+      heap.sort((a, b) => a.score - b.score);
+    } else if (s > heap[0].score) {
+      heap[0] = { index: i, score: s };
+      heap.sort((a, b) => a.score - b.score);
+    }
+  }
+  return heap.sort((a, b) => b.score - a.score);
+}
+
+function ftsSearch(index, query, type, k = TOP_K) {
+  const terms = tokenizeQuery(query);
+  if (terms.length === 0) return [];
+  const hits = [];
+  for (let i = 0; i < index.chunks.length; i++) {
+    const chunk = index.chunks[i];
+    if (type && chunk.doc_type !== type) continue;
+    const hay = [
+      chunk.text || "",
+      chunk.title || "",
+      chunk.heading_path || "",
+      chunk.tags || "",
+    ]
+      .join(" ")
+      .toLowerCase();
+    let score = 0;
+    for (const term of terms) {
+      const needle = term.toLowerCase();
+      if (hay.includes(needle)) score += 1;
+      if ((chunk.title || "").toLowerCase().includes(needle)) score += 2;
+      if ((chunk.heading_path || "").toLowerCase().includes(needle)) score += 1;
+    }
+    if (score > 0) hits.push({ index: i, score, chunk });
+  }
+  return hits.sort((a, b) => b.score - a.score).slice(0, k);
+}
+
+function makeSnippet(text, query, max = 160) {
+  const flat = text.replace(/\s+/g, " ").trim();
+  if (flat.length <= max) return flat;
+  const q = query.trim();
+  let start = 0;
+  if (q) {
+    const idx = flat.indexOf(q);
+    if (idx >= 0) start = Math.max(0, idx - Math.floor(max / 4));
+  }
+  const end = Math.min(flat.length, start + max);
+  const body = flat.slice(start, end);
+  return (start > 0 ? "…" : "") + body + (end < flat.length ? "…" : "");
+}
+
+function setup(root) {
+  const form = root.querySelector(".search-form");
+  const input = root.querySelector(".search-input");
+  const facet = root.querySelector(".search-facet");
+  const status = root.querySelector("[data-search-status]");
+  const resultsEl = root.querySelector("[data-search-results]");
+  const indexUrl = root.getAttribute("data-index");
+  const mode = root.getAttribute("data-mode") || "fts";
+  const modelBase = root.getAttribute("data-model-base");
+  const libBase = root.getAttribute("data-lib-base");
+  const compact = root.classList.contains("search--header");
+  if (!form || !input || !indexUrl) return;
+
+  let index = null;
+  let embed = null;
+  let busy = false;
+
+  const setStatus = (msg) => {
+    if (!status) return;
+    status.textContent = msg;
+  };
+
+  async function loadIndex() {
+    if (index) return index;
+    setStatus("検索インデックスを読み込み中…");
+    const res = await fetch(indexUrl);
+    if (!res.ok) throw new Error(`failed to fetch search-index.json (${res.status})`);
+    const json = await res.json();
+    const resolvedMode = json.mode === "hybrid" && json.embeddings ? "hybrid" : "fts";
+    if (resolvedMode === "hybrid") {
+      const dim = json.embeddings.dim;
+      index = { ...json, mode: "hybrid", vectors: decodeVectors(json.embeddings.vectors_b64, dim) };
+    } else {
+      index = { ...json, mode: "fts" };
+    }
+    return index;
+  }
+
+  async function loadEmbedder() {
+    if (embed) return embed;
+    setStatus("埋め込みモデルを読み込み中…（初回のみ）");
+    const libUrl = new URL(libBase, document.baseURI).href;
+    const tjs = await import(`${libUrl}transformers.web.js`);
+    const { env, pipeline } = tjs;
+    env.useBrowserCache = false;
+    const isRemoteBase = /^https?:\/\//i.test(modelBase || "");
+    if (isRemoteBase) {
+      env.allowLocalModels = false;
+      env.allowRemoteModels = true;
+      env.remoteHost = modelBase;
+      env.remotePathTemplate = "{model}/";
+    } else {
+      env.allowRemoteModels = false;
+      env.allowLocalModels = true;
+      env.localModelPath = modelBase;
+    }
+    const onnxWasm = env.backends?.onnx?.wasm;
+    if (onnxWasm) onnxWasm.wasmPaths = libUrl;
+    const extractor = await pipeline("feature-extraction", MODEL_ID, { dtype: "q8" });
+    embed = async (query) => {
+      const out = await extractor(QUERY_PREFIX + query, { pooling: "mean", normalize: true });
+      return new Float32Array(out.data);
+    };
+    return embed;
+  }
+
+  function render(hits, query) {
+    resultsEl.replaceChildren();
+    if (hits.length === 0) {
+      setStatus("該当なし。語を変えてお試しください。");
+      return;
+    }
+    setStatus(`${hits.length} 件`);
+    for (const { chunk, score } of hits) {
+      const li = document.createElement("li");
+      li.className = "search-hit";
+
+      const a = document.createElement("a");
+      a.className = "search-hit-title";
+      const anchor = chunk.heading_slug ? `#${chunk.heading_slug}` : "";
+      a.href = `${chunk.url || chunk.source.replace(/\.md$/i, ".html")}${anchor}`;
+      a.textContent = chunk.heading_path || chunk.title || chunk.source;
+      li.appendChild(a);
+
+      const meta = document.createElement("p");
+      meta.className = "search-hit-meta";
+      const scoreLabel = typeof score === "number" && score % 1 !== 0 ? score.toFixed(3) : String(score);
+      meta.textContent = `${chunk.doc_type || "-"} · ${chunk.source} · ${scoreLabel}`;
+      li.appendChild(meta);
+
+      const snippet = chunk.snippet || (chunk.text ? makeSnippet(chunk.text, query) : "");
+      if (snippet) {
+        const snip = document.createElement("p");
+        snip.className = "search-hit-snippet";
+        snip.textContent = snippet;
+        li.appendChild(snip);
+      }
+      resultsEl.appendChild(li);
+    }
+  }
+
+  async function runFts(query) {
+    const idx = await loadIndex();
+    setStatus("検索中…");
+    const type = facet ? facet.value : "";
+    const ranked = ftsSearch(idx, query, type);
+    render(
+      ranked.map((r) => ({ chunk: r.chunk, score: r.score })),
+      query,
+    );
+  }
+
+  async function runHybrid(query) {
+    const idx = await loadIndex();
+    const e = await loadEmbedder();
+    setStatus("検索中…");
+    const qv = await e(query);
+    const type = facet ? facet.value : "";
+    const allow = type ? (i) => idx.chunks[i].doc_type === type : null;
+    const ranked = topK(qv, idx.vectors, idx.embeddings.dim, TOP_K, allow);
+    const scale = idx.embeddings.scale || 1;
+    render(
+      ranked.map((r) => ({ chunk: idx.chunks[r.index], score: r.score / scale })),
+      query,
+    );
+  }
+
+  async function run(query) {
+    if (busy || !query.trim()) return;
+    busy = true;
+    try {
+      const idx = await loadIndex();
+      const useHybrid = (idx.mode || mode) === "hybrid";
+      if (useHybrid) await runHybrid(query);
+      else await runFts(query);
+    } catch (err) {
+      setStatus(`エラー: ${err && err.message ? err.message : err}`);
+    } finally {
+      busy = false;
+    }
+  }
+
+  form.addEventListener("submit", (e) => {
+    e.preventDefault();
+    run(input.value);
+  });
+}
+
+for (const root of document.querySelectorAll("[data-search]")) setup(root);

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@sorane/search",
+  "version": "0.2.0",
+  "description": "FTS and hybrid search indexing for sorane sites",
+  "type": "module",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/masanork/sorane.git",
+    "directory": "packages/search"
+  },
+  "homepage": "https://sorane.dev",
+  "bugs": "https://github.com/masanork/sorane/issues",
+  "files": [
+    "src",
+    "assets"
+  ],
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "engines": {
+    "node": ">=23.6"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^4.2.0",
+    "@sorane/okf": "0.2.0",
+    "better-sqlite3": "^12.11.1",
+    "remark-gfm": "^4.0.1",
+    "remark-parse": "^11.0.0",
+    "sqlite-vec": "^0.1.9",
+    "unified": "^11.0.0"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/mdast": "^4.0.4"
+  }
+}

package/src/build-index.ts

@@ -0,0 +1,115 @@
+import { readFileSync } from "node:fs";
+import { relative, resolve } from "node:path";
+import { chunkDocument } from "./chunker.ts";
+import type { EmbeddingProvider } from "./embeddings.ts";
+import { DOC_PREFIX } from "./embeddings.ts";
+import { hashContent, planIncremental } from "./incremental.ts";
+import { IndexStore } from "./store.ts";
+import { walkMarkdown } from "./walk.ts";
+
+export interface BuildIndexOptions {
+  readonly contentDir: string;
+  readonly indexPath: string;
+  readonly force?: boolean;
+  readonly embeddings?: EmbeddingProvider | null;
+  readonly onProgress?: (message: string) => void;
+}
+
+export interface BuildIndexResult {
+  readonly added: number;
+  readonly changed: number;
+  readonly removed: number;
+  readonly unchanged: number;
+  readonly chunks: number;
+  readonly fts: number;
+  readonly vec: number;
+  readonly mode: "hybrid" | "fts-only";
+}
+
+export async function buildSearchIndex(opts: BuildIndexOptions): Promise<BuildIndexResult> {
+  const contentDir = resolve(opts.contentDir);
+  const log = opts.onProgress ?? (() => {});
+  const hybrid = opts.embeddings != null;
+  const store = new IndexStore(opts.indexPath, {
+    fresh: opts.force === true,
+    dim: hybrid ? opts.embeddings!.dimensions : 256,
+  });
+
+  const files = walkMarkdown(contentDir);
+  const disk = new Map<string, string>();
+  const content = new Map<string, string>();
+  for (const abs of files) {
+    const rel = relative(contentDir, abs).replace(/\\/g, "/");
+    const text = readFileSync(abs, "utf8");
+    disk.set(rel, hashContent(text));
+    content.set(rel, text);
+  }
+
+  const indexed = opts.force ? new Map<string, string>() : store.sourceHashes();
+  const plan = planIncremental(disk, indexed);
+
+  log(
+    `incremental: added ${plan.added.length} / changed ${plan.changed.length} / ` +
+      `removed ${plan.removed.length} / unchanged ${plan.unchanged.length} (${files.length} files)` +
+      (hybrid ? " [hybrid]" : " [fts-only]"),
+  );
+
+  for (const rel of plan.removed) {
+    store.deleteBySource(rel);
+    log(`removed ${rel}`);
+  }
+
+  const targets = [...plan.added, ...plan.changed].sort();
+  let totalChunks = 0;
+  for (let i = 0; i < targets.length; i++) {
+    const rel = targets[i]!;
+    const text = content.get(rel)!;
+    const sha = disk.get(rel)!;
+    store.deleteBySource(rel);
+    const chunks = chunkDocument(text, rel);
+    if (chunks.length === 0) {
+      store.setSourceHash(rel, sha);
+      log(`[${i + 1}/${targets.length}] ${rel}: 0 chunks`);
+      continue;
+    }
+    const vectors = hybrid
+      ? await opts.embeddings!.embedBatch(chunks.map((c) => DOC_PREFIX + c.text))
+      : undefined;
+    store.addChunks(chunks, vectors);
+    store.setSourceHash(rel, sha);
+    totalChunks += chunks.length;
+    log(`[${i + 1}/${targets.length}] ${rel}: ${chunks.length} chunks (${totalChunks} total)`);
+  }
+
+  if (hybrid && opts.embeddings) {
+    store.setMeta({
+      modelId: opts.embeddings.modelId ?? "ruri-v3-30m",
+      dim: opts.embeddings.dimensions,
+      quant: opts.embeddings.quant ?? "q8",
+      modelSha256: opts.embeddings.modelSha256 ?? "",
+    });
+  } else {
+    store.setMeta();
+  }
+
+  const counts = store.counts();
+  store.close();
+
+  if (counts.chunks !== counts.fts) {
+    throw new Error(`index mismatch: chunks=${counts.chunks} fts=${counts.fts}`);
+  }
+  if (hybrid && counts.chunks !== counts.vec) {
+    throw new Error(`index mismatch: chunks=${counts.chunks} vec=${counts.vec}`);
+  }
+
+  return {
+    added: plan.added.length,
+    changed: plan.changed.length,
+    removed: plan.removed.length,
+    unchanged: plan.unchanged.length,
+    chunks: counts.chunks,
+    fts: counts.fts,
+    vec: counts.vec,
+    mode: hybrid ? "hybrid" : "fts-only",
+  };
+}

package/src/chunker.ts

@@ -0,0 +1,174 @@
+import { unified } from "unified";
+import remarkParse from "remark-parse";
+import remarkGfm from "remark-gfm";
+import type { Root, RootContent, Heading } from "mdast";
+import { extract } from "@sorane/okf";
+import { parseYaml } from "@sorane/okf";
+import { SlugLedger } from "./heading-slug.ts";
+
+export const MIN_BODY = 50;
+export const MAX_BODY = 800;
+
+export interface Chunk {
+  readonly source: string;
+  readonly chunkIndex: number;
+  readonly text: string;
+  readonly headingPath: string;
+  readonly headingSlug: string;
+  readonly docType: string;
+  readonly title: string;
+  readonly timestamp: string;
+  readonly tags: string;
+}
+
+interface Meta {
+  docType: string;
+  title: string;
+  timestamp: string;
+  tags: string;
+  skip: boolean;
+}
+
+function nodeToText(node: RootContent | Heading): string {
+  const anyNode = node as { type: string; value?: string; children?: RootContent[] };
+  if (anyNode.type === "text" || anyNode.type === "inlineCode") return anyNode.value ?? "";
+  if (anyNode.children) return anyNode.children.map((c) => nodeToText(c as RootContent)).join("");
+  return "";
+}
+
+function blockToText(node: RootContent): string {
+  switch (node.type) {
+    case "paragraph":
+    case "heading":
+    case "blockquote":
+      return nodeToText(node);
+    case "list":
+      return node.children.map((li) => nodeToText(li as RootContent)).join("\n");
+    case "code":
+    case "table":
+    case "html":
+    case "thematicBreak":
+      return "";
+    default:
+      return nodeToText(node);
+  }
+}
+
+function splitOversized(body: string): string[] {
+  if (body.length <= MAX_BODY) return [body];
+  const parts: string[] = [];
+  for (const para of body.split(/\n\s*\n/)) {
+    const p = para.trim();
+    if (!p) continue;
+    const last = parts[parts.length - 1];
+    if (last === undefined || last.length + p.length + 2 > MAX_BODY) parts.push(p);
+    else parts[parts.length - 1] = last + "\n\n" + p;
+  }
+  return parts.length ? parts : [body];
+}
+
+function slugifyTag(tag: string): string {
+  return tag
+    .trim()
+    .toLowerCase()
+    .replace(/\s+/g, "-")
+    .replace(/[^\w぀-ヿ㐀-鿿豈-﫿ｦ-ﾟ-]/g, "")
+    .replace(/-+/g, "-")
+    .replace(/^-+|-+$/g, "");
+}
+
+function readMeta(frontmatter: string | null): Meta {
+  const fm = (frontmatter ? (parseYaml(frontmatter) as Record<string, unknown>) : {}) ?? {};
+  const str = (k: string): string => {
+    const v = fm[k];
+    return v == null ? "" : String(v);
+  };
+  const rawTags = fm.tags;
+  const tagSlugs =
+    Array.isArray(rawTags)
+      ? rawTags.map((t) => slugifyTag(String(t))).filter(Boolean)
+      : typeof rawTags === "string"
+        ? [slugifyTag(rawTags)].filter(Boolean)
+        : [];
+  return {
+    docType: str("type"),
+    title: str("title"),
+    timestamp: str("timestamp"),
+    tags: tagSlugs.join(","),
+    skip: fm.isSystem === true,
+  };
+}
+
+function chunkBody(body: string, meta: Meta): { text: string; path: string; slug: string }[] {
+  const tree = unified().use(remarkParse).use(remarkGfm).parse(body) as Root;
+  const ledger = new SlugLedger();
+  const out: { text: string; path: string; slug: string }[] = [];
+
+  interface Section {
+    heading: Heading | null;
+    body: RootContent[];
+  }
+  const sections: Section[] = [];
+  let current: Section = { heading: null, body: [] };
+  for (const node of tree.children) {
+    if (node.type === "heading" && (node.depth === 2 || node.depth === 3)) {
+      if (current.heading || current.body.length) sections.push(current);
+      current = { heading: node, body: [] };
+    } else if (node.type === "heading" && node.depth === 1) {
+      if (current.heading || current.body.length) sections.push(current);
+      current = { heading: null, body: [] };
+    } else {
+      current.body.push(node);
+    }
+  }
+  if (current.heading || current.body.length) sections.push(current);
+
+  let lastH2 = "";
+  for (const sec of sections) {
+    const headingText = sec.heading ? nodeToText(sec.heading).trim() : "";
+    const slug = headingText ? ledger.next(headingText) : "";
+    let path: string;
+    if (sec.heading?.depth === 2) {
+      lastH2 = headingText;
+      path = [meta.title, headingText].filter(Boolean).join(" / ");
+    } else if (sec.heading?.depth === 3) {
+      path = [meta.title, lastH2, headingText].filter(Boolean).join(" / ");
+    } else {
+      path = meta.title;
+    }
+
+    const bodyText = sec.body.map(blockToText).filter(Boolean).join("\n\n").trim();
+    if (bodyText.length < MIN_BODY) continue;
+
+    for (const part of splitOversized(bodyText)) {
+      if (part.trim().length < MIN_BODY) continue;
+      out.push({ text: part.trim(), path, slug });
+    }
+  }
+  return out;
+}
+
+function isNotFoundPath(relPath: string): boolean {
+  const base = relPath.replace(/\\/g, "/").split("/").pop() ?? relPath;
+  return base.replace(/\.md$/i, "") === "404";
+}
+
+/** 1 文書を検索チャンク列へ。 */
+export function chunkDocument(source: string, relPath: string): Chunk[] {
+  const { frontmatter, body } = extract(source);
+  const meta = readMeta(frontmatter);
+  if (meta.skip || isNotFoundPath(relPath)) return [];
+
+  const raw = chunkBody(body, meta);
+  return raw.map((c, i) => ({
+    source: relPath,
+    chunkIndex: i,
+    text: c.text,
+    headingPath: c.path,
+    headingSlug: c.slug,
+    docType: meta.docType,
+    title: meta.title,
+    timestamp: meta.timestamp,
+    tags: meta.tags,
+  }));
+}

package/src/derive-web-index.ts

@@ -0,0 +1,59 @@
+import { existsSync, writeFileSync } from "node:fs";
+import { buildSourceDisclosureMap } from "./disclosure-map.ts";
+import { buildFtsWebIndex, buildWebIndex, defaultSourceUrl } from "./web-export.ts";
+
+export type WebSearchMode = "fts" | "hybrid";
+
+export interface DeriveResult {
+  readonly written: boolean;
+  readonly chunks: number;
+  readonly bytes: number;
+  readonly mode?: WebSearchMode;
+}
+
+export async function deriveWebIndex(
+  dbPath: string,
+  outPath: string,
+  sourceToUrl: (source: string) => string = defaultSourceUrl,
+  mode: WebSearchMode = "fts",
+  opts?: {
+    readonly contentDir?: string;
+    readonly machineReadable?: boolean;
+  },
+): Promise<DeriveResult> {
+  if (!existsSync(dbPath)) return { written: false, chunks: 0, bytes: 0 };
+  const { IndexStore } = await import("./store.ts");
+  const store = new IndexStore(dbPath);
+  try {
+    const counts = store.counts();
+    if (counts.chunks === 0) return { written: false, chunks: 0, bytes: 0 };
+
+    const { rows, vectors } = store.exportAll();
+    const disclosureMap =
+      opts?.contentDir && opts.machineReadable !== false
+        ? buildSourceDisclosureMap(
+            opts.contentDir,
+            rows.map((r) => r.source),
+          )
+        : undefined;
+    const exportOpts = {
+      disclosureMap,
+      machineReadable: opts?.machineReadable,
+    };
+
+    if (mode === "hybrid" && store.hasVectors()) {
+      const meta = store.readMeta();
+      const index = buildWebIndex(rows, vectors, meta, sourceToUrl, exportOpts);
+      const json = JSON.stringify(index);
+      writeFileSync(outPath, json, "utf8");
+      return { written: true, chunks: index.chunks.length, bytes: json.length, mode: "hybrid" };
+    }
+
+    const index = buildFtsWebIndex(rows, sourceToUrl, exportOpts);
+    const json = JSON.stringify(index);
+    writeFileSync(outPath, json, "utf8");
+    return { written: true, chunks: index.chunks.length, bytes: json.length, mode: "fts" };
+  } finally {
+    store.close();
+  }
+}

package/src/disclosure-map.ts

@@ -0,0 +1,42 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { resolveDigitalSourceType } from "@sorane/okf";
+
+const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---/;
+
+function parseFrontmatterScalar(block: string, key: string): string | undefined {
+  const re = new RegExp(`^${key}:\\s*(.+)$`, "m");
+  const m = block.match(re);
+  if (!m) return undefined;
+  const raw = m[1]!.trim();
+  if (
+    (raw.startsWith('"') && raw.endsWith('"')) ||
+    (raw.startsWith("'") && raw.endsWith("'"))
+  ) {
+    return raw.slice(1, -1);
+  }
+  return raw.length > 0 ? raw : undefined;
+}
+
+/** contentDir 内のソースパスから IPTC URI の disclosure マップを構築する。 */
+export function buildSourceDisclosureMap(
+  contentDir: string,
+  sources: readonly string[],
+): ReadonlyMap<string, string> {
+  const map = new Map<string, string>();
+  const seen = new Set<string>();
+  for (const source of sources) {
+    if (seen.has(source)) continue;
+    seen.add(source);
+    const abs = join(contentDir, source);
+    if (!existsSync(abs)) continue;
+    const text = readFileSync(abs, "utf8");
+    const fm = FRONTMATTER_RE.exec(text);
+    if (!fm) continue;
+    const dst = parseFrontmatterScalar(fm[1]!, "digitalSourceType");
+    if (!dst) continue;
+    const resolved = resolveDigitalSourceType(dst);
+    if (resolved) map.set(source, resolved.uri);
+  }
+  return map;
+}