md-semantic-search 0.3.0 → 0.4.0

package/src/frontmatter.mjs

@@ -0,0 +1,213 @@
+// @ts-check
+/**
+ * Typed YAML Frontmatter parser and metadata normalizer (issue #58).
+ * Parses YAML frontmatter into a validated document metadata object without
+ * unsafe executable tags. Supports scalars, inline/block lists, quotes, booleans,
+ * dates, tags normalization, and canonical document identity.
+ */
+
+/**
+ * @typedef {{
+ *   title?: string,
+ *   aliases: string[],
+ *   tags: string[],
+ *   project?: string,
+ *   type?: string,
+ *   status?: string,
+ *   canonical?: boolean,
+ *   canonicalRef?: string,
+ *   created?: string,
+ *   updated?: string,
+ *   custom: Record<string, string | number | boolean | string[]>,
+ * }} DocumentMetadata
+ */
+
+/**
+ * Parse raw YAML frontmatter text into a typed DocumentMetadata object.
+ * @param {string} rawYaml
+ * @returns {DocumentMetadata}
+ */
+export function parseFrontmatter(rawYaml) {
+  const metadata = {
+    aliases: /** @type {string[]} */ ([]),
+    tags: /** @type {string[]} */ ([]),
+    custom: /** @type {Record<string, string | number | boolean | string[]>} */ ({}),
+  };
+
+  if (!rawYaml || typeof rawYaml !== 'string') {
+    return metadata;
+  }
+
+  const lines = rawYaml.split(/\r?\n/);
+  let currentKey = null;
+  let currentBlockList = null;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const trimmed = line.trim();
+
+    if (!trimmed || trimmed.startsWith('#')) {
+      continue;
+    }
+
+    // Block list item: "  - item"
+    const blockListMatch = line.match(/^(\s+)-\s+(.+)$/);
+    if (blockListMatch && currentKey && currentBlockList) {
+      const val = parseYAMLValue(blockListMatch[2]);
+      if (typeof val === 'string' || typeof val === 'number' || typeof val === 'boolean') {
+        currentBlockList.push(String(val));
+      }
+      continue;
+    }
+
+    // Key-value pair: "key: value" or "key:"
+    const kvMatch = line.match(/^([a-zA-Z0-9_-]+):\s*(.*)$/);
+    if (kvMatch) {
+      if (currentKey && currentBlockList) {
+        setMetadataValue(metadata, currentKey, currentBlockList);
+        currentKey = null;
+        currentBlockList = null;
+      }
+
+      const key = kvMatch[1].trim();
+      const rawVal = kvMatch[2].trim();
+
+      if (rawVal === '') {
+        // Start of potential block list
+        currentKey = key;
+        currentBlockList = [];
+      } else {
+        const val = parseYAMLValue(rawVal);
+        setMetadataValue(metadata, key, val);
+      }
+    }
+  }
+
+  if (currentKey && currentBlockList) {
+    setMetadataValue(metadata, currentKey, currentBlockList);
+  }
+
+  return metadata;
+}
+
+/**
+ * Parse a raw YAML value string (supports quotes, inline arrays, booleans, numbers).
+ * @param {string} raw
+ * @returns {string | number | boolean | string[]}
+ */
+function parseYAMLValue(raw) {
+  const trimmed = raw.trim();
+
+  // Inline array: [item1, item2]
+  if (trimmed.startsWith('[') && trimmed.endsWith(']')) {
+    const inner = trimmed.slice(1, -1).trim();
+    if (!inner) return [];
+    return inner
+      .split(',')
+      .map((item) => parseYAMLValue(item))
+      .map((item) => (Array.isArray(item) ? item.join(' ') : String(item).trim()))
+      .filter((item) => item.length > 0);
+  }
+
+  // Quoted string
+  if ((trimmed.startsWith('"') && trimmed.endsWith('"')) || (trimmed.startsWith("'") && trimmed.endsWith("'"))) {
+    return trimmed.slice(1, -1).trim();
+  }
+
+  // Boolean
+  if (/^true$/i.test(trimmed)) return true;
+  if (/^false$/i.test(trimmed)) return false;
+
+  // Number
+  if (/^-?\d+(\.\d+)?$/.test(trimmed)) {
+    const num = Number(trimmed);
+    if (!Number.isNaN(num)) return num;
+  }
+
+  return trimmed;
+}
+
+/**
+ * Set a parsed key-value pair on the DocumentMetadata object.
+ * @param {DocumentMetadata} meta
+ * @param {string} key
+ * @param {string | number | boolean | string[]} val
+ */
+function setMetadataValue(meta, key, val) {
+  const normKey = key.toLowerCase().replace(/[-_]/g, '');
+
+  if (normKey === 'title' && typeof val === 'string') {
+    meta.title = val;
+    return;
+  }
+
+  if (normKey === 'alias' || normKey === 'aliases') {
+    const items = Array.isArray(val) ? val : [String(val)];
+    for (const item of items) {
+      const cleaned = item.trim().replace(/^["']|["']$/g, '');
+      if (cleaned && !meta.aliases.includes(cleaned)) {
+        meta.aliases.push(cleaned);
+      }
+    }
+    return;
+  }
+
+  if (normKey === 'tag' || normKey === 'tags') {
+    let items = [];
+    if (Array.isArray(val)) {
+      items = val;
+    } else if (typeof val === 'string') {
+      items = val.includes(',') ? val.split(',') : [val];
+    } else {
+      items = [String(val)];
+    }
+    for (const item of items) {
+      const cleaned = String(item)
+        .trim()
+        .replace(/^#/, '')
+        .toLowerCase();
+      if (cleaned && !meta.tags.includes(cleaned)) {
+        meta.tags.push(cleaned);
+      }
+    }
+    return;
+  }
+
+  if (normKey === 'project' && (typeof val === 'string' || typeof val === 'number')) {
+    meta.project = String(val);
+    return;
+  }
+
+  if (normKey === 'type' && typeof val === 'string') {
+    meta.type = val;
+    return;
+  }
+
+  if (normKey === 'status' && typeof val === 'string') {
+    meta.status = val;
+    return;
+  }
+
+  if (normKey === 'canonical') {
+    meta.canonical = Boolean(val);
+    return;
+  }
+
+  if ((normKey === 'canonicalref' || normKey === 'canonical_ref') && typeof val === 'string') {
+    meta.canonicalRef = val.replace(/^\[\[(.*)\]\]$/, '$1').trim();
+    return;
+  }
+
+  if (normKey === 'created') {
+    meta.created = String(val);
+    return;
+  }
+
+  if (normKey === 'updated') {
+    meta.updated = String(val);
+    return;
+  }
+
+  // Store custom property
+  meta.custom[key] = val;
+}

package/src/index-format.mjs

@@ -0,0 +1,134 @@
+// @ts-check
+import { decodeVec, parseSchemaVersion, resolveIndexDimension, SCHEMA_VERSION } from './core.mjs';
+import { validateLexicalIndex } from './lexical.mjs';
+import { resolveModel } from './models.mjs';
+
+const REBUILD = ' — run `mdss index` to rebuild';
+
+/** @param {unknown} value @param {string} location */
+export function inspectIndexSchema(value, location) {
+  if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+    throw new Error(`${location} root must be an object${REBUILD}`);
+  }
+  const index = /** @type {Record<string, unknown>} */ (value);
+  return { index, schema: parseSchemaVersion(index.schemaVersion) };
+}
+
+/** @param {unknown} value @param {number|undefined} expectedDim @param {string} where */
+export function validateNumericVector(value, expectedDim, where) {
+  if (!Array.isArray(value) && !(value instanceof Float32Array)) {
+    throw new Error(`${where}: missing vector${REBUILD}`);
+  }
+  if (value.length === 0) throw new Error(`${where}: vector must not be empty${REBUILD}`);
+  if (expectedDim !== undefined && value.length !== expectedDim) {
+    throw new Error(`${where}: vector has ${value.length} dims, expected ${expectedDim}${REBUILD}`);
+  }
+  for (let position = 0; position < value.length; position++) {
+    if (Array.isArray(value) && !Object.hasOwn(value, position)) {
+      throw new Error(`${where}: missing vector component at ${position}${REBUILD}`);
+    }
+    if (typeof value[position] !== 'number' || !Number.isFinite(value[position])) {
+      throw new Error(`${where}: non-finite vector value${REBUILD}`);
+    }
+  }
+  return value.length;
+}
+
+/** @param {unknown} chunk @param {number} position @param {{dim:number|undefined, encoding:'stored'|'loaded'|'none', allowMissingVector:(boolean|undefined)}} options */
+export function validateCurrentChunk(chunk, position, options) {
+  const prefix = `chunk ${position}`;
+  if (chunk === null || typeof chunk !== 'object' || Array.isArray(chunk)) {
+    throw new Error(`${prefix} must be an object${REBUILD}`);
+  }
+  const value = /** @type {Record<string, unknown>} */ (chunk);
+  for (const field of ['file', 'title', 'heading', 'text', 'chunkHash']) {
+    if (typeof value[field] !== 'string') {
+      throw new Error(`${prefix} ${field} must be a string${REBUILD}`);
+    }
+  }
+  if (!Array.isArray(value.headingPath) || value.headingPath.length === 0) {
+    throw new Error(`${prefix} headingPath must contain nonempty strings${REBUILD}`);
+  }
+  for (let pathPosition = 0; pathPosition < value.headingPath.length; pathPosition++) {
+    if (!Object.hasOwn(value.headingPath, pathPosition) ||
+        typeof value.headingPath[pathPosition] !== 'string' ||
+        value.headingPath[pathPosition].trim().length === 0) {
+      throw new Error(`${prefix} headingPath must contain nonempty strings${REBUILD}`);
+    }
+  }
+  if (value.headingPath[value.headingPath.length - 1] !== value.heading) {
+    throw new Error(`${prefix} headingPath leaf must equal heading${REBUILD}`);
+  }
+  if (options.encoding === 'none') return;
+  const vectorWhere = `${prefix}; chunk ${value.file}` +
+    (value.heading ? ` › ${value.heading}` : '');
+  if (value.vec === undefined && options.allowMissingVector) return;
+  if (options.encoding === 'stored') {
+    if (typeof value.vec !== 'string') {
+      throw new Error(`${prefix} vec must be canonical base64${REBUILD}`);
+    }
+    try {
+      decodeVec(value.vec, options.dim);
+    } catch (error) {
+      throw new Error(`${vectorWhere}: ${error.message}`);
+    }
+    return;
+  }
+  validateNumericVector(value.vec, options.dim, vectorWhere);
+}
+
+/**
+ * Shared vectors.json boundary for loader, diagnostics, stats, and recovery.
+ * Legacy schemas intentionally keep their historical loose chunk metadata.
+ * @param {unknown} value
+ * @param {string} location
+ * @param {{encoding?:'stored'|'loaded', validateVectors?:boolean,
+ *   allowMissingVectors?:boolean, validateLexical?:boolean}} [options]
+ */
+export function validateIndexEnvelope(value, location, options = {}) {
+  const { index, schema } = inspectIndexSchema(value, location);
+  if (schema > SCHEMA_VERSION) {
+    throw new Error(`${location} uses schema v${schema}, newer than this mdss supports; ` +
+      `supports up to v${SCHEMA_VERSION} — upgrade md-semantic-search`);
+  }
+  if (!Array.isArray(index.chunks)) {
+    throw new Error(`${location} chunks must be an array${REBUILD}`);
+  }
+  let modelName = index.model ?? index.modelAlias;
+  if (schema >= 3) {
+    if (typeof index.model !== 'string' || index.model.trim().length === 0) {
+      throw new Error(`schema v3 model must be a nonempty string${REBUILD}`);
+    }
+    modelName = index.model;
+  } else if (modelName !== undefined && modelName !== null && typeof modelName !== 'string') {
+    throw new Error(`${location} model metadata must be a string${REBUILD}`);
+  }
+  const model = resolveModel(/** @type {string|undefined} */ (modelName ?? undefined));
+  const dim = resolveIndexDimension(index.dim, model.dim);
+  if (schema < 3) {
+    return { index, schema, model, dim };
+  }
+  if (index.format !== 'binary-v1') {
+    throw new Error(`schema v3 format must be binary-v1${REBUILD}`);
+  }
+  if (typeof index.chunkCount !== 'number' || !Number.isSafeInteger(index.chunkCount) || index.chunkCount < 0 ||
+      index.chunkCount !== index.chunks.length) {
+    throw new Error(`schema v3 chunkCount must equal chunks.length${REBUILD}`);
+  }
+  if (index.chunks.length > 0 && dim === undefined) {
+    throw new Error(`nonempty schema v3 custom index requires explicit dim${REBUILD}`);
+  }
+  const encoding = options.validateVectors === false ? 'none' : (options.encoding ?? 'stored');
+  for (let position = 0; position < index.chunks.length; position++) {
+    validateCurrentChunk(index.chunks[position], position, {
+      dim, encoding, allowMissingVector: options.allowMissingVectors,
+    });
+  }
+  if (options.validateLexical !== false) {
+    const lexicalError = validateLexicalIndex(index.lexical, index.chunks.length);
+    if (lexicalError) {
+      throw new Error(`schema v3 lexical index is invalid (${lexicalError})${REBUILD}`);
+    }
+  }
+  return { index, schema, model, dim };
+}

package/src/index.mjs

@@ -0,0 +1,47 @@
+// @ts-check
+/**
+ * Public library API for md-semantic-search.
+ *
+ * Import from a consumer project:
+ *
+ *   import { buildIndex, search, loadIndex, searchIndex, resolveModel, MODELS } from 'md-semantic-search';
+ *
+ * One-shot usage (parses the index on every call):
+ *
+ *   await buildIndex({ db: './docs', indexDir: './.mdss', cacheDir: '~/.cache/mdss' });
+ *   const hits = await search({ indexDir: './.mdss', cacheDir: '~/.cache/mdss', query: 'rotate api token' });
+ *
+ * Repeated queries in one process (parse + model cached, issue #2):
+ *
+ *   const loaded = loadIndex('./.mdss');
+ *   const hitsA = await searchIndex({ loaded, cacheDir: '~/.cache/mdss', query: 'failover runbook' });
+ *   const hitsB = await searchIndex({ loaded, cacheDir: '~/.cache/mdss', query: 'db backup' });
+ *
+ * All functions are model- and path-agnostic: they take explicit paths, so the
+ * same API works on any folder of markdown anywhere on disk.
+ */
+
+export {
+  buildIndex, chunkHash,
+} from './indexer.mjs';
+
+export {
+  search, searchIndex, loadIndex,
+  tokenize, keywordScores, rrf, QueryEmbeddingCache,
+} from './search.mjs';
+
+export {
+  resolveModel, MODELS, DEFAULT_MODEL,
+  normalizeAdapter, embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint,
+} from './models.mjs';
+
+export {
+  getReranker, rerankScores, RERANK_MODEL,
+} from './rerank.mjs';
+
+export {
+  embed, getExtractor, cosine,
+  walkMarkdown, parseFile, chunkMarkdown,
+  splitFrontmatter, extractTitle, globToRegExp,
+  encodeVec, decodeVec, isBinaryIndex,
+} from './core.mjs';