md-semantic-search 0.5.0 → 1.0.2

package/src/core.ts

@@ -9,8 +9,9 @@ import os from 'node:os';
 import { resolveModel, ModelAdapter, Pooling, DType } from './models.js';
 import { chunkMarkdownStructural, PARSER_VERSION, StructuralChunk } from './markdown-parser.js';
 import { parseFrontmatter, DocumentMetadata } from './frontmatter.js';
+import { asymmetricCosineInt8 } from './quantization.js';
 
-export { PARSER_VERSION, parseFrontmatter, resolveModel, DocumentMetadata, ModelAdapter, Pooling, DType };
+export { PARSER_VERSION, parseFrontmatter, resolveModel, DocumentMetadata, ModelAdapter, Pooling, DType, asymmetricCosineInt8 };
 
 const _extractors = new Map<string, any>();
 
@@ -287,7 +288,17 @@ export async function embed(
   return out.tolist();
 }
 
-export function cosine(a: Float32Array | number[], b: Float32Array | number[], bOffset: number = 0): number {
+export function cosine(
+  a: Float32Array | number[],
+  b: Float32Array | Int8Array | number[],
+  bOffset: number = 0
+): number {
+  if (b instanceof Int8Array) {
+    const qF32 = a instanceof Float32Array ? a : Float32Array.from(a);
+    const chunkInt8 = bOffset === 0 && b.length === qF32.length ? b : b.subarray(bOffset, bOffset + qF32.length);
+    return asymmetricCosineInt8(qF32, chunkInt8);
+  }
+
   const len = a.length;
   let s0 = 0,
     s1 = 0,
@@ -300,19 +311,19 @@ export function cosine(a: Float32Array | number[], b: Float32Array | number[], b
   let i = 0;
   const limit = len - 7;
   while (i < limit) {
-    s0 += a[i] * b[bOffset + i];
-    s1 += a[i + 1] * b[bOffset + i + 1];
-    s2 += a[i + 2] * b[bOffset + i + 2];
-    s3 += a[i + 3] * b[bOffset + i + 3];
-    s4 += a[i + 4] * b[bOffset + i + 4];
-    s5 += a[i + 5] * b[bOffset + i + 5];
-    s6 += a[i + 6] * b[bOffset + i + 6];
-    s7 += a[i + 7] * b[bOffset + i + 7];
+    s0 += a[i] * (b as ArrayLike<number>)[bOffset + i];
+    s1 += a[i + 1] * (b as ArrayLike<number>)[bOffset + i + 1];
+    s2 += a[i + 2] * (b as ArrayLike<number>)[bOffset + i + 2];
+    s3 += a[i + 3] * (b as ArrayLike<number>)[bOffset + i + 3];
+    s4 += a[i + 4] * (b as ArrayLike<number>)[bOffset + i + 4];
+    s5 += a[i + 5] * (b as ArrayLike<number>)[bOffset + i + 5];
+    s6 += a[i + 6] * (b as ArrayLike<number>)[bOffset + i + 6];
+    s7 += a[i + 7] * (b as ArrayLike<number>)[bOffset + i + 7];
     i += 8;
   }
   let s = s0 + s1 + (s2 + s3) + (s4 + s5) + (s6 + s7);
   while (i < len) {
-    s += a[i] * b[bOffset + i];
+    s += a[i] * (b as ArrayLike<number>)[bOffset + i];
     i++;
   }
   return s;
@@ -419,6 +430,15 @@ export function assertSafePath(targetPath: string, allowedRoots?: string[]): str
   if (typeof targetPath !== 'string' || !targetPath.trim()) {
     throw new Error('path must be a non-empty string');
   }
+  if (targetPath.includes('\0')) {
+    throw new Error('Forbidden path: null byte detected in path');
+  }
+  if (targetPath.startsWith('\\\\') || targetPath.startsWith('//')) {
+    throw new Error('Forbidden path: UNC network paths not allowed');
+  }
+  if (/^[a-zA-Z]:[\\/]/.test(targetPath) && process.platform !== 'win32') {
+    throw new Error('Forbidden path: Windows drive paths not allowed on non-Windows systems');
+  }
   const resolved = path.resolve(targetPath);
   let canonical = resolved;
   try {
@@ -431,36 +451,24 @@ export function assertSafePath(targetPath: string, allowedRoots?: string[]): str
   if (!roots || roots.length === 0) {
     if (process.env.MDSS_ROOT_GUARD) {
       roots = process.env.MDSS_ROOT_GUARD.split(path.delimiter).map((p) => path.resolve(p.trim())).filter(Boolean);
-    } else {
-      roots = [path.resolve(process.cwd())];
-      try {
-        const parent = path.resolve(process.cwd(), '..');
-        if (parent) roots.push(parent);
-      } catch {}
-      try {
-        const home = os.homedir();
-        if (home) roots.push(path.resolve(home));
-      } catch {}
-      try {
-        const tmp = os.tmpdir();
-        if (tmp) roots.push(path.resolve(tmp));
-      } catch {}
     }
   }
 
-  const isAllowed = roots.some((root) => {
-    let canonicalRoot = root;
-    try {
-      canonicalRoot = fs.realpathSync(root);
-    } catch {}
-    const rel = path.relative(canonicalRoot, canonical);
-    return !rel.startsWith('..') && !path.isAbsolute(rel);
-  });
+  if (roots && roots.length > 0) {
+    const isAllowed = roots.some((root) => {
+      let canonicalRoot = root;
+      try {
+        canonicalRoot = fs.realpathSync(root);
+      } catch {}
+      const rel = path.relative(canonicalRoot, canonical);
+      return !rel.startsWith('..') && !path.isAbsolute(rel);
+    });
 
-  if (!isAllowed) {
-    throw new Error(
-      `path traversal guard: "${targetPath}" (resolving to "${canonical}") is outside allowed directory roots`
-    );
+    if (!isAllowed) {
+      throw new Error(
+        `path traversal guard: "${targetPath}" (resolving to "${canonical}") is outside allowed directory roots`
+      );
+    }
   }
 
   return canonical;

package/src/federation.ts

@@ -0,0 +1,118 @@
+/**
+ * Multi-Vault & Multi-Repository Search Federation (issue #97).
+ *
+ * Coordinates querying across multiple independent vaults / repositories,
+ * executing per-vault index loads in parallel and merging candidates with
+ * calibrated reciprocal rank fusion (RRF) and vault attribution.
+ */
+import path from 'node:path';
+import fs from 'node:fs';
+import { loadIndex, searchIndex, LoadedIndex, SearchOptions, SearchResultHit } from './search.js';
+
+export interface VaultConfig {
+  name?: string;
+  path: string;
+  indexDir?: string;
+}
+
+export interface FederatedSearchOptions extends Omit<SearchOptions, 'loaded' | 'cacheDir'> {
+  vaults: Array<string | VaultConfig>;
+  cacheDir?: string;
+}
+
+export interface FederatedSearchResultHit extends SearchResultHit {
+  vault: string;
+  vaultPath: string;
+}
+
+export async function searchFederated(opts: FederatedSearchOptions): Promise<FederatedSearchResultHit[]> {
+  const { vaults, k = 10, cacheDir = '', ...searchOpts } = opts;
+  if (!vaults || vaults.length === 0) {
+    throw new Error('Federated search requires at least one vault specified');
+  }
+
+  const normalizedVaults: VaultConfig[] = vaults.map((v) => {
+    if (typeof v === 'string') {
+      const absPath = path.resolve(v);
+      const name = path.basename(absPath);
+      return { name, path: absPath };
+    }
+    const absPath = path.resolve(v.path);
+    const name = v.name || path.basename(absPath);
+    return { name, path: absPath, indexDir: v.indexDir };
+  });
+
+  // Load and query each vault concurrently
+  const perVaultResults = await Promise.all(
+    normalizedVaults.map(async (vault) => {
+      const indexDir = vault.indexDir || path.join(vault.path, '.mdss');
+      if (!fs.existsSync(indexDir)) {
+        return { vault, hits: [] as SearchResultHit[], error: `No index at ${indexDir}` };
+      }
+
+      let loaded: LoadedIndex;
+      try {
+        loaded = loadIndex(indexDir);
+      } catch (err: any) {
+        return { vault, hits: [] as SearchResultHit[], error: err.message };
+      }
+
+      try {
+        // Query vault index with a slightly larger pool to ensure healthy global fusion
+        const poolK = Math.max(k * 2, 20);
+        const resolvedCache = cacheDir || path.join(vault.path, '.cache');
+        const hits = await searchIndex({
+          ...searchOpts,
+          cacheDir: resolvedCache,
+          loaded,
+          k: poolK,
+        });
+        return { vault, hits, error: null };
+      } catch (err: any) {
+        return { vault, hits: [] as SearchResultHit[], error: err.message };
+      }
+    })
+  );
+
+  // If single vault, return directly with attribution
+  if (normalizedVaults.length === 1) {
+    const { vault, hits } = perVaultResults[0];
+    return hits.slice(0, k).map((h) => ({
+      ...h,
+      vault: vault.name || path.basename(vault.path),
+      vaultPath: vault.path,
+    }));
+  }
+
+  // Multi-vault reciprocal rank fusion (RRF)
+  // Collect all hits across vaults
+  const hitKeys = new Map<string, { hit: SearchResultHit; vault: VaultConfig; rrfSum: number }>();
+
+  for (const { vault, hits } of perVaultResults) {
+    hits.forEach((hit, rank) => {
+      const key = `${vault.path}::${hit.file}::${hit.heading || ''}`;
+      const rrfScore = 1.0 / (60 + rank + 1);
+      const existing = hitKeys.get(key);
+      if (!existing) {
+        hitKeys.set(key, { hit, vault, rrfSum: rrfScore });
+      } else {
+        existing.rrfSum += rrfScore;
+      }
+    });
+  }
+
+  // Sort fused hits by combined rrfSum (or fallback to score)
+  const fused = Array.from(hitKeys.values())
+    .sort((a, b) => {
+      if (b.rrfSum !== a.rrfSum) return b.rrfSum - a.rrfSum;
+      return b.hit.score - a.hit.score;
+    })
+    .slice(0, k);
+
+  return fused.map(({ hit, vault, rrfSum }) => ({
+    ...hit,
+    score: rrfSum,
+    vault: vault.name || path.basename(vault.path),
+    vaultPath: vault.path,
+  }));
+}

package/src/filter.ts

@@ -0,0 +1,368 @@
+/**
+ * Rich Frontmatter & Tag Filter Expression DSL (issue #94).
+ * Supports boolean expressions (AND, OR, NOT, grouping parentheses)
+ * across frontmatter tags, properties, dates, and custom fields.
+ */
+import { DocumentMetadata } from './frontmatter.js';
+
+export type FilterNode =
+  | { type: 'AND'; left: FilterNode; right: FilterNode }
+  | { type: 'OR'; left: FilterNode; right: FilterNode }
+  | { type: 'NOT'; child: FilterNode }
+  | {
+      type: 'COMPARE';
+      field: string;
+      op: ':' | 'contains' | '=' | '==' | '!=' | '>' | '>=' | '<' | '<=';
+      value: string | number | boolean;
+    };
+
+export function tokenizeFilter(expr: string): string[] {
+  const tokens: string[] = [];
+  let i = 0;
+  while (i < expr.length) {
+    const ch = expr[i];
+
+    if (/\s/.test(ch)) {
+      i++;
+      continue;
+    }
+
+    if (ch === '(' || ch === ')') {
+      tokens.push(ch);
+      i++;
+      continue;
+    }
+
+    // Quoted string
+    if (ch === '"' || ch === "'") {
+      const quote = ch;
+      let str = '';
+      i++;
+      while (i < expr.length && expr[i] !== quote) {
+        if (expr[i] === '\\' && i + 1 < expr.length) {
+          str += expr[i + 1];
+          i += 2;
+        } else {
+          str += expr[i];
+          i++;
+        }
+      }
+      i++; // skip closing quote
+      tokens.push(`"${str}"`);
+      continue;
+    }
+
+    // Two-character operators: >=, <=, !=, ==
+    if (i + 1 < expr.length) {
+      const two = expr.slice(i, i + 2);
+      if (two === '>=' || two === '<=' || two === '!=' || two === '==') {
+        tokens.push(two);
+        i += 2;
+        continue;
+      }
+    }
+
+    // Single-character operators: >, <, =, :
+    if (ch === '>' || ch === '<' || ch === '=' || ch === ':') {
+      tokens.push(ch);
+      i++;
+      continue;
+    }
+
+    // Word / identifier / value token
+    let word = '';
+    while (
+      i < expr.length &&
+      !/\s/.test(expr[i]) &&
+      expr[i] !== '(' &&
+      expr[i] !== ')' &&
+      expr[i] !== ':' &&
+      expr[i] !== '=' &&
+      expr[i] !== '!' &&
+      expr[i] !== '>' &&
+      expr[i] !== '<' &&
+      expr[i] !== '"' &&
+      expr[i] !== "'"
+    ) {
+      word += expr[i];
+      i++;
+    }
+    if (word) {
+      tokens.push(word);
+    } else if (i < expr.length) {
+      tokens.push(expr[i]);
+      i++;
+    }
+  }
+  return tokens;
+}
+
+export function parseFilter(expr: string): FilterNode {
+  const tokens = tokenizeFilter(expr);
+  if (tokens.length === 0) {
+    throw new Error('Empty filter expression');
+  }
+
+  let pos = 0;
+
+  function peek(): string | undefined {
+    return tokens[pos];
+  }
+
+  function consume(): string {
+    return tokens[pos++];
+  }
+
+  function parseExpression(): FilterNode {
+    return parseOr();
+  }
+
+  function parseOr(): FilterNode {
+    let node = parseAnd();
+    while (pos < tokens.length && peek()?.toUpperCase() === 'OR') {
+      consume(); // OR
+      const beforePos = pos;
+      const right = parseAnd();
+      if (pos === beforePos) {
+        throw new Error(`Invalid syntax in filter expression near token "${peek()}"`);
+      }
+      node = { type: 'OR', left: node, right };
+    }
+    return node;
+  }
+
+  function parseAnd(): FilterNode {
+    let node = parseNot();
+    while (
+      pos < tokens.length &&
+      peek()?.toUpperCase() !== 'OR' &&
+      peek() !== ')'
+    ) {
+      if (peek()?.toUpperCase() === 'AND') {
+        consume(); // AND
+      }
+      const beforePos = pos;
+      const right = parseNot();
+      if (pos === beforePos) {
+        throw new Error(`Invalid syntax in filter expression near token "${peek()}"`);
+      }
+      node = { type: 'AND', left: node, right };
+    }
+    return node;
+  }
+
+  function parseNot(): FilterNode {
+    if (pos < tokens.length && (peek()?.toUpperCase() === 'NOT' || peek() === '!')) {
+      consume(); // NOT / !
+      const child = parsePrimary();
+      return { type: 'NOT', child };
+    }
+    return parsePrimary();
+  }
+
+  function parsePrimary(): FilterNode {
+    const token = peek();
+    if (!token) {
+      throw new Error(`Unexpected end of filter expression at position ${pos}`);
+    }
+
+    if (token === '(') {
+      consume(); // (
+      const node = parseExpression();
+      if (peek() !== ')') {
+        throw new Error('Expected closing parenthesis ")" in filter expression');
+      }
+      consume(); // )
+      return node;
+    }
+
+    // Comparison term: <field> <op> <value> or <field>:<value>
+    const fieldToken = consume();
+    const cleanField = fieldToken.replace(/^["']|["']$/g, '');
+
+    let op: any = ':';
+    let rawVal = '';
+
+    const nextTok = peek();
+    if (
+      nextTok === ':' ||
+      nextTok === '=' ||
+      nextTok === '==' ||
+      nextTok === '!=' ||
+      nextTok === '>' ||
+      nextTok === '>=' ||
+      nextTok === '<' ||
+      nextTok === '<=' ||
+      nextTok?.toLowerCase() === 'contains'
+    ) {
+      op = consume();
+      if (op === '==') op = '=';
+      rawVal = consume();
+      if (!rawVal) {
+        throw new Error(`Missing value after operator "${op}" for field "${cleanField}"`);
+      }
+    } else {
+      op = ':';
+      rawVal = cleanField;
+    }
+
+    const unquoted = rawVal.replace(/^["']|["']$/g, '');
+    let parsedVal: string | number | boolean = unquoted;
+    if (/^true$/i.test(unquoted)) parsedVal = true;
+    else if (/^false$/i.test(unquoted)) parsedVal = false;
+    else if (/^-?\d+(\.\d+)?$/.test(unquoted)) parsedVal = Number(unquoted);
+
+    return {
+      type: 'COMPARE',
+      field: cleanField,
+      op,
+      value: parsedVal,
+    };
+  }
+
+  const result = parseExpression();
+  if (pos < tokens.length) {
+    throw new Error(`Unexpected trailing token in filter: "${tokens[pos]}"`);
+  }
+  return result;
+}
+
+export function evaluateFilter(
+  node: FilterNode | string,
+  meta?: DocumentMetadata,
+  doc?: { file?: string; title?: string }
+): boolean {
+  if (typeof node === 'string') {
+    if (!node.trim()) return true;
+    try {
+      node = parseFilter(node);
+    } catch {
+      return false;
+    }
+  }
+
+  if (node.type === 'AND') {
+    return evaluateFilter(node.left, meta, doc) && evaluateFilter(node.right, meta, doc);
+  }
+
+  if (node.type === 'OR') {
+    return evaluateFilter(node.left, meta, doc) || evaluateFilter(node.right, meta, doc);
+  }
+
+  if (node.type === 'NOT') {
+    return !evaluateFilter(node.child, meta, doc);
+  }
+
+  if (node.type === 'COMPARE') {
+    return evaluateCompare(node.field, node.op, node.value, meta, doc);
+  }
+
+  return true;
+}
+
+function evaluateCompare(
+  field: string,
+  op: string,
+  targetVal: string | number | boolean,
+  meta?: DocumentMetadata,
+  doc?: { file?: string; title?: string }
+): boolean {
+  const normField = field.toLowerCase().replace(/[-_]/g, '');
+
+  let actualVal: any = undefined;
+
+  if (normField === 'file' || normField === 'path') {
+    actualVal = doc?.file;
+  } else if (normField === 'title') {
+    actualVal = meta?.title || doc?.title;
+  } else if (normField === 'tag' || normField === 'tags') {
+    actualVal = meta?.tags || [];
+  } else if (normField === 'alias' || normField === 'aliases') {
+    actualVal = meta?.aliases || [];
+  } else if (normField === 'project') {
+    actualVal = meta?.project;
+  } else if (normField === 'type') {
+    actualVal = meta?.type;
+  } else if (normField === 'status') {
+    actualVal = meta?.status;
+  } else if (normField === 'canonical') {
+    actualVal = meta?.canonical;
+  } else if (normField === 'date' || normField === 'created') {
+    actualVal = meta?.created;
+  } else if (normField === 'updated') {
+    actualVal = meta?.updated;
+  } else if (meta?.custom && field in meta.custom) {
+    actualVal = meta.custom[field];
+  }
+
+  // Tag array membership
+  if (normField === 'tag' || normField === 'tags' || Array.isArray(actualVal)) {
+    const list = Array.isArray(actualVal) ? actualVal : [];
+    const searchTarget = String(targetVal).toLowerCase().replace(/^#/, '');
+
+    if (op === ':' || op === 'contains' || op === '=' || op === '==') {
+      return list.some((item) => String(item).toLowerCase().replace(/^#/, '') === searchTarget);
+    }
+    if (op === '!=') {
+      return !list.some((item) => String(item).toLowerCase().replace(/^#/, '') === searchTarget);
+    }
+  }
+
+  if (actualVal === undefined || actualVal === null) {
+    if (op === '!=') return true;
+    return false;
+  }
+
+  // Boolean compare
+  if (typeof targetVal === 'boolean') {
+    const b = Boolean(actualVal);
+    return op === '!=' ? b !== targetVal : b === targetVal;
+  }
+
+  // Number compare
+  if (typeof targetVal === 'number') {
+    const num = Number(actualVal);
+    if (Number.isNaN(num)) return false;
+    switch (op) {
+      case '=':
+      case ':':
+        return num === targetVal;
+      case '!=':
+        return num !== targetVal;
+      case '>':
+        return num > targetVal;
+      case '>=':
+        return num >= targetVal;
+      case '<':
+        return num < targetVal;
+      case '<=':
+        return num <= targetVal;
+      default:
+        return false;
+    }
+  }
+
+  // String / Date comparison
+  const strActual = String(actualVal).toLowerCase();
+  const strTarget = String(targetVal).toLowerCase();
+
+  switch (op) {
+    case '=':
+      return strActual === strTarget;
+    case ':':
+    case 'contains':
+      return strActual.includes(strTarget);
+    case '!=':
+      return strActual !== strTarget;
+    case '>':
+      return strActual > strTarget;
+    case '>=':
+      return strActual >= strTarget;
+    case '<':
+      return strActual < strTarget;
+    case '<=':
+      return strActual <= strTarget;
+    default:
+      return false;
+  }
+}

package/src/index-format.ts

@@ -13,7 +13,7 @@ export function inspectIndexSchema(value: unknown, location: string): { index: R
 }
 
 export function validateNumericVector(value: unknown, expectedDim: number | undefined, where: string): number {
-  if (!Array.isArray(value) && !(value instanceof Float32Array)) {
+  if (!Array.isArray(value) && !(value instanceof Float32Array || value instanceof Int8Array)) {
     throw new Error(`${where}: missing vector${REBUILD}`);
   }
   const vec = value as ArrayLike<number>;

package/src/index.ts

@@ -52,4 +52,42 @@ export {
   ANN_THRESHOLD, DEFAULT_NPROBE,
 } from './ivf.js';
 
-export { runTui } from './tui.js';
+export {
+  extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes,
+  computePageRank, expandGraphNeighborhood,
+} from './wikilinks.js';
+
+export {
+  parseFilter, tokenizeFilter, evaluateFilter, FilterNode,
+} from './filter.js';
+
+export {
+  serializeBinaryIndex, deserializeBinaryIndex, readBinaryHeader,
+  BINARY_MAGIC, BINARY_HEADER_SIZE,
+} from './binary-format.js';
+
+export {
+  quantizeToInt8, dequantizeFromInt8, asymmetricCosineInt8,
+  INT8_SCALE, INT8_INV_SCALE,
+} from './quantization.js';
+
+export {
+  searchFederated, FederatedSearchOptions, FederatedSearchResultHit, VaultConfig,
+} from './federation.js';
+
+export {
+  createFileWatcher, FileWatcher, WatcherOptions,
+} from './watcher.js';
+
+export {
+  askQuestion, synthesizeAnswer, extractAnswerFallback,
+  AskOptions, SynthesisResult, Citation, DEFAULT_RAG_SYSTEM_PROMPT,
+} from './rag.js';
+
+export {
+  extractKeyphraseTags, summarizePassage, analyzeNote, NoteAnalysis,
+} from './summarize.js';
+
+export { runTui, TuiOptions } from './tui.js';
+
+