md-semantic-search 0.3.0 → 0.5.0

package/dist/frontmatter.js

@@ -0,0 +1,150 @@
+/**
+ * Typed YAML Frontmatter parser and metadata normalizer (issue #58).
+ * Parses YAML frontmatter into a validated document metadata object without
+ * unsafe executable tags. Supports scalars, inline/block lists, quotes, booleans,
+ * dates, tags normalization, and canonical document identity.
+ */
+export function parseFrontmatter(rawYaml) {
+    const metadata = {
+        aliases: [],
+        tags: [],
+        custom: {},
+    };
+    if (!rawYaml || typeof rawYaml !== 'string') {
+        return metadata;
+    }
+    const lines = rawYaml.split(/\r?\n/);
+    let currentKey = null;
+    let currentBlockList = null;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const trimmed = line.trim();
+        if (!trimmed || trimmed.startsWith('#')) {
+            continue;
+        }
+        const blockListMatch = line.match(/^(\s+)-\s+(.+)$/);
+        if (blockListMatch && currentKey && currentBlockList) {
+            const val = parseYAMLValue(blockListMatch[2]);
+            if (typeof val === 'string' || typeof val === 'number' || typeof val === 'boolean') {
+                currentBlockList.push(String(val));
+            }
+            continue;
+        }
+        const kvMatch = line.match(/^([a-zA-Z0-9_-]+):\s*(.*)$/);
+        if (kvMatch) {
+            if (currentKey && currentBlockList) {
+                setMetadataValue(metadata, currentKey, currentBlockList);
+                currentKey = null;
+                currentBlockList = null;
+            }
+            const key = kvMatch[1].trim();
+            const rawVal = kvMatch[2].trim();
+            if (rawVal === '') {
+                currentKey = key;
+                currentBlockList = [];
+            }
+            else {
+                const val = parseYAMLValue(rawVal);
+                setMetadataValue(metadata, key, val);
+            }
+        }
+    }
+    if (currentKey && currentBlockList) {
+        setMetadataValue(metadata, currentKey, currentBlockList);
+    }
+    return metadata;
+}
+function parseYAMLValue(raw) {
+    const trimmed = raw.trim();
+    if (trimmed.startsWith('[') && trimmed.endsWith(']')) {
+        const inner = trimmed.slice(1, -1).trim();
+        if (!inner)
+            return [];
+        return inner
+            .split(',')
+            .map((item) => parseYAMLValue(item))
+            .map((item) => (Array.isArray(item) ? item.join(' ') : String(item).trim()))
+            .filter((item) => item.length > 0);
+    }
+    if ((trimmed.startsWith('"') && trimmed.endsWith('"')) || (trimmed.startsWith("'") && trimmed.endsWith("'"))) {
+        return trimmed.slice(1, -1).trim();
+    }
+    if (/^true$/i.test(trimmed))
+        return true;
+    if (/^false$/i.test(trimmed))
+        return false;
+    if (/^-?\d+(\.\d+)?$/.test(trimmed)) {
+        const num = Number(trimmed);
+        if (!Number.isNaN(num))
+            return num;
+    }
+    return trimmed;
+}
+function setMetadataValue(meta, key, val) {
+    const normKey = key.toLowerCase().replace(/[-_]/g, '');
+    if (normKey === 'title' && typeof val === 'string') {
+        meta.title = val;
+        return;
+    }
+    if (normKey === 'alias' || normKey === 'aliases') {
+        const items = Array.isArray(val) ? val : [String(val)];
+        for (const item of items) {
+            const cleaned = item.trim().replace(/^["']|["']$/g, '');
+            if (cleaned && !meta.aliases.includes(cleaned)) {
+                meta.aliases.push(cleaned);
+            }
+        }
+        return;
+    }
+    if (normKey === 'tag' || normKey === 'tags') {
+        let items;
+        if (Array.isArray(val)) {
+            items = val;
+        }
+        else if (typeof val === 'string') {
+            items = val.includes(',') ? val.split(',') : [val];
+        }
+        else {
+            items = [String(val)];
+        }
+        for (const item of items) {
+            const cleaned = String(item)
+                .trim()
+                .replace(/^#/, '')
+                .toLowerCase();
+            if (cleaned && !meta.tags.includes(cleaned)) {
+                meta.tags.push(cleaned);
+            }
+        }
+        return;
+    }
+    if (normKey === 'project' && (typeof val === 'string' || typeof val === 'number')) {
+        meta.project = String(val);
+        return;
+    }
+    if (normKey === 'type' && typeof val === 'string') {
+        meta.type = val;
+        return;
+    }
+    if (normKey === 'status' && typeof val === 'string') {
+        meta.status = val;
+        return;
+    }
+    if (normKey === 'canonical') {
+        meta.canonical = Boolean(val);
+        return;
+    }
+    if ((normKey === 'canonicalref' || normKey === 'canonical_ref') && typeof val === 'string') {
+        meta.canonicalRef = val.replace(/^\[\[(.*)\]\]$/, '$1').trim();
+        return;
+    }
+    if (normKey === 'created') {
+        meta.created = String(val);
+        return;
+    }
+    if (normKey === 'updated') {
+        meta.updated = String(val);
+        return;
+    }
+    meta.custom[key] = val;
+}

package/dist/index-format.d.ts

@@ -0,0 +1,23 @@
+import { ModelAdapter } from './models.js';
+export declare function inspectIndexSchema(value: unknown, location: string): {
+    index: Record<string, unknown>;
+    schema: number;
+};
+export declare function validateNumericVector(value: unknown, expectedDim: number | undefined, where: string): number;
+export declare function validateCurrentChunk(chunk: unknown, position: number, options: {
+    dim: number | undefined;
+    encoding: 'stored' | 'loaded' | 'none';
+    allowMissingVector?: boolean;
+}): void;
+export interface ValidatedIndexEnvelope {
+    index: Record<string, any>;
+    schema: number;
+    model: ModelAdapter;
+    dim: number | undefined;
+}
+export declare function validateIndexEnvelope(value: unknown, location: string, options?: {
+    encoding?: 'stored' | 'loaded';
+    validateVectors?: boolean;
+    allowMissingVectors?: boolean;
+    validateLexical?: boolean;
+}): ValidatedIndexEnvelope;

package/dist/index-format.js

@@ -0,0 +1,125 @@
+import { decodeVec, parseSchemaVersion, resolveIndexDimension, SCHEMA_VERSION } from './core.js';
+import { validateLexicalIndex } from './lexical.js';
+import { resolveModel } from './models.js';
+const REBUILD = ' — run `mdss index` to rebuild';
+export function inspectIndexSchema(value, location) {
+    if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+        throw new Error(`${location} root must be an object${REBUILD}`);
+    }
+    const index = value;
+    return { index, schema: parseSchemaVersion(index.schemaVersion) };
+}
+export function validateNumericVector(value, expectedDim, where) {
+    if (!Array.isArray(value) && !(value instanceof Float32Array)) {
+        throw new Error(`${where}: missing vector${REBUILD}`);
+    }
+    const vec = value;
+    if (vec.length === 0)
+        throw new Error(`${where}: vector must not be empty${REBUILD}`);
+    if (expectedDim !== undefined && vec.length !== expectedDim) {
+        throw new Error(`${where}: vector has ${vec.length} dims, expected ${expectedDim}${REBUILD}`);
+    }
+    for (let position = 0; position < vec.length; position++) {
+        if (Array.isArray(value) && !Object.hasOwn(value, position)) {
+            throw new Error(`${where}: missing vector component at ${position}${REBUILD}`);
+        }
+        if (typeof vec[position] !== 'number' || !Number.isFinite(vec[position])) {
+            throw new Error(`${where}: non-finite vector value${REBUILD}`);
+        }
+    }
+    return vec.length;
+}
+export function validateCurrentChunk(chunk, position, options) {
+    const prefix = `chunk ${position}`;
+    if (chunk === null || typeof chunk !== 'object' || Array.isArray(chunk)) {
+        throw new Error(`${prefix} must be an object${REBUILD}`);
+    }
+    const value = chunk;
+    for (const field of ['file', 'title', 'heading', 'text', 'chunkHash']) {
+        if (typeof value[field] !== 'string') {
+            throw new Error(`${prefix} ${field} must be a string${REBUILD}`);
+        }
+    }
+    if (!Array.isArray(value.headingPath) || value.headingPath.length === 0) {
+        throw new Error(`${prefix} headingPath must contain nonempty strings${REBUILD}`);
+    }
+    for (let pathPosition = 0; pathPosition < value.headingPath.length; pathPosition++) {
+        if (!Object.hasOwn(value.headingPath, pathPosition) ||
+            typeof value.headingPath[pathPosition] !== 'string' ||
+            value.headingPath[pathPosition].trim().length === 0) {
+            throw new Error(`${prefix} headingPath must contain nonempty strings${REBUILD}`);
+        }
+    }
+    if (value.headingPath[value.headingPath.length - 1] !== value.heading) {
+        throw new Error(`${prefix} headingPath leaf must equal heading${REBUILD}`);
+    }
+    if (options.encoding === 'none')
+        return;
+    const vectorWhere = `${prefix}; chunk ${value.file}` + (value.heading ? ` › ${value.heading}` : '');
+    if (value.vec === undefined && options.allowMissingVector)
+        return;
+    if (options.encoding === 'stored') {
+        if (typeof value.vec !== 'string') {
+            throw new Error(`${prefix} vec must be canonical base64${REBUILD}`);
+        }
+        try {
+            decodeVec(value.vec, options.dim);
+        }
+        catch (error) {
+            throw new Error(`${vectorWhere}: ${error.message}`, { cause: error });
+        }
+        return;
+    }
+    validateNumericVector(value.vec, options.dim, vectorWhere);
+}
+export function validateIndexEnvelope(value, location, options = {}) {
+    const { index, schema } = inspectIndexSchema(value, location);
+    if (schema > SCHEMA_VERSION) {
+        throw new Error(`${location} uses schema v${schema}, newer than this mdss supports; supports up to v${SCHEMA_VERSION} — upgrade md-semantic-search`);
+    }
+    if (!Array.isArray(index.chunks)) {
+        throw new Error(`${location} chunks must be an array${REBUILD}`);
+    }
+    let modelName = index.model ?? index.modelAlias;
+    if (schema >= 3) {
+        if (typeof index.model !== 'string' || index.model.trim().length === 0) {
+            throw new Error(`schema v3 model must be a nonempty string${REBUILD}`);
+        }
+        modelName = index.model;
+    }
+    else if (modelName !== undefined && modelName !== null && typeof modelName !== 'string') {
+        throw new Error(`${location} model metadata must be a string${REBUILD}`);
+    }
+    const model = resolveModel(modelName);
+    const dim = resolveIndexDimension(index.dim, model.dim);
+    if (schema < 3) {
+        return { index, schema, model, dim };
+    }
+    if (index.format !== 'binary-v1') {
+        throw new Error(`schema v3 format must be binary-v1${REBUILD}`);
+    }
+    if (typeof index.chunkCount !== 'number' ||
+        !Number.isSafeInteger(index.chunkCount) ||
+        index.chunkCount < 0 ||
+        index.chunkCount !== index.chunks.length) {
+        throw new Error(`schema v3 chunkCount must equal chunks.length${REBUILD}`);
+    }
+    if (index.chunks.length > 0 && dim === undefined) {
+        throw new Error(`nonempty schema v3 custom index requires explicit dim${REBUILD}`);
+    }
+    const encoding = options.validateVectors === false ? 'none' : (options.encoding ?? 'stored');
+    for (let position = 0; position < index.chunks.length; position++) {
+        validateCurrentChunk(index.chunks[position], position, {
+            dim,
+            encoding,
+            allowMissingVector: options.allowMissingVectors,
+        });
+    }
+    if (options.validateLexical !== false) {
+        const lexicalError = validateLexicalIndex(index.lexical, index.chunks.length);
+        if (lexicalError) {
+            throw new Error(`schema v3 lexical index is invalid (${lexicalError})${REBUILD}`);
+        }
+    }
+    return { index, schema, model, dim };
+}

package/dist/index.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Public library API for md-semantic-search.
+ *
+ * Import from a consumer project:
+ *
+ *   import { buildIndex, search, loadIndex, searchIndex, resolveModel, MODELS } from 'md-semantic-search';
+ *
+ * One-shot usage (parses the index on every call):
+ *
+ *   await buildIndex({ db: './docs', indexDir: './.mdss', cacheDir: '~/.cache/mdss' });
+ *   const hits = await search({ indexDir: './.mdss', cacheDir: '~/.cache/mdss', query: 'rotate api token' });
+ *
+ * Repeated queries in one process (parse + model cached, issue #2):
+ *
+ *   const loaded = loadIndex('./.mdss');
+ *   const hitsA = await searchIndex({ loaded, cacheDir: '~/.cache/mdss', query: 'failover runbook' });
+ *   const hitsB = await searchIndex({ loaded, cacheDir: '~/.cache/mdss', query: 'db backup' });
+ *
+ * All functions are model- and path-agnostic: they take explicit paths, so the
+ * same API works on any folder of markdown anywhere on disk.
+ */
+export { buildIndex, chunkHash } from './indexer.js';
+export { search, searchIndex, loadIndex, tokenize, keywordScores, rrf, QueryEmbeddingCache, } from './search.js';
+export { resolveModel, MODELS, DEFAULT_MODEL, normalizeAdapter, embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint, } from './models.js';
+export { getReranker, rerankScores, RERANK_MODEL, } from './rerank.js';
+export { embed, getExtractor, cosine, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, } from './core.js';
+export { MCP_TOOLS, handleMcpRequest, startMcpServer, } from './mcp.js';
+export { trainIVF, searchIVFCandidates, serializeIVF, deserializeIVF, ANN_THRESHOLD, DEFAULT_NPROBE, } from './ivf.js';
+export { runTui } from './tui.js';

package/dist/index.js

@@ -0,0 +1,29 @@
+/**
+ * Public library API for md-semantic-search.
+ *
+ * Import from a consumer project:
+ *
+ *   import { buildIndex, search, loadIndex, searchIndex, resolveModel, MODELS } from 'md-semantic-search';
+ *
+ * One-shot usage (parses the index on every call):
+ *
+ *   await buildIndex({ db: './docs', indexDir: './.mdss', cacheDir: '~/.cache/mdss' });
+ *   const hits = await search({ indexDir: './.mdss', cacheDir: '~/.cache/mdss', query: 'rotate api token' });
+ *
+ * Repeated queries in one process (parse + model cached, issue #2):
+ *
+ *   const loaded = loadIndex('./.mdss');
+ *   const hitsA = await searchIndex({ loaded, cacheDir: '~/.cache/mdss', query: 'failover runbook' });
+ *   const hitsB = await searchIndex({ loaded, cacheDir: '~/.cache/mdss', query: 'db backup' });
+ *
+ * All functions are model- and path-agnostic: they take explicit paths, so the
+ * same API works on any folder of markdown anywhere on disk.
+ */
+export { buildIndex, chunkHash } from './indexer.js';
+export { search, searchIndex, loadIndex, tokenize, keywordScores, rrf, QueryEmbeddingCache, } from './search.js';
+export { resolveModel, MODELS, DEFAULT_MODEL, normalizeAdapter, embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint, } from './models.js';
+export { getReranker, rerankScores, RERANK_MODEL, } from './rerank.js';
+export { embed, getExtractor, cosine, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, } from './core.js';
+export { MCP_TOOLS, handleMcpRequest, startMcpServer, } from './mcp.js';
+export { trainIVF, searchIVFCandidates, serializeIVF, deserializeIVF, ANN_THRESHOLD, DEFAULT_NPROBE, } from './ivf.js';
+export { runTui } from './tui.js';

package/dist/indexer.d.ts

@@ -0,0 +1,70 @@
+import { LexicalIndex } from './lexical.js';
+export interface IndexChunk {
+    file: string;
+    title: string;
+    heading: string;
+    headingPath: string[];
+    text: string;
+    vec?: number[] | Float32Array | string;
+    chunkHash?: string;
+}
+export interface PersistedIndex {
+    schemaVersion?: number;
+    format?: string;
+    model?: string | null;
+    modelAlias?: string;
+    adapterFingerprint?: string;
+    dim?: number;
+    db?: string;
+    built?: string;
+    complete?: boolean;
+    chunkCount?: number;
+    hashes?: Record<string, string>;
+    lexical?: LexicalIndex;
+    chunks: IndexChunk[];
+}
+export declare function canonicalPassage(chunk: {
+    title: string;
+    heading: string;
+    headingPath?: string[];
+    text: string;
+}): string;
+export declare function chunkHash(model: {
+    id: string;
+    revision?: string;
+    passagePrefix?: string;
+    pooling?: string;
+}, chunk: {
+    title: string;
+    heading: string;
+    headingPath?: string[];
+    text: string;
+}): string;
+export interface BuildIndexOptions {
+    db: string;
+    indexDir: string;
+    cacheDir: string;
+    modelName: string;
+    ignore?: string[];
+    offline?: boolean;
+    log?: (s: string) => void;
+    embedFn?: any;
+    _lockHeld?: boolean;
+    maxRetries?: number;
+    onProgress?: (done: number, total: number, chunksPerSec: number) => void;
+    workers?: number;
+    ann?: boolean;
+}
+export interface BuildIndexResult {
+    files: number;
+    skipped: number;
+    chunks: number;
+    reused: number;
+    reusedChunks: number;
+    reusedFiles: number;
+    embedded: number;
+    dim: number;
+    model: string;
+    vectorsPath: string;
+}
+export declare function buildIndex(opts: BuildIndexOptions): Promise<BuildIndexResult>;