mdorigin 0.1.8 → 0.2.1

package/dist/search.d.ts

@@ -19,11 +19,13 @@ export interface SearchHit {
         score: number;
     };
 }
+export interface SearchQueryOptions {
+    topK?: number;
+    relativePathPrefix?: string;
+    metadata?: Record<string, string>;
+}
 export interface SearchApi {
-    search(query: string, options?: {
-        topK?: number;
-        relativePathPrefix?: string;
-    }): Promise<SearchHit[]>;
+    search(query: string, options?: SearchQueryOptions): Promise<SearchHit[]>;
 }
 export interface SearchBundleEntry {
     path: string;
@@ -32,6 +34,13 @@ export interface SearchBundleEntry {
     text?: string;
     base64?: string;
 }
+export interface ExternalSearchBundleEntry {
+    path: string;
+    mediaType: string;
+    storageKind: 'assets' | 'r2';
+    storageKey: string;
+    byteSize: number;
+}
 export interface BuildSearchBundleOptions {
     rootDir: string;
     outDir: string;
@@ -39,21 +48,35 @@ export interface BuildSearchBundleOptions {
     draftMode?: 'include' | 'exclude';
     embeddingBackend?: 'hashing' | 'model2vec';
     model?: string;
+    incremental?: boolean;
+    cachePath?: string;
 }
 export interface BuildSearchBundleResult {
     outputDir: string;
     documentCount: number;
     chunkCount: number;
     vectorDimensions: number;
+    cachePath?: string;
+    incremental?: {
+        scannedDocumentCount: number;
+        newDocumentCount: number;
+        changedDocumentCount: number;
+        unchangedDocumentCount: number;
+        removedDocumentCount: number;
+        activeDocumentCount: number;
+        activeChunkCount: number;
+    };
 }
 export interface SearchBundleOptions {
     indexDir: string;
     query: string;
     topK?: number;
     relativePathPrefix?: string;
+    metadata?: Record<string, string>;
 }
 export declare function buildSearchBundle(options: BuildSearchBundleOptions): Promise<BuildSearchBundleResult>;
 export declare function searchBundle(options: SearchBundleOptions): Promise<SearchHit[]>;
 export declare function createSearchApiFromDirectory(indexDir: string): Promise<SearchApi>;
 export declare function createSearchApiFromBundle(bundleEntries: SearchBundleEntry[]): SearchApi;
+export declare function createSearchApiFromExternalBundle(bundleEntries: ExternalSearchBundleEntry[], loadResponse: (entry: ExternalSearchBundleEntry) => Promise<Response>): SearchApi;
 export {};

package/dist/search.js

@@ -1,59 +1,100 @@
-import { readdir, readFile, realpath, stat } from 'node:fs/promises';
+import { rmSync } from 'node:fs';
+import { mkdtemp, mkdir, readdir, readFile, realpath, stat, writeFile, } from 'node:fs/promises';
+import os from 'node:os';
 import path from 'node:path';
 import { inferDirectoryContentType } from './core/content-type.js';
 import { getDirectoryIndexCandidates } from './core/directory-index.js';
-import { getDocumentSummary, getDocumentTitle, parseMarkdownDocument, } from './core/markdown.js';
+import { getDocumentSummary, getDocumentTitle, parseMarkdownDocument, stripMachineOnlyMarkdownComments, stripManagedIndexBlock, } from './core/markdown.js';
+import { isIgnoredContentName } from './core/content-store.js';
+const OVERVIEW_CONTENT_FILENAMES = new Set(['readme.md', 'index.md', 'skill.md']);
+const materializedSearchBundleDirectories = new Set();
+let searchBundleDirectoryCleanupRegistered = false;
 export async function buildSearchBundle(options) {
     const buildModule = await loadIndexbindBuildModule();
     const rootDir = path.resolve(options.rootDir);
+    const outputDir = path.resolve(options.outDir);
     const documents = await collectSearchDocuments(rootDir, options.siteConfig, {
         draftMode: options.draftMode ?? 'exclude',
     });
-    const stats = await buildModule.buildCanonicalBundle(path.resolve(options.outDir), documents, {
+    const buildOptions = {
         embeddingBackend: options.embeddingBackend ?? 'model2vec',
         model: options.model,
         sourceRootId: path.basename(rootDir),
         sourceRootPath: rootDir,
-    });
+    };
+    if (options.incremental) {
+        const cachePath = path.resolve(options.cachePath ?? defaultSearchBuildCachePath(outputDir));
+        await mkdir(path.dirname(cachePath), { recursive: true });
+        const removedRelativePaths = await resolveRemovedRelativePaths(cachePath, documents.map((document) => document.relativePath));
+        const incrementalStats = await buildModule.updateBuildCache(cachePath, documents, buildOptions, removedRelativePaths);
+        const stats = await buildModule.exportCanonicalBundleFromBuildCache(cachePath, outputDir);
+        await writeSearchBuildState(cachePath, documents.map((document) => document.relativePath));
+        return {
+            outputDir,
+            documentCount: stats.documentCount,
+            chunkCount: stats.chunkCount,
+            vectorDimensions: stats.vectorDimensions,
+            cachePath,
+            incremental: incrementalStats,
+        };
+    }
+    const stats = await buildModule.buildCanonicalBundle(outputDir, documents, buildOptions);
     return {
-        outputDir: path.resolve(options.outDir),
+        outputDir,
         documentCount: stats.documentCount,
         chunkCount: stats.chunkCount,
         vectorDimensions: stats.vectorDimensions,
     };
 }
 export async function searchBundle(options) {
-    const webModule = await loadIndexbindWebModule();
-    const index = await webModule.openWebIndex(path.resolve(options.indexDir));
-    return rerankSearchHits(await index.search(options.query, {
+    const searchContext = await openSearchContextFromDirectory(path.resolve(options.indexDir));
+    return rerankSearchHits(hydrateSearchHits(await searchContext.index.search(options.query, buildIndexbindSearchOptions({
         topK: options.topK ?? 10,
         relativePathPrefix: options.relativePathPrefix,
-    }));
+        metadata: options.metadata,
+    })), searchContext.documentsById));
 }
 export async function createSearchApiFromDirectory(indexDir) {
-    const webModule = await loadIndexbindWebModule();
-    const index = await webModule.openWebIndex(path.resolve(indexDir));
+    const searchContext = await openSearchContextFromDirectory(path.resolve(indexDir));
     return {
         async search(query, options) {
-            return rerankSearchHits(await index.search(query, {
+            return rerankSearchHits(hydrateSearchHits(await searchContext.index.search(query, buildIndexbindSearchOptions({
                 topK: options?.topK,
                 relativePathPrefix: options?.relativePathPrefix,
-            }));
+                metadata: options?.metadata,
+            })), searchContext.documentsById));
         },
     };
 }
 export function createSearchApiFromBundle(bundleEntries) {
-    let indexPromise = null;
+    let searchContextPromise = null;
+    return {
+        async search(query, options) {
+            if (searchContextPromise === null) {
+                searchContextPromise = openSearchContextFromBundle(bundleEntries, async (entry) => createInlineSearchBundleResponse(entry));
+            }
+            const searchContext = await searchContextPromise;
+            return rerankSearchHits(hydrateSearchHits(await searchContext.index.search(query, buildIndexbindSearchOptions({
+                topK: options?.topK,
+                relativePathPrefix: options?.relativePathPrefix,
+                metadata: options?.metadata,
+            })), searchContext.documentsById));
+        },
+    };
+}
+export function createSearchApiFromExternalBundle(bundleEntries, loadResponse) {
+    let searchContextPromise = null;
     return {
         async search(query, options) {
-            if (indexPromise === null) {
-                indexPromise = openWebIndexFromBundle(bundleEntries);
+            if (searchContextPromise === null) {
+                searchContextPromise = openSearchContextFromBundle(bundleEntries, loadResponse);
             }
-            const index = await indexPromise;
-            return rerankSearchHits(await index.search(query, {
+            const searchContext = await searchContextPromise;
+            return rerankSearchHits(hydrateSearchHits(await searchContext.index.search(query, buildIndexbindSearchOptions({
                 topK: options?.topK,
                 relativePathPrefix: options?.relativePathPrefix,
-            }));
+                metadata: options?.metadata,
+            })), searchContext.documentsById));
         },
     };
 }
@@ -77,12 +118,15 @@ async function collectSearchDocuments(rootDir, siteConfig, options) {
             canonicalUrl: absoluteCanonicalUrl,
             title: getDocumentTitle(parsed.meta, parsed.body, fallbackTitleFromRelativePath(document.relativePath)),
             summary: getDocumentSummary(parsed.meta, parsed.body),
-            content: markdown,
+            content: buildSearchableMarkdownBody(parsed.body),
             metadata: buildSearchMetadata(document.relativePath, canonicalPath, parsed.meta, siteConfig),
         });
     }
     return documents;
 }
+function buildSearchableMarkdownBody(markdownBody) {
+    return stripMachineOnlyMarkdownComments(stripManagedIndexBlock(markdownBody)).trim();
+}
 async function listSearchDocuments(rootDir) {
     const results = [];
     await walkDirectory(rootDir, '', new Set(), results);
@@ -102,7 +146,7 @@ async function walkDirectory(absoluteDirectoryPath, relativeDirectoryPath, visit
         results.push(indexFile);
     }
     for (const entry of entries) {
-        if (entry.name.startsWith('.')) {
+        if (isIgnoredContentName(entry.name)) {
             continue;
         }
         const absoluteEntryPath = path.join(absoluteDirectoryPath, entry.name);
@@ -160,7 +204,7 @@ async function inspectDirectoryShape(directoryPath) {
     let hasExtraMarkdownFiles = false;
     let hasAssetFiles = false;
     for (const entry of entries) {
-        if (entry.name.startsWith('.')) {
+        if (isIgnoredContentName(entry.name)) {
             continue;
         }
         const absoluteEntryPath = path.join(directoryPath, entry.name);
@@ -196,6 +240,8 @@ function buildSearchMetadata(relativePath, canonicalPath, meta, siteConfig) {
         markdownPath: `/${relativePath}`,
         canonicalPath,
         siteTitle: siteConfig.siteTitle,
+        section: getSearchSection(relativePath),
+        isOverview: isOverviewContentPath(relativePath),
     };
     if (meta.type === 'page' || meta.type === 'post') {
         metadata.type = meta.type;
@@ -211,6 +257,17 @@ function buildSearchMetadata(relativePath, canonicalPath, meta, siteConfig) {
     }
     return metadata;
 }
+function getSearchSection(relativePath) {
+    const directory = path.posix.dirname(relativePath.replaceAll('\\', '/'));
+    if (directory === '.') {
+        return '';
+    }
+    const [firstSegment] = directory.split('/', 1);
+    return firstSegment ?? '';
+}
+function isOverviewContentPath(relativePath) {
+    return OVERVIEW_CONTENT_FILENAMES.has(path.posix.basename(relativePath).toLowerCase());
+}
 function fallbackTitleFromRelativePath(relativePath) {
     const baseName = path.posix.basename(relativePath);
     if (DIRECTORY_INDEX_FILENAMES_LOWER.has(baseName.toLowerCase())) {
@@ -249,6 +306,33 @@ async function pathExists(filePath) {
         throw error;
     }
 }
+function defaultSearchBuildCachePath(outputDir) {
+    return path.join(path.dirname(outputDir), `${path.basename(outputDir)}.indexbind-cache.sqlite`);
+}
+function getSearchBuildStatePath(cachePath) {
+    return `${cachePath}.state.json`;
+}
+async function resolveRemovedRelativePaths(cachePath, currentRelativePaths) {
+    const previousRelativePaths = await readSearchBuildState(cachePath);
+    const current = new Set(currentRelativePaths);
+    return previousRelativePaths.filter((relativePath) => !current.has(relativePath));
+}
+async function readSearchBuildState(cachePath) {
+    const statePath = getSearchBuildStatePath(cachePath);
+    if (!(await pathExists(statePath))) {
+        return [];
+    }
+    const parsed = JSON.parse(await readFile(statePath, 'utf8'));
+    return Array.isArray(parsed.relativePaths)
+        ? parsed.relativePaths.filter((value) => typeof value === 'string')
+        : [];
+}
+async function writeSearchBuildState(cachePath, relativePaths) {
+    await writeFile(getSearchBuildStatePath(cachePath), JSON.stringify({
+        version: 1,
+        relativePaths: [...relativePaths].sort((left, right) => left.localeCompare(right)),
+    }, null, 2), 'utf8');
+}
 function isIgnoredSkillSupportDirectory(name) {
     return (name === 'scripts' ||
         name === 'references' ||
@@ -279,8 +363,10 @@ async function loadIndexbindCloudflareModule() {
         throw new Error(`Search query requires the optional package "indexbind". Install it first, for example: npm install indexbind`, { cause: error instanceof Error ? error : undefined });
     }
 }
-async function openWebIndexFromBundle(bundleEntries) {
-    const cloudflareModule = await loadIndexbindCloudflareModule();
+async function openWebIndexFromVirtualBundle(bundleEntries, loadResponse) {
+    if (isNodeRuntime()) {
+        return openWebIndexFromMaterializedBundle(bundleEntries, loadResponse);
+    }
     const baseUrl = 'https://mdorigin-search.invalid/';
     const originalFetch = globalThis.fetch;
     const bundleMap = new Map(bundleEntries.map((entry) => [new URL(entry.path, baseUrl).toString(), entry]));
@@ -292,31 +378,209 @@ async function openWebIndexFromBundle(bundleEntries) {
                 : input.url;
         const entry = bundleMap.get(requestUrl);
         if (entry) {
-            const headers = new Headers({ 'content-type': entry.mediaType });
-            const binaryBody = decodeBase64(entry.base64 ?? '');
-            const body = entry.kind === 'text'
-                ? entry.text ?? ''
-                : new Blob([new Uint8Array(binaryBody)], {
-                    type: entry.mediaType,
-                });
-            return new Response(body, {
-                status: 200,
-                headers,
-            });
+            return loadResponse(entry);
         }
         return originalFetch(input, init);
     };
     try {
-        return await cloudflareModule.openWebIndex(new URL(baseUrl));
+        try {
+            const cloudflareModule = await loadIndexbindCloudflareModule();
+            return await cloudflareModule.openWebIndex(new URL(baseUrl));
+        }
+        catch (error) {
+            if (!isCloudflareWasmImportError(error)) {
+                throw error;
+            }
+            const webModule = await loadIndexbindWebModule();
+            return await webModule.openWebIndex(new URL(baseUrl));
+        }
     }
     finally {
         globalThis.fetch = originalFetch;
     }
 }
+async function openSearchContextFromDirectory(indexDir) {
+    const webModule = await loadIndexbindWebModule();
+    const [index, documentsById] = await Promise.all([
+        webModule.openWebIndex(indexDir),
+        readSearchBundleDocumentsFromDirectory(indexDir),
+    ]);
+    return { index, documentsById };
+}
+async function openSearchContextFromBundle(bundleEntries, loadResponse) {
+    const [index, documentsById] = await Promise.all([
+        openWebIndexFromVirtualBundle(bundleEntries, loadResponse),
+        readSearchBundleDocumentsFromResponses(bundleEntries, loadResponse),
+    ]);
+    return { index, documentsById };
+}
+async function openWebIndexFromMaterializedBundle(bundleEntries, loadResponse) {
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), 'mdorigin-search-bundle-'));
+    registerMaterializedSearchBundleDirectory(tempDir);
+    for (const entry of bundleEntries) {
+        const response = await loadResponse(entry);
+        if (!response.ok) {
+            throw new Error(`Failed to materialize search bundle file ${entry.path}: ${response.status} ${response.statusText}`);
+        }
+        const outputPath = resolveMaterializedSearchBundlePath(tempDir, entry.path);
+        await mkdir(path.dirname(outputPath), { recursive: true });
+        if (entry.mediaType.startsWith('text/') || entry.mediaType.includes('json')) {
+            await writeFile(outputPath, await response.text(), 'utf8');
+            continue;
+        }
+        await writeFile(outputPath, new Uint8Array(await response.arrayBuffer()));
+    }
+    const webModule = await loadIndexbindWebModule();
+    return webModule.openWebIndex(tempDir);
+}
+function createInlineSearchBundleResponse(entry) {
+    const headers = new Headers({ 'content-type': entry.mediaType });
+    const binaryBody = decodeBase64(entry.base64 ?? '');
+    const body = entry.kind === 'text'
+        ? entry.text ?? ''
+        : new Blob([new Uint8Array(binaryBody)], {
+            type: entry.mediaType,
+        });
+    return new Response(body, {
+        status: 200,
+        headers,
+    });
+}
+function isCloudflareWasmImportError(error) {
+    const targetMessages = [
+        'Unknown file extension ".wasm"',
+        'WebAssembly.Module(): Argument 0 must be a buffer source',
+    ];
+    const visited = new Set();
+    let current = error;
+    while (current && typeof current === 'object' && !visited.has(current)) {
+        visited.add(current);
+        if (current instanceof Error) {
+            const currentMessage = current.message;
+            if (targetMessages.some((message) => currentMessage.includes(message))) {
+                return true;
+            }
+        }
+        const cause = current.cause;
+        if (!cause) {
+            break;
+        }
+        current = cause;
+    }
+    return false;
+}
 function decodeBase64(value) {
     const decoded = atob(value);
     return Uint8Array.from(decoded, (character) => character.charCodeAt(0));
 }
+function buildIndexbindSearchOptions(options) {
+    const normalized = {};
+    if (typeof options.topK === 'number' &&
+        Number.isFinite(options.topK) &&
+        Number.isInteger(options.topK) &&
+        options.topK > 0) {
+        normalized.topK = options.topK;
+    }
+    if (typeof options.relativePathPrefix === 'string' && options.relativePathPrefix !== '') {
+        normalized.relativePathPrefix = options.relativePathPrefix;
+    }
+    if (options.metadata && Object.keys(options.metadata).length > 0) {
+        normalized.metadata = options.metadata;
+    }
+    return Object.keys(normalized).length > 0 ? normalized : undefined;
+}
+function hydrateSearchHits(hits, documentsById) {
+    return hits.map((hit) => {
+        const document = documentsById.get(hit.docId);
+        if (!document) {
+            return hit;
+        }
+        return {
+            ...hit,
+            canonicalUrl: hit.canonicalUrl ?? document.canonicalUrl ?? undefined,
+            title: hit.title ?? document.title ?? undefined,
+            summary: hit.summary ?? document.summary ?? undefined,
+            metadata: Object.keys(hit.metadata).length > 0
+                ? hit.metadata
+                : document.metadata ?? hit.metadata,
+        };
+    });
+}
+async function readSearchBundleDocumentsFromDirectory(indexDir) {
+    const manifest = JSON.parse(await readFile(path.join(indexDir, 'manifest.json'), 'utf8'));
+    const documentsFile = manifest.files?.documents;
+    if (!documentsFile) {
+        return new Map();
+    }
+    const documents = JSON.parse(await readFile(path.join(indexDir, documentsFile), 'utf8'));
+    return new Map(documents.map((document) => [document.docId, document]));
+}
+async function readSearchBundleDocumentsFromResponses(bundleEntries, loadResponse) {
+    const manifestEntry = bundleEntries.find((entry) => entry.path === 'manifest.json');
+    if (!manifestEntry) {
+        return new Map();
+    }
+    const manifest = JSON.parse(await loadBundleResponseText(manifestEntry, loadResponse));
+    const documentsFile = manifest.files?.documents;
+    if (!documentsFile) {
+        return new Map();
+    }
+    const documentsEntry = bundleEntries.find((entry) => entry.path === documentsFile);
+    if (!documentsEntry) {
+        return new Map();
+    }
+    const documents = JSON.parse(await loadBundleResponseText(documentsEntry, loadResponse));
+    return new Map(documents.map((document) => [document.docId, document]));
+}
+async function loadBundleResponseText(entry, loadResponse) {
+    const response = await loadResponse(entry);
+    if (!response.ok) {
+        throw new Error(`Failed to read search bundle file ${entry.path}: ${response.status} ${response.statusText}`);
+    }
+    return response.text();
+}
+function isNodeRuntime() {
+    return (typeof process !== 'undefined' &&
+        typeof process.versions === 'object' &&
+        typeof process.versions.node === 'string');
+}
+function resolveMaterializedSearchBundlePath(baseDir, entryPath) {
+    if (path.isAbsolute(entryPath)) {
+        throw new Error(`Search bundle path must be relative: ${entryPath}`);
+    }
+    const normalizedEntryPath = entryPath.replaceAll('/', path.sep);
+    const outputPath = path.resolve(baseDir, normalizedEntryPath);
+    const relativeOutputPath = path.relative(baseDir, outputPath);
+    if (relativeOutputPath === '' ||
+        relativeOutputPath.startsWith(`..${path.sep}`) ||
+        relativeOutputPath === '..' ||
+        path.isAbsolute(relativeOutputPath)) {
+        throw new Error(`Search bundle path escapes materialized bundle directory: ${entryPath}`);
+    }
+    return outputPath;
+}
+function registerMaterializedSearchBundleDirectory(directoryPath) {
+    materializedSearchBundleDirectories.add(directoryPath);
+    if (searchBundleDirectoryCleanupRegistered || !isNodeRuntime()) {
+        return;
+    }
+    const cleanup = () => {
+        for (const stagedDirectory of materializedSearchBundleDirectories) {
+            rmSync(stagedDirectory, { recursive: true, force: true });
+        }
+        materializedSearchBundleDirectories.clear();
+    };
+    process.once('exit', cleanup);
+    process.once('SIGINT', () => {
+        cleanup();
+        process.exit(130);
+    });
+    process.once('SIGTERM', () => {
+        cleanup();
+        process.exit(143);
+    });
+    searchBundleDirectoryCleanupRegistered = true;
+}
 function rerankSearchHits(hits) {
     const remaining = [...hits];
     const ordered = [];
@@ -359,8 +623,7 @@ function compareHits(left, right) {
     return left.relativePath.localeCompare(right.relativePath);
 }
 function isOverviewSearchHit(hit) {
-    const baseName = path.posix.basename(hit.relativePath).toLowerCase();
-    return baseName === 'readme.md' || baseName === 'index.md';
+    return OVERVIEW_CONTENT_FILENAMES.has(path.posix.basename(hit.relativePath).toLowerCase());
 }
 function getTopLevelSection(relativePath) {
     const normalized = relativePath.replaceAll('\\', '/');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mdorigin",
-  "version": "0.1.8",
+  "version": "0.2.1",
   "type": "module",
   "description": "Markdown-first publishing for humans and agents.",
   "repository": {
@@ -66,9 +66,11 @@
   },
   "dependencies": {
     "gray-matter": "^4.0.3",
+    "rehype-raw": "^7.0.0",
+    "rehype-stringify": "^10.0.1",
     "remark": "^15.0.1",
     "remark-gfm": "^4.0.1",
-    "remark-html": "^16.0.1",
+    "remark-rehype": "^11.1.2",
     "tsx": "^4.20.5"
   },
   "devDependencies": {
@@ -76,6 +78,6 @@
     "typescript": "^5.9.2"
   },
   "optionalDependencies": {
-    "indexbind": "^0.2.1"
+    "indexbind": "^0.3.0"
   }
 }

package/dist/html/template-kind.d.ts

@@ -1 +0,0 @@
-export type TemplateName = 'document' | 'catalog';

package/dist/html/template-kind.js

@@ -1 +0,0 @@
-export {};