mdorigin 0.1.0 → 0.1.2

package/README.md

@@ -2,15 +2,51 @@
 
 `mdorigin` is a markdown-first publishing engine.
 
-It treats markdown as the only source of truth, serves raw `.md` directly for agents, and renders `.html` views for humans from the same directory tree.
+It treats markdown as the only source of truth, serves raw `.md` directly for agents, renders `.html` views for humans from the same directory tree, can return markdown from extensionless routes when clients send `Accept: text/markdown`, supports frontmatter aliases for old URL redirects, and can publish skill-style bundles built around `SKILL.md`.
 
 ## Install
 
 ```bash
-npm install mdorigin
+npm install -g mdorigin
 ```
 
-## Development
+Then run it directly:
+
+```bash
+mdorigin dev --root docs/site
+```
+
+If you prefer a project-local install instead, use `npm install --save-dev mdorigin` and run it with `npx --no-install mdorigin ...`.
+
+## Optional Search
+
+`mdorigin` can build a local retrieval bundle through the optional [`indexbind`](https://github.com/jolestar/indexbind) package:
+
+```bash
+npm install indexbind
+mdorigin build search --root docs/site
+mdorigin search --index dist/search "cloudflare deploy"
+```
+
+`build search` now defaults to the higher-quality `model2vec` backend. If you need a smaller or faster fallback, you can opt back into hashing:
+
+```bash
+mdorigin build search --root docs/site --embedding-backend hashing
+```
+
+To expose the same search bundle from the site runtime:
+
+```bash
+mdorigin dev --root docs/site --search dist/search
+mdorigin build cloudflare --root docs/site --search dist/search
+```
+
+Runtime endpoints:
+
+- `/api/search?q=cloudflare+deploy`
+- `/api/openapi.json`
+
+## Repo Development
 
 ```bash
 npm install
@@ -19,6 +55,19 @@ npm run dev -- --root docs/site
 npm run build:index -- --root docs/site
 ```
 
+## Release
+
+Publishing is handled by GitHub Actions through npm trusted publishing.
+
+- workflow: `.github/workflows/release.yml`
+- trigger: push a tag like `v0.1.2`, or run the workflow manually
+
+The npm package settings still need a one-time trusted publisher entry for:
+
+- owner: `jolestar`
+- repository: `mdorigin`
+- workflow file: `release.yml`
+
 ## Docs
 
 - Getting started: [`docs/site/guides/getting-started.md`](docs/site/guides/getting-started.md)

package/dist/adapters/cloudflare.d.ts

@@ -1,5 +1,6 @@
 import { type ContentEntryKind } from '../core/content-store.js';
 import type { ResolvedSiteConfig } from '../core/site-config.js';
+import { type SearchBundleEntry } from '../search.js';
 export interface CloudflareManifestEntry {
     path: string;
     kind: ContentEntryKind;
@@ -10,6 +11,7 @@ export interface CloudflareManifestEntry {
 export interface CloudflareManifest {
     entries: CloudflareManifestEntry[];
     siteConfig?: ResolvedSiteConfig;
+    searchEntries?: SearchBundleEntry[];
 }
 export interface ExportedHandlerLike {
     fetch(request: Request): Promise<Response>;

package/dist/adapters/cloudflare.js

@@ -1,5 +1,6 @@
 import { MemoryContentStore, } from '../core/content-store.js';
 import { handleSiteRequest } from '../core/request-handler.js';
+import { createSearchApiFromBundle } from '../search.js';
 export function createCloudflareWorker(manifest) {
     const store = new MemoryContentStore(manifest.entries.map((entry) => {
         if (entry.kind === 'text') {
@@ -17,6 +18,9 @@ export function createCloudflareWorker(manifest) {
             bytes: decodeBase64(entry.base64 ?? ''),
         };
     }));
+    const searchApi = manifest.searchEntries && manifest.searchEntries.length > 0
+        ? createSearchApiFromBundle(manifest.searchEntries)
+        : undefined;
     return {
         async fetch(request) {
             const url = new URL(request.url);
@@ -24,15 +28,28 @@ export function createCloudflareWorker(manifest) {
                 draftMode: 'exclude',
                 siteConfig: manifest.siteConfig ?? {
                     siteTitle: 'mdorigin',
+                    siteUrl: undefined,
+                    favicon: undefined,
+                    logo: undefined,
                     showDate: true,
                     showSummary: true,
                     theme: 'paper',
                     template: 'document',
                     topNav: [],
+                    footerNav: [],
+                    footerText: undefined,
+                    socialLinks: [],
+                    editLink: undefined,
                     showHomeIndex: true,
+                    catalogInitialPostCount: 10,
+                    catalogLoadMoreStep: 10,
                     siteTitleConfigured: false,
                     siteDescriptionConfigured: false,
                 },
+                acceptHeader: request.headers.get('accept') ?? undefined,
+                searchParams: url.searchParams,
+                requestUrl: request.url,
+                searchApi,
             });
             const headers = new Headers(siteResponse.headers);
             const body = siteResponse.body instanceof Uint8Array

package/dist/adapters/node.d.ts

@@ -1,10 +1,12 @@
 import { type IncomingMessage, type ServerResponse } from 'node:http';
 import type { ContentStore } from '../core/content-store.js';
 import type { ResolvedSiteConfig } from '../core/site-config.js';
+import type { SearchApi } from '../search.js';
 export interface NodeAdapterOptions {
     rootDir: string;
     draftMode: 'include' | 'exclude';
     siteConfig: ResolvedSiteConfig;
+    searchApi?: SearchApi;
 }
 export declare function createFileSystemContentStore(rootDir: string): ContentStore;
 export declare function createNodeRequestListener(options: NodeAdapterOptions): (request: IncomingMessage, response: ServerResponse) => Promise<void>;

package/dist/adapters/node.js

@@ -1,5 +1,5 @@
 import { createServer } from 'node:http';
-import { readFile, readdir } from 'node:fs/promises';
+import { readFile, readdir, stat } from 'node:fs/promises';
 import path from 'node:path';
 import { getMediaTypeForPath, isLikelyTextPath, normalizeContentPath, normalizeDirectoryPath, } from '../core/content-store.js';
 import { handleSiteRequest } from '../core/request-handler.js';
@@ -12,10 +12,14 @@ export function createFileSystemContentStore(rootDir) {
                 return null;
             }
             const filePath = path.resolve(resolvedRootDir, normalizedPath);
-            if (!filePath.startsWith(`${resolvedRootDir}${path.sep}`) && filePath !== resolvedRootDir) {
+            if (!isVisiblePathWithinRoot(resolvedRootDir, filePath)) {
                 return null;
             }
             try {
+                const fileStats = await stat(filePath);
+                if (!fileStats.isFile()) {
+                    return null;
+                }
                 const mediaType = getMediaTypeForPath(normalizedPath);
                 if (isLikelyTextPath(normalizedPath)) {
                     const text = await readFile(filePath, 'utf8');
@@ -47,21 +51,49 @@ export function createFileSystemContentStore(rootDir) {
                 return null;
             }
             const directoryPath = path.resolve(resolvedRootDir, normalizedPath);
-            if (!directoryPath.startsWith(`${resolvedRootDir}${path.sep}`) &&
-                directoryPath !== resolvedRootDir) {
+            if (!isVisiblePathWithinRoot(resolvedRootDir, directoryPath)) {
                 return null;
             }
             try {
+                const directoryStats = await stat(directoryPath);
+                if (!directoryStats.isDirectory()) {
+                    return null;
+                }
                 const entries = await readdir(directoryPath, { withFileTypes: true });
-                return entries
+                const resolvedEntries = await Promise.all(entries
                     .filter((entry) => !entry.name.startsWith('.'))
-                    .map((entry) => ({
-                    name: entry.name,
-                    path: normalizedPath === ''
+                    .map(async (entry) => {
+                    const childVisiblePath = normalizedPath === ''
                         ? entry.name
-                        : `${normalizedPath}/${entry.name}`,
-                    kind: entry.isDirectory() ? 'directory' : 'file',
-                }))
+                        : `${normalizedPath}/${entry.name}`;
+                    const childFilePath = path.resolve(resolvedRootDir, childVisiblePath);
+                    try {
+                        const childStats = await stat(childFilePath);
+                        if (childStats.isDirectory()) {
+                            return {
+                                name: entry.name,
+                                path: childVisiblePath,
+                                kind: 'directory',
+                            };
+                        }
+                        if (childStats.isFile()) {
+                            return {
+                                name: entry.name,
+                                path: childVisiblePath,
+                                kind: 'file',
+                            };
+                        }
+                    }
+                    catch (error) {
+                        if (isNodeNotFound(error)) {
+                            return null;
+                        }
+                        throw error;
+                    }
+                    return null;
+                }));
+                return resolvedEntries
+                    .filter((entry) => entry !== null)
                     .sort((left, right) => {
                     if (left.kind !== right.kind) {
                         return left.kind === 'directory' ? -1 : 1;
@@ -78,6 +110,10 @@ export function createFileSystemContentStore(rootDir) {
         },
     };
 }
+function isVisiblePathWithinRoot(rootDir, candidatePath) {
+    return (candidatePath === rootDir ||
+        candidatePath.startsWith(`${rootDir}${path.sep}`));
+}
 export function createNodeRequestListener(options) {
     const store = createFileSystemContentStore(options.rootDir);
     return async function onRequest(request, response) {
@@ -86,6 +122,10 @@ export function createNodeRequestListener(options) {
             const siteResponse = await handleSiteRequest(store, url.pathname, {
                 draftMode: options.draftMode,
                 siteConfig: options.siteConfig,
+                acceptHeader: request.headers.accept,
+                searchParams: url.searchParams,
+                requestUrl: url.toString(),
+                searchApi: options.searchApi,
             });
             response.statusCode = siteResponse.status;
             for (const [headerName, headerValue] of Object.entries(siteResponse.headers)) {

package/dist/cli/build-cloudflare.js

@@ -5,7 +5,7 @@ import { applySiteConfigFrontmatterDefaults, loadSiteConfig, } from '../core/sit
 export async function runBuildCloudflareCommand(argv) {
     const args = parseArgs(argv);
     if (!args.root) {
-        console.error('Usage: mdorigin build cloudflare --root <content-dir> [--out ./dist/cloudflare] [--config mdorigin.config.json]');
+        console.error('Usage: mdorigin build cloudflare --root <content-dir> [--out ./dist/cloudflare] [--config mdorigin.config.json] [--search ./dist/search]');
         process.exitCode = 1;
         return;
     }
@@ -21,6 +21,7 @@ export async function runBuildCloudflareCommand(argv) {
         rootDir,
         outDir: path.resolve(args.out ?? 'dist/cloudflare'),
         siteConfig,
+        searchDir: args.search ? path.resolve(args.search) : undefined,
     });
     console.log(`cloudflare worker written to ${result.workerFile}`);
 }
@@ -42,6 +43,11 @@ function parseArgs(argv) {
         if (argument === '--config' && nextValue) {
             result.config = nextValue;
             index += 1;
+            continue;
+        }
+        if (argument === '--search' && nextValue) {
+            result.search = nextValue;
+            index += 1;
         }
     }
     return result;

package/dist/cli/build-search.d.ts

@@ -0,0 +1 @@
+export declare function runBuildSearchCommand(rawArgs: string[]): Promise<void>;

package/dist/cli/build-search.js

@@ -0,0 +1,44 @@
+import path from 'node:path';
+import { buildSearchBundle } from '../search.js';
+import { applySiteConfigFrontmatterDefaults, loadSiteConfig } from '../core/site-config.js';
+import { createFileSystemContentStore } from '../adapters/node.js';
+export async function runBuildSearchCommand(rawArgs) {
+    const args = parseArgs(rawArgs);
+    if (!args.root) {
+        throw new Error('Usage: mdorigin build search --root <content-dir> [--out ./dist/search] [--embedding-backend model2vec|hashing] [--model sentence-transformers/all-MiniLM-L6-v2] [--config mdorigin.config.json]');
+    }
+    const rootDir = path.resolve(args.root);
+    const store = createFileSystemContentStore(rootDir);
+    const siteConfig = await applySiteConfigFrontmatterDefaults(store, await loadSiteConfig({
+        rootDir,
+        configPath: args.config,
+    }));
+    const result = await buildSearchBundle({
+        rootDir,
+        outDir: path.resolve(args.out ?? 'dist/search'),
+        siteConfig,
+        embeddingBackend: args.embeddingBackend ?? 'model2vec',
+        model: args.model,
+    });
+    console.log(`search bundle written to ${result.outputDir} (${result.documentCount} documents, ${result.chunkCount} chunks)`);
+}
+function parseArgs(rawArgs) {
+    const parsed = {};
+    for (let index = 0; index < rawArgs.length; index += 1) {
+        const arg = rawArgs[index];
+        if (arg.startsWith('--')) {
+            const value = rawArgs[index + 1];
+            if (value && !value.startsWith('--')) {
+                parsed[arg.slice(2)] = value;
+                index += 1;
+            }
+        }
+    }
+    return {
+        root: parsed.root,
+        out: parsed.out,
+        embeddingBackend: parsed['embedding-backend'],
+        model: parsed.model,
+        config: parsed.config,
+    };
+}

package/dist/cli/dev.js

@@ -1,10 +1,11 @@
 import path from 'node:path';
 import { createFileSystemContentStore, createNodeServer } from '../adapters/node.js';
 import { applySiteConfigFrontmatterDefaults, loadSiteConfig, } from '../core/site-config.js';
+import { createSearchApiFromDirectory } from '../search.js';
 export async function runDevCommand(argv) {
     const args = parseArgs(argv);
     if (!args.root) {
-        console.error('Usage: mdorigin dev --root <content-dir> [--port 3000] [--config mdorigin.config.json]');
+        console.error('Usage: mdorigin dev --root <content-dir> [--port 3000] [--config mdorigin.config.json] [--search ./dist/search]');
         process.exitCode = 1;
         return;
     }
@@ -21,6 +22,9 @@ export async function runDevCommand(argv) {
         rootDir,
         draftMode: 'include',
         siteConfig,
+        searchApi: args.search
+            ? await createSearchApiFromDirectory(path.resolve(args.search))
+            : undefined,
     });
     await new Promise((resolve, reject) => {
         server.once('error', reject);
@@ -47,6 +51,11 @@ function parseArgs(argv) {
         if (argument === '--config' && nextValue) {
             result.config = nextValue;
             index += 1;
+            continue;
+        }
+        if (argument === '--search' && nextValue) {
+            result.search = nextValue;
+            index += 1;
         }
     }
     return result;

package/dist/cli/main.d.ts

@@ -1 +1,2 @@
+#!/usr/bin/env node
 export {};

package/dist/cli/main.js

@@ -1,7 +1,10 @@
+#!/usr/bin/env node
 import { runBuildIndexCommand } from './build-index.js';
 import { runBuildCloudflareCommand } from './build-cloudflare.js';
+import { runBuildSearchCommand } from './build-search.js';
 import { runDevCommand } from './dev.js';
 import { runInitCloudflareCommand } from './init-cloudflare.js';
+import { runSearchCommand } from './search.js';
 async function main() {
     const [command, subcommand, ...rest] = process.argv.slice(2);
     if (command === 'dev') {
@@ -16,16 +19,26 @@ async function main() {
         await runBuildIndexCommand(rest);
         return;
     }
+    if (command === 'build' && subcommand === 'search') {
+        await runBuildSearchCommand(rest);
+        return;
+    }
     if (command === 'init' && subcommand === 'cloudflare') {
         await runInitCloudflareCommand(rest);
         return;
     }
+    if (command === 'search') {
+        await runSearchCommand([subcommand, ...rest].filter(isDefined));
+        return;
+    }
     console.error([
         'Usage:',
-        '  mdorigin dev --root <content-dir> [--port 3000] [--config mdorigin.config.json]',
+        '  mdorigin dev --root <content-dir> [--port 3000] [--config mdorigin.config.json] [--search ./dist/search]',
         '  mdorigin build index (--root <content-dir> | --dir <content-dir>)',
-        '  mdorigin build cloudflare --root <content-dir> [--out ./dist/cloudflare] [--config mdorigin.config.json]',
+        '  mdorigin build search --root <content-dir> [--out ./dist/search] [--embedding-backend model2vec|hashing] [--model sentence-transformers/all-MiniLM-L6-v2] [--config mdorigin.config.json]',
+        '  mdorigin build cloudflare --root <content-dir> [--out ./dist/cloudflare] [--config mdorigin.config.json] [--search ./dist/search]',
         '  mdorigin init cloudflare [--dir .] [--entry ./dist/cloudflare/worker.mjs] [--name <worker-name>] [--compatibility-date 2026-03-20] [--force]',
+        '  mdorigin search --index <search-dir> [--top-k 10] <query>',
     ].join('\n'));
     process.exitCode = 1;
 }

package/dist/cli/search.d.ts

@@ -0,0 +1 @@
+export declare function runSearchCommand(rawArgs: string[]): Promise<void>;

package/dist/cli/search.js

@@ -0,0 +1,36 @@
+import path from 'node:path';
+import { searchBundle } from '../search.js';
+export async function runSearchCommand(rawArgs) {
+    const args = parseArgs(rawArgs);
+    if (!args.indexDir || !args.query) {
+        throw new Error('Usage: mdorigin search --index <search-dir> [--top-k 10] <query>');
+    }
+    const hits = await searchBundle({
+        indexDir: path.resolve(args.indexDir),
+        query: args.query,
+        topK: args.topK,
+    });
+    console.log(JSON.stringify(hits, null, 2));
+}
+function parseArgs(rawArgs) {
+    const flags = {};
+    const positionals = [];
+    for (let index = 0; index < rawArgs.length; index += 1) {
+        const arg = rawArgs[index];
+        if (arg.startsWith('--')) {
+            const value = rawArgs[index + 1];
+            if (value && !value.startsWith('--')) {
+                flags[arg.slice(2)] = value;
+                index += 1;
+            }
+            continue;
+        }
+        positionals.push(arg);
+    }
+    const topK = flags['top-k'] ? Number.parseInt(flags['top-k'], 10) : undefined;
+    return {
+        indexDir: flags.index,
+        topK: Number.isInteger(topK) && topK > 0 ? topK : undefined,
+        query: positionals.join(' ').trim(),
+    };
+}

package/dist/cloudflare.d.ts

@@ -6,12 +6,14 @@ export type { CloudflareManifest, CloudflareManifestEntry };
 export interface BuildCloudflareManifestOptions {
     rootDir: string;
     siteConfig: ResolvedSiteConfig;
+    searchDir?: string;
 }
 export interface WriteCloudflareBundleOptions {
     rootDir: string;
     outDir: string;
     siteConfig: ResolvedSiteConfig;
     packageImport?: string;
+    searchDir?: string;
 }
 export interface InitCloudflareProjectOptions {
     projectDir: string;

package/dist/cloudflare.js

@@ -1,4 +1,4 @@
-import { mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises';
+import { mkdir, readdir, readFile, realpath, stat, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { createCloudflareWorker } from './adapters/cloudflare.js';
 import { getMediaTypeForPath, isLikelyTextPath, normalizeContentPath, } from './core/content-store.js';
@@ -30,10 +30,14 @@ export async function buildCloudflareManifest(options) {
             base64: (await readFile(filePath)).toString('base64'),
         });
     }
+    const searchEntries = options.searchDir
+        ? await readBundleEntries(path.resolve(options.searchDir))
+        : undefined;
     entries.sort((left, right) => left.path.localeCompare(right.path));
     return {
         entries,
         siteConfig: options.siteConfig,
+        searchEntries,
     };
 }
 export async function writeCloudflareBundle(options) {
@@ -41,6 +45,7 @@ export async function writeCloudflareBundle(options) {
     const manifest = await buildCloudflareManifest({
         rootDir: options.rootDir,
         siteConfig: options.siteConfig,
+        searchDir: options.searchDir,
     });
     const packageImport = options.packageImport ?? 'mdorigin/cloudflare-runtime';
     const workerFile = path.join(outDir, 'worker.mjs');
@@ -84,21 +89,52 @@ export async function initCloudflareProject(options) {
     await writeFile(configFile, wranglerConfig, 'utf8');
     return { configFile };
 }
-async function listFiles(directory) {
+async function listFiles(directory, visitedRealDirectories = new Set()) {
+    const directoryRealPath = await realpath(directory);
+    if (visitedRealDirectories.has(directoryRealPath)) {
+        return [];
+    }
+    visitedRealDirectories.add(directoryRealPath);
     const entries = await readdir(directory, { withFileTypes: true });
     const files = [];
     for (const entry of entries) {
         const fullPath = path.join(directory, entry.name);
-        if (entry.isDirectory()) {
-            files.push(...(await listFiles(fullPath)));
+        const entryStats = await stat(fullPath);
+        if (entryStats.isDirectory()) {
+            files.push(...(await listFiles(fullPath, visitedRealDirectories)));
             continue;
         }
-        if (entry.isFile()) {
+        if (entryStats.isFile()) {
             files.push(fullPath);
         }
     }
     return files;
 }
+async function readBundleEntries(directory) {
+    const files = await listFiles(directory);
+    const entries = [];
+    for (const filePath of files) {
+        const relativePath = path.relative(directory, filePath).replaceAll(path.sep, '/');
+        const mediaType = getMediaTypeForPath(relativePath);
+        if (isLikelyTextPath(relativePath) || relativePath.endsWith('.json')) {
+            entries.push({
+                path: relativePath,
+                kind: 'text',
+                mediaType,
+                text: await readFile(filePath, 'utf8'),
+            });
+            continue;
+        }
+        entries.push({
+            path: relativePath,
+            kind: 'binary',
+            mediaType,
+            base64: (await readFile(filePath)).toString('base64'),
+        });
+    }
+    entries.sort((left, right) => left.path.localeCompare(right.path));
+    return entries;
+}
 async function pathExists(filePath) {
     try {
         await stat(filePath);

package/dist/core/api.d.ts

@@ -0,0 +1,13 @@
+import type { ResolvedSiteConfig } from './site-config.js';
+import type { SearchApi } from '../search.js';
+export interface ApiRouteOptions {
+    searchApi?: SearchApi;
+    siteConfig: ResolvedSiteConfig;
+    requestUrl?: string;
+}
+export interface ApiRouteResponse {
+    status: number;
+    headers: Record<string, string>;
+    body: string;
+}
+export declare function handleApiRoute(pathname: string, searchParams: URLSearchParams | undefined, options: ApiRouteOptions): Promise<ApiRouteResponse | null>;