mdorigin 0.1.7 → 0.2.0

package/dist/cloudflare.js

@@ -1,12 +1,23 @@
-import { mkdir, readdir, readFile, realpath, stat, writeFile } from 'node:fs/promises';
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { createReadStream } from 'node:fs';
+import { copyFile, mkdir, readFile, readdir, realpath, rm, stat, writeFile, } from 'node:fs/promises';
 import path from 'node:path';
 import { createCloudflareWorker } from './adapters/cloudflare.js';
-import { getMediaTypeForPath, isLikelyTextPath, normalizeContentPath, } from './core/content-store.js';
+import { getMediaTypeForPath, isIgnoredContentName, isLikelyTextPath, normalizeContentPath, } from './core/content-store.js';
 export { createCloudflareWorker };
+const DEFAULT_ASSETS_MAX_BYTES = 25 * 1024 * 1024;
+const DEFAULT_ASSETS_BINDING = 'ASSETS';
+const DEFAULT_R2_BINDING = 'MDORIGIN_R2';
+const BUNDLE_FILE_NAME = 'bundle.json';
+const R2_STATE_FILE_NAME = 'r2-sync-state.json';
 export async function buildCloudflareManifest(options) {
     const rootDir = path.resolve(options.rootDir);
     const files = await listFiles(rootDir);
     const entries = [];
+    const binaryMode = options.binaryMode ?? 'inline';
+    const assetsMaxBytes = options.assetsMaxBytes ?? DEFAULT_ASSETS_MAX_BYTES;
+    const r2Binding = options.r2Binding ?? DEFAULT_R2_BINDING;
     for (const filePath of files) {
         const relativePath = path.relative(rootDir, filePath).replaceAll(path.sep, '/');
         const normalizedPath = normalizeContentPath(relativePath);
@@ -23,12 +34,21 @@ export async function buildCloudflareManifest(options) {
             });
             continue;
         }
-        entries.push({
-            path: normalizedPath,
-            kind: 'binary',
-            mediaType,
-            base64: (await readFile(filePath)).toString('base64'),
-        });
+        if (binaryMode === 'inline') {
+            const bytes = await readFile(filePath);
+            entries.push({
+                path: normalizedPath,
+                kind: 'binary',
+                mediaType,
+                base64: bytes.toString('base64'),
+            });
+            continue;
+        }
+        const fileStats = await stat(filePath);
+        entries.push(await buildExternalBinaryEntry(filePath, normalizedPath, mediaType, fileStats.size, {
+            assetsMaxBytes,
+            r2Binding,
+        }));
     }
     const searchEntries = options.searchDir
         ? await readBundleEntries(path.resolve(options.searchDir))
@@ -38,17 +58,32 @@ export async function buildCloudflareManifest(options) {
         entries,
         siteConfig: options.siteConfig,
         searchEntries,
+        runtime: binaryMode === 'external'
+            ? {
+                binaryMode,
+                r2Binding,
+            }
+            : {
+                binaryMode,
+            },
     };
 }
 export async function writeCloudflareBundle(options) {
     const outDir = path.resolve(options.outDir);
+    const binaryMode = options.binaryMode ?? 'inline';
+    const assetsMaxBytes = options.assetsMaxBytes ?? DEFAULT_ASSETS_MAX_BYTES;
+    const r2Binding = options.r2Binding ?? DEFAULT_R2_BINDING;
     const manifest = await buildCloudflareManifest({
         rootDir: options.rootDir,
         siteConfig: options.siteConfig,
         searchDir: options.searchDir,
+        binaryMode,
+        assetsMaxBytes,
+        r2Binding,
     });
     const packageImport = options.packageImport ?? 'mdorigin/cloudflare-runtime';
     const workerFile = path.join(outDir, 'worker.mjs');
+    const bundleFile = path.join(outDir, BUNDLE_FILE_NAME);
     const configImportPath = options.configModulePath
         ? toPosixPath(path.relative(outDir, options.configModulePath))
         : null;
@@ -83,10 +118,18 @@ export async function writeCloudflareBundle(options) {
         '',
     ].join('\n');
     await mkdir(outDir, { recursive: true });
+    const metadata = await writeExternalBinaryStaging(path.resolve(options.rootDir), outDir, manifest, {
+        binaryMode,
+        assetsMaxBytes,
+        r2Binding,
+        siteTitle: options.siteConfig.siteTitle,
+    });
     await writeFile(workerFile, workerSource, 'utf8');
+    await writeFile(bundleFile, JSON.stringify(metadata, null, 2), 'utf8');
     return {
         manifest,
         workerFile,
+        bundleFile,
     };
 }
 export async function initCloudflareProject(options) {
@@ -96,8 +139,14 @@ export async function initCloudflareProject(options) {
     if (existing && !options.force) {
         throw new Error(`Refusing to overwrite ${configFile}. Re-run with --force to replace it.`);
     }
+    const bundleMetadata = await readCloudflareBundleMetadata(options.workerEntry);
+    if (bundleMetadata?.binaryMode === 'external' &&
+        bundleMetadata.r2Objects.length > 0 &&
+        !options.r2Bucket) {
+        throw new Error('Cloudflare bundle contains R2-backed binaries. Re-run init cloudflare with --r2-bucket <bucket-name>.');
+    }
     const workerName = options.workerName ??
-        slugifyWorkerName(options.siteTitle) ??
+        slugifyWorkerName(bundleMetadata?.siteTitle ?? options.siteTitle) ??
         'mdorigin-site';
     const compatibilityDate = options.compatibilityDate ?? '2026-03-20';
     const wranglerConfig = [
@@ -107,13 +156,197 @@ export async function initCloudflareProject(options) {
         `  "main": ${JSON.stringify(toPosixPath(path.relative(projectDir, options.workerEntry)))},`,
         `  "compatibility_date": ${JSON.stringify(compatibilityDate)},`,
         '  "compatibility_flags": ["nodejs_compat"]',
+        bundleMetadata?.binaryMode === 'external' && bundleMetadata.assetsDir
+            ? [
+                ',',
+                '  "assets": {',
+                `    "directory": ${JSON.stringify(toPosixPath(path.relative(projectDir, path.join(path.dirname(options.workerEntry), bundleMetadata.assetsDir))))},`,
+                `    "binding": ${JSON.stringify(DEFAULT_ASSETS_BINDING)},`,
+                '    "run_worker_first": true',
+                '  }',
+            ].join('\n')
+            : '',
+        bundleMetadata?.binaryMode === 'external' &&
+            bundleMetadata.r2Objects.length > 0 &&
+            bundleMetadata.r2Binding &&
+            options.r2Bucket
+            ? [
+                ',',
+                '  "r2_buckets": [',
+                '    {',
+                `      "binding": ${JSON.stringify(bundleMetadata.r2Binding)},`,
+                `      "bucket_name": ${JSON.stringify(options.r2Bucket)}`,
+                '    }',
+                '  ]',
+            ].join('\n')
+            : '',
         '}',
         '',
-    ].join('\n');
+    ]
+        .filter(Boolean)
+        .join('\n');
     await mkdir(projectDir, { recursive: true });
     await writeFile(configFile, wranglerConfig, 'utf8');
     return { configFile };
 }
+export async function syncCloudflareR2(options) {
+    const outDir = path.resolve(options.dir);
+    const bundleFile = path.join(outDir, BUNDLE_FILE_NAME);
+    const metadata = await readBundleMetadataFile(bundleFile);
+    if (metadata.binaryMode !== 'external' || metadata.r2Objects.length === 0) {
+        throw new Error(`No R2-backed binaries found in ${bundleFile}.`);
+    }
+    const stateFile = path.join(outDir, R2_STATE_FILE_NAME);
+    const state = await readR2SyncState(stateFile);
+    const runCommand = options.runCommand ?? runWranglerCommand;
+    let uploadedCount = 0;
+    let skippedCount = 0;
+    for (const object of metadata.r2Objects) {
+        const stateKey = `${options.bucketName}:${object.storageKey}`;
+        if (!options.force && state.uploaded[stateKey]) {
+            skippedCount += 1;
+            continue;
+        }
+        const filePath = path.join(outDir, object.file);
+        const result = runCommand('wrangler', [
+            'r2',
+            'object',
+            'put',
+            `${options.bucketName}/${object.storageKey}`,
+            '--file',
+            filePath,
+            '--content-type',
+            object.mediaType,
+            '--remote',
+        ]);
+        if (result.status !== 0) {
+            throw new Error(result.stderr || `Failed to upload R2 object ${object.storageKey}.`);
+        }
+        state.uploaded[stateKey] = {
+            syncedAt: new Date().toISOString(),
+        };
+        uploadedCount += 1;
+    }
+    await writeFile(stateFile, JSON.stringify(state, null, 2), 'utf8');
+    return {
+        uploadedCount,
+        skippedCount,
+        stateFile,
+    };
+}
+async function buildExternalBinaryEntry(filePath, normalizedPath, mediaType, byteSize, options) {
+    if (byteSize <= options.assetsMaxBytes) {
+        return {
+            path: normalizedPath,
+            kind: 'binary',
+            mediaType,
+            storageKind: 'assets',
+            storageKey: normalizedPath,
+            byteSize,
+        };
+    }
+    return {
+        path: normalizedPath,
+        kind: 'binary',
+        mediaType,
+        storageKind: 'r2',
+        storageKey: await buildR2StorageKey(filePath, normalizedPath),
+        byteSize,
+    };
+}
+async function writeExternalBinaryStaging(rootDir, outDir, manifest, options) {
+    const assetsDir = path.join(outDir, 'assets');
+    const r2Dir = path.join(outDir, 'r2');
+    await rm(assetsDir, { recursive: true, force: true });
+    await rm(r2Dir, { recursive: true, force: true });
+    const r2Objects = new Map();
+    if (options.binaryMode === 'external') {
+        for (const entry of manifest.entries) {
+            if (entry.kind !== 'binary' || !('storageKind' in entry)) {
+                continue;
+            }
+            const sourceFile = path.resolve(rootDir, entry.path);
+            if (entry.storageKind === 'assets') {
+                const targetFile = path.join(assetsDir, entry.storageKey);
+                await mkdir(path.dirname(targetFile), { recursive: true });
+                await copyFile(sourceFile, targetFile);
+                continue;
+            }
+            const relativeFile = toPosixPath(path.join('r2', entry.storageKey));
+            const targetFile = path.join(outDir, relativeFile);
+            if (!r2Objects.has(entry.storageKey)) {
+                await mkdir(path.dirname(targetFile), { recursive: true });
+                await copyFile(sourceFile, targetFile);
+                r2Objects.set(entry.storageKey, {
+                    path: entry.path,
+                    mediaType: entry.mediaType,
+                    storageKey: entry.storageKey,
+                    file: relativeFile,
+                    byteSize: entry.byteSize,
+                });
+            }
+        }
+    }
+    return {
+        version: 1,
+        workerEntry: 'worker.mjs',
+        binaryMode: options.binaryMode,
+        assetsMaxBytes: options.binaryMode === 'external' ? options.assetsMaxBytes : undefined,
+        assetsDir: options.binaryMode === 'external' ? 'assets' : undefined,
+        r2Dir: options.binaryMode === 'external' ? 'r2' : undefined,
+        r2Binding: options.binaryMode === 'external' && r2Objects.size > 0
+            ? options.r2Binding
+            : undefined,
+        siteTitle: options.siteTitle,
+        r2Objects: Array.from(r2Objects.values()).sort((left, right) => left.storageKey.localeCompare(right.storageKey)),
+    };
+}
+async function readCloudflareBundleMetadata(workerEntry) {
+    const bundleFile = path.join(path.dirname(workerEntry), BUNDLE_FILE_NAME);
+    if (!(await pathExists(bundleFile))) {
+        return null;
+    }
+    return readBundleMetadataFile(bundleFile);
+}
+async function readBundleMetadataFile(bundleFile) {
+    return JSON.parse(await readFile(bundleFile, 'utf8'));
+}
+async function readR2SyncState(stateFile) {
+    if (!(await pathExists(stateFile))) {
+        return { version: 1, uploaded: {} };
+    }
+    return JSON.parse(await readFile(stateFile, 'utf8'));
+}
+async function buildR2StorageKey(filePath, normalizedPath) {
+    const extension = path.posix.extname(normalizedPath).toLowerCase();
+    const hash = await hashFile(filePath);
+    return extension ? `binary/${hash}${extension}` : `binary/${hash}`;
+}
+async function hashFile(filePath) {
+    const hash = createHash('sha256');
+    await new Promise((resolve, reject) => {
+        const stream = createReadStream(filePath);
+        stream.on('data', (chunk) => {
+            hash.update(chunk);
+        });
+        stream.on('end', () => resolve());
+        stream.on('error', reject);
+    });
+    return hash.digest('hex');
+}
+function runWranglerCommand(command, args) {
+    const result = spawnSync(command, args, {
+        encoding: 'utf8',
+        stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    if (result.error) {
+        throw result.error;
+    }
+    return {
+        status: result.status,
+        stderr: result.stderr ?? '',
+    };
+}
 async function listFiles(directory, visitedRealDirectories = new Set()) {
     const directoryRealPath = await realpath(directory);
     if (visitedRealDirectories.has(directoryRealPath)) {
@@ -123,6 +356,9 @@ async function listFiles(directory, visitedRealDirectories = new Set()) {
     const entries = await readdir(directory, { withFileTypes: true });
     const files = [];
     for (const entry of entries) {
+        if (isIgnoredContentName(entry.name)) {
+            continue;
+        }
         const fullPath = path.join(directory, entry.name);
         const entryStats = await stat(fullPath);
         if (entryStats.isDirectory()) {

package/dist/core/content-store.d.ts

@@ -15,6 +15,7 @@ export interface ContentStore {
     get(contentPath: string): Promise<ContentEntry | null>;
     listDirectory(contentPath: string): Promise<ContentDirectoryEntry[] | null>;
 }
+export declare function isIgnoredContentName(name: string): boolean;
 export declare class MemoryContentStore implements ContentStore {
     private readonly entries;
     constructor(entries: Iterable<ContentEntry>);

package/dist/core/content-store.js

@@ -1,4 +1,7 @@
 import path from 'node:path';
+export function isIgnoredContentName(name) {
+    return name.startsWith('.');
+}
 export class MemoryContentStore {
     entries;
     constructor(entries) {
@@ -23,7 +26,7 @@ export class MemoryContentStore {
                 continue;
             }
             const [firstSegment, ...rest] = remainder.split('/');
-            if (firstSegment.startsWith('.')) {
+            if (isIgnoredContentName(firstSegment)) {
                 continue;
             }
             if (rest.length === 0) {
@@ -53,6 +56,7 @@ const MEDIA_TYPES = new Map([
     ['.js', 'text/javascript; charset=utf-8'],
     ['.json', 'application/json; charset=utf-8'],
     ['.md', 'text/markdown; charset=utf-8'],
+    ['.mp4', 'video/mp4'],
     ['.pdf', 'application/pdf'],
     ['.py', 'text/plain; charset=utf-8'],
     ['.png', 'image/png'],
@@ -84,6 +88,9 @@ export function normalizeContentPath(inputPath) {
         resolved.includes('/../')) {
         return null;
     }
+    if (resolved.split('/').some(isIgnoredContentName)) {
+        return null;
+    }
     return resolved;
 }
 export function normalizeDirectoryPath(inputPath) {

package/dist/core/extensions.d.ts

@@ -1,7 +1,5 @@
 import type { ManagedIndexEntry, ParsedDocumentMeta } from './markdown.js';
 import type { EditLinkConfig, ResolvedSiteConfig, SiteLogo, SiteNavItem, SiteSocialLink } from './site-config.js';
-import type { TemplateName } from '../html/template-kind.js';
-import type { BuiltInThemeName } from '../html/theme.js';
 type MaybePromise<T> = T | Promise<T>;
 export interface IndexTransformContext {
     mode: 'build' | 'render';
@@ -11,7 +9,7 @@ export interface IndexTransformContext {
     siteConfig?: ResolvedSiteConfig;
 }
 export interface PageRenderModel {
-    kind: 'document' | 'catalog';
+    kind: 'page' | 'listing';
     requestPath: string;
     sourcePath: string;
     siteTitle: string;
@@ -27,8 +25,6 @@ export interface PageRenderModel {
     date?: string;
     showSummary: boolean;
     showDate: boolean;
-    theme: BuiltInThemeName;
-    template: TemplateName;
     topNav: SiteNavItem[];
     footerNav: SiteNavItem[];
     footerText?: string;
@@ -38,10 +34,10 @@ export interface PageRenderModel {
     stylesheetContent?: string;
     canonicalPath?: string;
     alternateMarkdownPath?: string;
-    catalogEntries: ManagedIndexEntry[];
-    catalogRequestPath: string;
-    catalogInitialPostCount: number;
-    catalogLoadMoreStep: number;
+    listingEntries: ManagedIndexEntry[];
+    listingRequestPath: string;
+    listingInitialPostCount: number;
+    listingLoadMoreStep: number;
     searchEnabled: boolean;
 }
 export interface RenderHookContext {

package/dist/core/markdown.js

@@ -1,7 +1,9 @@
 import matter from 'gray-matter';
 import { remark } from 'remark';
 import remarkGfm from 'remark-gfm';
-import remarkHtml from 'remark-html';
+import remarkRehype from 'remark-rehype';
+import rehypeRaw from 'rehype-raw';
+import rehypeStringify from 'rehype-stringify';
 export function getDocumentTitle(meta, body, fallback) {
     return (firstNonEmptyString(meta.title, meta.name) ??
         extractFirstHeading(body) ??
@@ -24,7 +26,9 @@ export async function parseMarkdownDocument(sourcePath, markdown) {
 export async function renderMarkdown(markdown) {
     const output = await remark()
         .use(remarkGfm)
-        .use(remarkHtml)
+        .use(remarkRehype, { allowDangerousHtml: true })
+        .use(rehypeRaw)
+        .use(rehypeStringify, { allowDangerousHtml: true })
         .process(markdown);
     return String(output);
 }
@@ -78,15 +82,39 @@ export function extractManagedIndexEntries(markdown) {
         }
         const rawHref = entryMatch[2];
         const href = rewriteMarkdownHref(rawHref);
+        const explicitKind = extractManagedIndexKind(lines.slice(1));
         entries.push({
-            kind: href.endsWith('/') ? 'directory' : 'article',
+            kind: explicitKind ?? (href.endsWith('/') ? 'directory' : 'article'),
             title: entryMatch[1],
             href,
-            detail: lines[1]?.trim() || undefined,
+            detail: extractManagedIndexDetail(lines.slice(1)),
         });
     }
     return entries;
 }
+function extractManagedIndexKind(lines) {
+    for (const line of lines) {
+        const trimmed = line.trim();
+        const match = trimmed.match(/^<!--\s*mdorigin:index\s+kind=(article|directory)\s*-->$/);
+        if (match) {
+            return match[1];
+        }
+    }
+    return undefined;
+}
+function extractManagedIndexDetail(lines) {
+    for (const line of lines) {
+        const trimmed = line.trim();
+        if (trimmed === '') {
+            continue;
+        }
+        if (/^<!--\s*mdorigin:index\s+kind=(article|directory)\s*-->$/.test(trimmed)) {
+            continue;
+        }
+        return trimmed;
+    }
+    return undefined;
+}
 function normalizeMeta(data) {
     const meta = { ...data };
     if (typeof data.title === 'string') {