@songmu/mdhq 0.0.2

package/dist/get-page.js

@@ -0,0 +1,308 @@
+import { loadConfig, resolveRoot } from "./config/config.js";
+import { resolveHostConfig } from "./config/match.js";
+import { convertHtml } from "./convert/convert-html.js";
+import { localizeAssets } from "./assets/localize.js";
+import { httpDateToRfc3339, isRfc3339DateTime, rfc3339ToHttpDate } from "./date.js";
+import { MdhqError } from "./errors.js";
+import { buildFrontmatter, markdownContentDigest, refreshFrontmatter, serializeDocument } from "./frontmatter/frontmatter.js";
+import { fetchHtml, fetchWithEnvProxy } from "./http/fetch.js";
+import { transformMarkdown } from "./markdown/transform.js";
+import { storagePathForUrl } from "./path/storage-path.js";
+import { inspectDestination, saveDocument } from "./storage/save.js";
+import { normalizeHost, parseHttpUrl, sameHttpTarget } from "./url/identity.js";
+function varyNames(value) {
+    return [
+        ...new Set((value ?? "")
+            .split(",")
+            .map((name) => name.trim().toLowerCase())
+            .filter(Boolean))
+    ];
+}
+function hasCredentialHeaders(headers) {
+    return (headers ?? []).some((header) => {
+        const name = header.name.toLowerCase();
+        return name === "authorization" || name === "cookie";
+    });
+}
+export async function getPage(options) {
+    const requestedUrl = parseHttpUrl(options.url).href;
+    const loaded = await loadConfig(options.configPath);
+    const warnings = [];
+    const warn = (warning) => {
+        warnings.push(warning);
+        options.onWarning?.(warning);
+    };
+    for (const warning of loaded.warnings) {
+        warn(warning);
+    }
+    const root = resolveRoot(options.root, loaded.config);
+    return getPageAttempt({ options, requestedUrl, loaded, warnings, warn, root }, requestedUrl, 2, true);
+}
+async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeadersAllowed) {
+    const { options, requestedUrl, loaded, warnings, warn, root } = context;
+    const requested = new URL(fetchUrl);
+    const requestedConfig = resolveHostConfig(normalizeHost(requested), requested.pathname, loaded.config.hosts ?? {});
+    const requestedEntryKey = requestedConfig?.entryQueryKey ?? undefined;
+    const requestedPath = storagePathForUrl({
+        root,
+        url: requested,
+        ...(requestedEntryKey ? { entryQueryKey: requestedEntryKey } : {})
+    });
+    const requestedExisting = await inspectDestination(requestedPath, fetchUrl, requestedEntryKey, root);
+    if (requestedExisting && !options.update) {
+        return {
+            requestedUrl,
+            sourceUrl: requestedExisting.sourceUrl,
+            path: requestedPath,
+            status: "skipped",
+            assets: [],
+            warnings
+        };
+    }
+    const headers = callerHeadersAllowed ? options.headers : [];
+    const userAgent = options.userAgent ?? loaded.config.userAgent;
+    const timeoutMs = options.timeoutMs ?? loaded.config.timeoutMs;
+    const maxResponseBytes = options.maxResponseBytes ?? loaded.config.maxResponseBytes;
+    const maxRedirects = options.maxRedirects ?? loaded.config.maxRedirects;
+    const http = {
+        ...(headers ? { headers } : {}),
+        ...(userAgent ? { userAgent } : {}),
+        ...(timeoutMs !== undefined ? { timeoutMs } : {}),
+        ...(maxResponseBytes !== undefined ? { maxResponseBytes } : {}),
+        ...(maxRedirects !== undefined ? { maxRedirects } : {})
+    };
+    let conditional;
+    if (options.update &&
+        requestedExisting &&
+        requestedExisting.vary?.length === 0 &&
+        !hasCredentialHeaders(headers) &&
+        sameHttpTarget(requestedExisting.sourceUrl, fetchUrl)) {
+        if (requestedExisting.etag) {
+            conditional = { etag: requestedExisting.etag };
+        }
+        else {
+            const lastModified = rfc3339ToHttpDate(requestedExisting.lastModified);
+            if (lastModified) {
+                conditional = { lastModified };
+            }
+        }
+    }
+    const fetched = await fetchHtml(fetchUrl, {
+        ...http,
+        ...(conditional ? { conditional } : {})
+    });
+    const responseHeadersAllowed = callerHeadersAllowed && fetched.customHeadersAllowed;
+    const now = options.now?.() ?? new Date();
+    const normalizeLastModified = (value, fallback) => {
+        const validFallback = rfc3339ToHttpDate(fallback) ? fallback : undefined;
+        if (!value) {
+            return validFallback;
+        }
+        const normalized = httpDateToRfc3339(value);
+        if (!normalized) {
+            warn({
+                code: "INVALID_LAST_MODIFIED",
+                message: `Invalid Last-Modified response header: ${value}`,
+                url: fetched.finalUrl
+            });
+            return undefined;
+        }
+        return normalized;
+    };
+    if (fetched.notModified) {
+        if (!requestedExisting || !conditional) {
+            throw new MdhqError("FETCH_FAILED", `HTTP 304 for ${fetchUrl} without a matching stored validator`);
+        }
+        const lastModified = normalizeLastModified(fetched.lastModified, requestedExisting.lastModified);
+        const vary = varyNames(fetched.vary);
+        const validatorsReusable = vary.length === 0 && !hasCredentialHeaders(headers);
+        const etag = validatorsReusable
+            ? fetched.etag ?? requestedExisting.etag
+            : undefined;
+        const reusableLastModified = validatorsReusable ? lastModified : undefined;
+        const frontmatter = refreshFrontmatter(requestedExisting.frontmatter, {
+            sourceUrl: requestedExisting.sourceUrl,
+            requestedUrl,
+            created: isRfc3339DateTime(requestedExisting.created)
+                ? requestedExisting.created
+                : now,
+            modified: now,
+            contentDigest: requestedExisting.contentDigest,
+            ...(etag ? { etag } : {}),
+            ...(reusableLastModified
+                ? { lastModified: reusableLastModified }
+                : {}),
+            ...(vary.length > 0
+                ? { vary }
+                : etag || reusableLastModified
+                    ? { vary: [] }
+                    : {}),
+            ...(loaded.config.frontmatter ? { config: loaded.config.frontmatter } : {})
+        });
+        const content = serializeDocument(frontmatter, requestedExisting.markdown);
+        const storageStatus = await saveDocument({
+            path: requestedPath,
+            content,
+            sourceUrl: requestedExisting.sourceUrl,
+            update: true,
+            expectedContent: requestedExisting.content,
+            root,
+            ...(requestedEntryKey ? { entryQueryKey: requestedEntryKey } : {})
+        });
+        if (storageStatus === "conflicted") {
+            if (retriesRemaining === 0) {
+                throw new MdhqError("STORAGE_ERROR", `Destination changed repeatedly while updating ${requestedPath}`);
+            }
+            return getPageAttempt(context, fetchUrl, retriesRemaining - 1, responseHeadersAllowed);
+        }
+        return {
+            requestedUrl,
+            sourceUrl: requestedExisting.sourceUrl,
+            path: requestedPath,
+            status: storageStatus === "updated" ? "unchanged" : storageStatus,
+            assets: [],
+            warnings
+        };
+    }
+    const finalUrl = new URL(fetched.finalUrl);
+    const matchedConfig = resolveHostConfig(normalizeHost(finalUrl), finalUrl.pathname, loaded.config.hosts ?? {});
+    const entryQueryKey = matchedConfig?.entryQueryKey ?? undefined;
+    const markdownPath = storagePathForUrl({
+        root,
+        url: finalUrl,
+        ...(entryQueryKey ? { entryQueryKey } : {})
+    });
+    const existing = await inspectDestination(markdownPath, finalUrl.href, entryQueryKey, root);
+    if (options.update &&
+        existing &&
+        markdownPath !== requestedPath) {
+        if (retriesRemaining === 0) {
+            throw new MdhqError("STORAGE_ERROR", `Redirect destination changed repeatedly while updating ${markdownPath}`);
+        }
+        return getPageAttempt(context, finalUrl.href, retriesRemaining - 1, responseHeadersAllowed);
+    }
+    if (existing && !options.update) {
+        return {
+            requestedUrl,
+            sourceUrl: finalUrl.href,
+            path: markdownPath,
+            status: "skipped",
+            assets: [],
+            warnings
+        };
+    }
+    const expectedExisting = markdownPath === requestedPath ? requestedExisting : existing;
+    const responseCredentialed = responseHeadersAllowed && hasCredentialHeaders(http.headers);
+    const converted = await convertHtml({
+        html: fetched.html,
+        url: finalUrl,
+        defuddle: {
+            ...loaded.config.defuddle,
+            fetch: fetchWithEnvProxy,
+            useAsync: options.useAsync ??
+                loaded.config.defuddle?.useAsync ??
+                loaded.config.useAsync ??
+                true
+        }
+    });
+    let metadata = converted.metadata;
+    if (converted.metadata.image) {
+        try {
+            const image = new URL(converted.metadata.image, finalUrl);
+            if (image.protocol !== "http:" && image.protocol !== "https:") {
+                throw new TypeError(`Unsupported image URL scheme: ${image.protocol}`);
+            }
+            metadata = {
+                ...converted.metadata,
+                image: image.href
+            };
+        }
+        catch {
+            const { image: _invalidImage, ...metadataWithoutImage } = converted.metadata;
+            metadata = metadataWithoutImage;
+            warn({
+                code: "INVALID_IMAGE_URL",
+                message: `Invalid representative image URL: ${converted.metadata.image}`,
+                url: finalUrl.href
+            });
+        }
+    }
+    const transformed = transformMarkdown(converted.markdown, finalUrl.href);
+    const assetsEnabled = options.assets ?? loaded.config.assets ?? true;
+    const localized = assetsEnabled
+        ? await localizeAssets({
+            markdown: transformed.markdown,
+            imageUrls: transformed.imageUrls,
+            ...(metadata.image ? { representativeImage: metadata.image } : {}),
+            markdownPath,
+            root,
+            baseUrl: finalUrl.href,
+            http: responseHeadersAllowed ? http : { ...http, headers: [] },
+            warn
+        })
+        : {
+            markdown: transformed.markdown,
+            assets: [],
+            ...(metadata.image ? { representativeImage: metadata.image } : {})
+        };
+    const created = isRfc3339DateTime(expectedExisting?.created)
+        ? expectedExisting.created
+        : now;
+    const contentDigest = markdownContentDigest(localized.markdown);
+    const lastModified = normalizeLastModified(fetched.lastModified);
+    const vary = varyNames(fetched.vary);
+    const validatorsReusable = vary.length === 0 && !responseCredentialed;
+    const etag = validatorsReusable ? fetched.etag : undefined;
+    const reusableLastModified = validatorsReusable ? lastModified : undefined;
+    const frontmatter = buildFrontmatter({
+        metadata,
+        sourceUrl: finalUrl.href,
+        requestedUrl,
+        created,
+        modified: now,
+        contentDigest,
+        ...(etag ? { etag } : {}),
+        ...(reusableLastModified ? { lastModified: reusableLastModified } : {}),
+        ...(vary.length > 0
+            ? { vary }
+            : etag || reusableLastModified
+                ? { vary: [] }
+                : {}),
+        ...(localized.representativeImage
+            ? { image: localized.representativeImage }
+            : {}),
+        ...(localized.representativeImageSource
+            ? { imageSource: localized.representativeImageSource }
+            : {}),
+        ...(loaded.config.frontmatter ? { config: loaded.config.frontmatter } : {})
+    });
+    const content = serializeDocument(frontmatter, localized.markdown);
+    const storageStatus = await saveDocument({
+        path: markdownPath,
+        content,
+        sourceUrl: finalUrl.href,
+        update: options.update ?? false,
+        ...(options.update
+            ? { expectedContent: expectedExisting?.content ?? null }
+            : {}),
+        root,
+        ...(entryQueryKey ? { entryQueryKey } : {})
+    });
+    if (storageStatus === "conflicted") {
+        if (retriesRemaining === 0) {
+            throw new MdhqError("STORAGE_ERROR", `Destination changed repeatedly while updating ${markdownPath}`);
+        }
+        return getPageAttempt(context, fetchUrl, retriesRemaining - 1, responseHeadersAllowed);
+    }
+    return {
+        requestedUrl,
+        sourceUrl: finalUrl.href,
+        path: markdownPath,
+        status: storageStatus === "updated" &&
+            expectedExisting?.contentDigest === contentDigest
+            ? "unchanged"
+            : storageStatus,
+        assets: localized.assets,
+        warnings
+    };
+}

package/dist/http/fetch.d.ts

@@ -0,0 +1,46 @@
+import type { HeaderValue } from "../types.js";
+export interface FetchResourceOptions {
+    headers?: HeaderValue[];
+    userAgent?: string;
+    timeoutMs?: number;
+    maxResponseBytes?: number;
+    maxRedirects?: number;
+    acceptedContentTypes?: string[];
+    allowNotModified?: boolean;
+    conditional?: {
+        etag?: string;
+        lastModified?: string;
+    };
+}
+export interface FetchedResource {
+    body: Uint8Array;
+    finalUrl: string;
+    contentType: string;
+    status: number;
+    customHeadersAllowed: boolean;
+    redirected: boolean;
+    notModified: boolean;
+    etag?: string;
+    lastModified?: string;
+    vary?: string;
+    cacheControl?: string;
+}
+export declare const fetchWithEnvProxy: typeof globalThis.fetch;
+export declare function fetchResource(input: string | URL, options?: FetchResourceOptions): Promise<FetchedResource>;
+export type FetchedHtml = {
+    notModified: true;
+    finalUrl: string;
+    customHeadersAllowed: boolean;
+    etag?: string;
+    lastModified?: string;
+    vary?: string;
+} | {
+    notModified: false;
+    html: string;
+    finalUrl: string;
+    customHeadersAllowed: boolean;
+    etag?: string;
+    lastModified?: string;
+    vary?: string;
+};
+export declare function fetchHtml(input: string | URL, options?: FetchResourceOptions): Promise<FetchedHtml>;

package/dist/http/fetch.js

@@ -0,0 +1,195 @@
+import { EnvHttpProxyAgent, Headers, fetch as undiciFetch } from "undici";
+import { MdhqError } from "../errors.js";
+import { DEFAULT_USER_AGENT } from "../version.js";
+import { parseHttpUrl } from "../url/identity.js";
+const proxyAgent = new EnvHttpProxyAgent();
+const undiciProxyFetch = undiciFetch;
+export const fetchWithEnvProxy = (input, init) => undiciProxyFetch(input, { ...init, dispatcher: proxyAgent });
+function requestHeaders(options, includeCustomHeaders, includeStoredConditional) {
+    const headers = new Headers({
+        accept: options.acceptedContentTypes?.join(", ") ?? "*/*",
+        "user-agent": options.userAgent ?? DEFAULT_USER_AGENT
+    });
+    if (includeCustomHeaders) {
+        for (const header of options.headers ?? []) {
+            headers.append(header.name, header.value);
+        }
+    }
+    if (options.conditional) {
+        headers.delete("if-none-match");
+        headers.delete("if-modified-since");
+    }
+    if (includeStoredConditional && options.conditional) {
+        if (options.conditional.etag) {
+            headers.set("if-none-match", options.conditional.etag);
+        }
+        else if (options.conditional.lastModified) {
+            headers.set("if-modified-since", options.conditional.lastModified);
+        }
+    }
+    return headers;
+}
+function contentType(value) {
+    return value?.split(";", 1)[0]?.trim().toLowerCase() ?? "";
+}
+async function readLimited(response, limit) {
+    const length = Number(response.headers.get("content-length"));
+    if (Number.isFinite(length) && length > limit) {
+        await response.body?.cancel().catch(() => undefined);
+        throw new MdhqError("RESPONSE_TOO_LARGE", `Response exceeds ${limit} bytes`);
+    }
+    if (!response.body) {
+        return new Uint8Array();
+    }
+    const reader = response.body.getReader();
+    const chunks = [];
+    let total = 0;
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done) {
+            break;
+        }
+        total += value.byteLength;
+        if (total > limit) {
+            await reader.cancel();
+            throw new MdhqError("RESPONSE_TOO_LARGE", `Response exceeds ${limit} bytes`);
+        }
+        chunks.push(value);
+    }
+    const body = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        body.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return body;
+}
+export async function fetchResource(input, options = {}) {
+    const timeoutMs = options.timeoutMs ?? 30_000;
+    const maxResponseBytes = options.maxResponseBytes ?? 20 * 1024 * 1024;
+    const maxRedirects = options.maxRedirects ?? 10;
+    let url = parseHttpUrl(input);
+    let customHeadersAllowed = true;
+    for (let redirects = 0;; redirects += 1) {
+        let response;
+        try {
+            response = (await undiciFetch(url, {
+                dispatcher: proxyAgent,
+                headers: requestHeaders(options, customHeadersAllowed, redirects === 0),
+                redirect: "manual",
+                signal: AbortSignal.timeout(timeoutMs)
+            }));
+        }
+        catch (error) {
+            throw new MdhqError("FETCH_FAILED", `Failed to fetch ${url.href}`, { cause: error });
+        }
+        if ([301, 302, 303, 307, 308].includes(response.status)) {
+            const location = response.headers.get("location");
+            if (!location) {
+                await response.body?.cancel().catch(() => undefined);
+                throw new MdhqError("FETCH_FAILED", `Redirect response has no Location: ${url.href}`);
+            }
+            if (redirects >= maxRedirects) {
+                await response.body?.cancel().catch(() => undefined);
+                throw new MdhqError("TOO_MANY_REDIRECTS", `Too many redirects: ${String(input)}`);
+            }
+            let nextUrl;
+            try {
+                nextUrl = parseHttpUrl(new URL(location, url));
+            }
+            catch (error) {
+                await response.body?.cancel().catch(() => undefined);
+                if (error instanceof MdhqError && error.code === "UNSUPPORTED_SCHEME") {
+                    throw error;
+                }
+                throw new MdhqError("FETCH_FAILED", `Invalid redirect Location for ${url.href}: ${location}`, { cause: error });
+            }
+            if (nextUrl.origin !== url.origin) {
+                customHeadersAllowed = false;
+            }
+            await response.body?.cancel().catch(() => undefined);
+            url = nextUrl;
+            continue;
+        }
+        const etag = response.headers.get("etag")?.trim() || undefined;
+        const lastModified = response.headers.get("last-modified")?.trim() || undefined;
+        const vary = response.headers.get("vary")?.trim() || undefined;
+        const cacheControl = response.headers.get("cache-control")?.trim() || undefined;
+        if (response.status === 304 && options.allowNotModified && redirects === 0) {
+            await response.body?.cancel().catch(() => undefined);
+            return {
+                body: new Uint8Array(),
+                finalUrl: url.href,
+                contentType: "",
+                status: response.status,
+                customHeadersAllowed,
+                redirected: redirects > 0,
+                notModified: true,
+                ...(etag ? { etag } : {}),
+                ...(lastModified ? { lastModified } : {}),
+                ...(vary ? { vary } : {}),
+                ...(cacheControl ? { cacheControl } : {})
+            };
+        }
+        if (!response.ok) {
+            await response.body?.cancel().catch(() => undefined);
+            throw new MdhqError("FETCH_FAILED", `HTTP ${response.status} for ${url.href}`);
+        }
+        const type = contentType(response.headers.get("content-type"));
+        if (options.acceptedContentTypes && !options.acceptedContentTypes.includes(type)) {
+            await response.body?.cancel().catch(() => undefined);
+            throw new MdhqError("UNSUPPORTED_CONTENT_TYPE", `Unsupported Content-Type ${type || "(missing)"} for ${url.href}`);
+        }
+        let body;
+        try {
+            body = await readLimited(response, maxResponseBytes);
+        }
+        catch (error) {
+            if (error instanceof MdhqError) {
+                throw error;
+            }
+            throw new MdhqError("FETCH_FAILED", `Failed to read response body from ${url.href}`, {
+                cause: error
+            });
+        }
+        return {
+            body,
+            finalUrl: url.href,
+            contentType: type,
+            status: response.status,
+            customHeadersAllowed,
+            redirected: redirects > 0,
+            notModified: false,
+            ...(etag ? { etag } : {}),
+            ...(lastModified ? { lastModified } : {}),
+            ...(vary ? { vary } : {}),
+            ...(cacheControl ? { cacheControl } : {})
+        };
+    }
+}
+export async function fetchHtml(input, options = {}) {
+    const resource = await fetchResource(input, {
+        ...options,
+        allowNotModified: true,
+        acceptedContentTypes: ["text/html", "application/xhtml+xml"]
+    });
+    if (resource.notModified) {
+        return {
+            notModified: true,
+            finalUrl: resource.finalUrl,
+            customHeadersAllowed: resource.customHeadersAllowed,
+            ...(resource.etag ? { etag: resource.etag } : {}),
+            ...(resource.lastModified ? { lastModified: resource.lastModified } : {}),
+            ...(resource.vary ? { vary: resource.vary } : {})
+        };
+    }
+    return {
+        notModified: false,
+        html: new TextDecoder().decode(resource.body),
+        finalUrl: resource.finalUrl,
+        customHeadersAllowed: resource.customHeadersAllowed,
+        ...(resource.etag ? { etag: resource.etag } : {}),
+        ...(resource.lastModified ? { lastModified: resource.lastModified } : {}),
+        ...(resource.vary ? { vary: resource.vary } : {})
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export { convertHtml } from "./convert/convert-html.js";
+export { getPage } from "./get-page.js";
+export { MdhqError } from "./errors.js";
+export type { MdhqErrorCode } from "./errors.js";
+export type { MdhqConfig } from "./config/config.js";
+export type { AssetResult, ConvertedPage, ConvertHtmlOptions, GetPageOptions, GetPageResult, HeaderValue, MdhqWarning, PageMetadata } from "./types.js";

package/dist/index.js

@@ -0,0 +1,3 @@
+export { convertHtml } from "./convert/convert-html.js";
+export { getPage } from "./get-page.js";
+export { MdhqError } from "./errors.js";

package/dist/list-files.d.ts

@@ -0,0 +1,8 @@
+import type { MdhqWarning } from "./types.js";
+export interface ListMarkdownFilesOptions {
+    root?: string;
+    configPath?: string;
+    fullPath?: boolean;
+    onWarning?: (warning: MdhqWarning) => void;
+}
+export declare function listMarkdownFiles(options?: ListMarkdownFilesOptions): Promise<string[]>;

package/dist/list-files.js

@@ -0,0 +1,35 @@
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+import { loadConfig, resolveRoot } from "./config/config.js";
+import { MdhqError } from "./errors.js";
+async function collectMarkdownFiles(root, relativeDirectory, files) {
+    const directory = path.join(root, relativeDirectory);
+    const entries = await readdir(directory, { withFileTypes: true });
+    for (const entry of entries) {
+        const relativePath = path.join(relativeDirectory, entry.name);
+        if (entry.isDirectory()) {
+            await collectMarkdownFiles(root, relativePath, files);
+        }
+        else if (entry.isFile() && entry.name.endsWith(".md")) {
+            files.push(relativePath);
+        }
+    }
+}
+export async function listMarkdownFiles(options = {}) {
+    const loaded = await loadConfig(options.configPath);
+    for (const warning of loaded.warnings) {
+        options.onWarning?.(warning);
+    }
+    const root = resolveRoot(options.root, loaded.config);
+    const files = [];
+    try {
+        await collectMarkdownFiles(root, "", files);
+    }
+    catch (error) {
+        throw new MdhqError("STORAGE_ERROR", `Failed to list storage root: ${root}`, {
+            cause: error
+        });
+    }
+    files.sort();
+    return options.fullPath ? files.map((file) => path.join(root, file)) : files;
+}

package/dist/markdown/transform.d.ts

@@ -0,0 +1,6 @@
+export interface MarkdownTransformResult {
+    markdown: string;
+    imageUrls: string[];
+}
+export declare function transformMarkdown(markdown: string, baseUrl: string): MarkdownTransformResult;
+export declare function rewriteImageUrls(markdown: string, replacements: ReadonlyMap<string, string>): string;