@songmu/mdhq 0.0.2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Songmu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,126 @@
+# mdhq - Markdown headquarters (?)
+
+`mdhq` saves web pages as Markdown in a [ghq](https://github.com/x-motemen/ghq)-inspired filesystem layout. It
+uses [Defuddle](https://defuddle.md/) for content extraction and keeps all state in Markdown and asset
+files rather than a database.
+
+## Requirements
+
+- Node.js 22 or newer
+
+## Installation
+
+```sh
+npm install --global @songmu/mdhq
+```
+
+## CLI
+
+```sh
+mdhq get https://example.com/article
+mdhq get --update https://example.com/article
+mdhq get --no-assets https://example.com/article
+mdhq get --json --header 'Cookie: session=value' https://example.com/article
+mdhq list
+mdhq list --full-path
+mdhq root
+```
+
+`mdhq get` prints the absolute Markdown path to stdout by default. Warnings
+are written to stderr. `--json` returns the requested URL, final source URL,
+Markdown path, status, downloaded assets, and warnings.
+
+`mdhq list` recursively lists `.md` files below the storage root, one per
+line, in sorted root-relative form. Use `-p` or `--full-path` to print absolute
+paths. Directory symbolic links are not followed.
+
+`mdhq root` prints the absolute effective storage root.
+
+The storage root is selected in this order:
+
+1. `--root`
+2. `MDHQ_ROOT`
+3. `root` in the configuration file
+4. `$XDG_DATA_HOME/mdhq`, or `~/.local/share/mdhq`
+
+The configuration file is
+`$XDG_CONFIG_HOME/mdhq/config.json`, or
+`~/.config/mdhq/config.json`.
+
+```json
+{
+  "root": "/path/to/mdhq",
+  "assets": false,
+  "useAsync": true,
+  "frontmatter": {
+    "exclude": ["description"],
+    "values": {
+      "collection": "reading"
+    }
+  },
+  "hosts": {
+    "*.example.com": {
+      "entryQueryKey": "entry_id",
+      "paths": {
+        "/search/*": {
+          "entryQueryKey": null
+        }
+      }
+    }
+  }
+}
+```
+
+Exact host and path patterns take precedence over globs. Among matching globs,
+the pattern with the longest literal portion wins. Equally specific matching
+patterns are rejected.
+
+## Library API
+
+```ts
+import { convertHtml, getPage } from "@songmu/mdhq";
+
+const converted = await convertHtml({
+  html,
+  url: "https://example.com/article"
+});
+
+const saved = await getPage({
+  url: "https://example.com/article",
+  root: "/path/to/mdhq",
+  assets: false
+});
+```
+
+`convertHtml` performs extraction without fetching or writing files.
+`getPage` fetches, converts, optionally localizes images, adds frontmatter,
+and saves the document. Set `assets: false` or use `--no-assets` to keep
+absolute image URLs without creating `_assets`.
+
+Saved frontmatter uses Obsidian Web Clipper-compatible names such as `title`,
+`source`, `author`, `published`, `created`, and `description`. mdhq also stores
+`modified` for the last meaningful note change and safe HTTP validators for
+conditional updates. It does not add `type` or `tags` by default; use
+`frontmatter.values` to opt into values such as `"type": "clip"`.
+
+An update returns `updated` when the normalized Markdown body or user-facing
+frontmatter changes and `unchanged` when HTTP returns 304 or the fetched note
+content is unchanged.
+
+## Documentation
+
+- [Current specification](docs/specification.md)
+- [Configuration reference](docs/configuration.md)
+- [Library API reference](docs/library-api.md)
+
+## Development
+
+```sh
+npm install
+npm test
+npm run typecheck
+npm run build
+npm run test:package
+```
+
+CI runs the same checks on Linux, Windows, and macOS.

package/dist/assets/localize.d.ts

@@ -0,0 +1,19 @@
+import { type FetchResourceOptions } from "../http/fetch.js";
+import type { AssetResult, MdhqWarning } from "../types.js";
+export interface LocalizeAssetsOptions {
+    markdown: string;
+    imageUrls: string[];
+    representativeImage?: string;
+    markdownPath: string;
+    root: string;
+    baseUrl: string;
+    http?: FetchResourceOptions;
+    warn: (warning: MdhqWarning) => void;
+}
+export interface LocalizeAssetsResult {
+    markdown: string;
+    assets: AssetResult[];
+    representativeImage?: string;
+    representativeImageSource?: string;
+}
+export declare function localizeAssets(options: LocalizeAssetsOptions): Promise<LocalizeAssetsResult>;

package/dist/assets/localize.js

@@ -0,0 +1,364 @@
+import { createHash } from "node:crypto";
+import path from "node:path";
+import { lstat, readFile, unlink } from "node:fs/promises";
+import { fetchResource } from "../http/fetch.js";
+import { rewriteImageUrls } from "../markdown/transform.js";
+import { publishFileExclusive, replaceFileAtomic, withDestinationLock } from "../storage/atomic.js";
+const CONTENT_TYPE_EXTENSIONS = {
+    "image/avif": ".avif",
+    "image/gif": ".gif",
+    "image/jpeg": ".jpg",
+    "image/png": ".png",
+    "image/svg+xml": ".svg",
+    "image/webp": ".webp"
+};
+const IMAGE_EXTENSIONS = new Set(Object.values(CONTENT_TYPE_EXTENSIONS));
+IMAGE_EXTENSIONS.add(".jpeg");
+IMAGE_EXTENSIONS.add(".jfif");
+const ASSET_CACHE_VERSION = 1;
+function assetExtension(contentType, finalUrl) {
+    const fromType = CONTENT_TYPE_EXTENSIONS[contentType];
+    if (fromType) {
+        return fromType;
+    }
+    const extension = path.posix.extname(new URL(finalUrl).pathname).toLowerCase();
+    return /^\.[a-z0-9]{1,8}$/u.test(extension) ? extension : ".bin";
+}
+async function saveAsset(assetPath, body, root) {
+    if (await publishFileExclusive(assetPath, body, root)) {
+        return "saved";
+    }
+    const existing = await readFile(assetPath);
+    if (Buffer.from(body).equals(existing)) {
+        return "reused";
+    }
+    throw new Error(`Asset digest collision: ${assetPath}`);
+}
+function cachePathForUrl(root, sourceUrl) {
+    const digest = createHash("sha256").update(sourceUrl).digest("hex");
+    return path.join(root, "_assets", ".cache", `${digest}.json`);
+}
+function varyNames(value) {
+    return [
+        ...new Set((value ?? "")
+            .split(",")
+            .map((name) => name.trim().toLowerCase())
+            .filter(Boolean))
+    ];
+}
+function hasCredentialHeaders(headers) {
+    return (headers ?? []).some((header) => {
+        const name = header.name.toLowerCase();
+        return name === "authorization" || name === "cookie";
+    });
+}
+function hasNoStore(value) {
+    return (value ?? "")
+        .split(",")
+        .some((directive) => directive.trim().toLowerCase().split("=", 1)[0] === "no-store");
+}
+function requestHasNoStore(headers) {
+    return (headers ?? []).some((header) => header.name.toLowerCase() === "cache-control" &&
+        hasNoStore(header.value));
+}
+function parseAssetCacheEntry(value) {
+    if (!value || typeof value !== "object") {
+        return undefined;
+    }
+    const entry = value;
+    if (entry.version !== ASSET_CACHE_VERSION ||
+        typeof entry.sourceUrl !== "string" ||
+        typeof entry.finalUrl !== "string" ||
+        typeof entry.assetFile !== "string" ||
+        !/^[a-f0-9]{64}\.[a-z0-9]{1,8}$/u.test(entry.assetFile) ||
+        typeof entry.contentType !== "string" ||
+        !Array.isArray(entry.vary) ||
+        !entry.vary.every((name) => typeof name === "string") ||
+        (entry.etag !== undefined && typeof entry.etag !== "string") ||
+        (entry.lastModified !== undefined && typeof entry.lastModified !== "string")) {
+        return undefined;
+    }
+    return {
+        version: ASSET_CACHE_VERSION,
+        sourceUrl: entry.sourceUrl,
+        finalUrl: entry.finalUrl,
+        assetFile: entry.assetFile,
+        contentType: entry.contentType,
+        ...(typeof entry.etag === "string" ? { etag: entry.etag } : {}),
+        ...(typeof entry.lastModified === "string"
+            ? { lastModified: entry.lastModified }
+            : {}),
+        vary: entry.vary
+    };
+}
+async function readAssetCache(cachePath, sourceUrl) {
+    try {
+        const metadata = await lstat(cachePath);
+        if (!metadata.isFile() || metadata.isSymbolicLink()) {
+            return {
+                writable: false,
+                warning: {
+                    code: "ASSET_CACHE_INVALID",
+                    message: `Invalid asset cache metadata: ${cachePath}`,
+                    url: sourceUrl
+                }
+            };
+        }
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            return { writable: true };
+        }
+        throw error;
+    }
+    const content = await readFile(cachePath, "utf8");
+    try {
+        const entry = parseAssetCacheEntry(JSON.parse(content));
+        if (entry?.sourceUrl === sourceUrl) {
+            return { entry, writable: true };
+        }
+    }
+    catch {
+        // Report malformed cache metadata and recover with an unconditional fetch.
+    }
+    return {
+        writable: true,
+        warning: {
+            code: "ASSET_CACHE_INVALID",
+            message: `Invalid asset cache metadata: ${cachePath}`,
+            url: sourceUrl
+        }
+    };
+}
+async function existingCachedAsset(entry, root) {
+    const assetPath = path.join(root, "_assets", entry.assetFile);
+    try {
+        const metadata = await lstat(assetPath);
+        return metadata.isFile() && !metadata.isSymbolicLink()
+            ? assetPath
+            : undefined;
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            return undefined;
+        }
+        throw error;
+    }
+}
+function serializeAssetCache(entry) {
+    return `${JSON.stringify(entry, null, 2)}\n`;
+}
+function sameCacheEntry(left, right) {
+    if (!left || !right) {
+        return left === right;
+    }
+    return (left.sourceUrl === right.sourceUrl &&
+        left.finalUrl === right.finalUrl &&
+        left.assetFile === right.assetFile &&
+        left.contentType === right.contentType &&
+        left.etag === right.etag &&
+        left.lastModified === right.lastModified &&
+        left.vary.length === right.vary.length &&
+        left.vary.every((name, index) => name === right.vary[index]));
+}
+async function removeAssetCache(cachePath) {
+    try {
+        await unlink(cachePath);
+    }
+    catch (error) {
+        if (error.code !== "ENOENT") {
+            throw error;
+        }
+    }
+}
+async function processAsset(sourceUrl, options, representativeImage) {
+    let cacheWarning;
+    try {
+        const sameOrigin = new URL(sourceUrl).origin === new URL(options.baseUrl).origin;
+        const http = {
+            ...options.http,
+            ...(sameOrigin ? {} : { headers: [] })
+        };
+        const cachePath = cachePathForUrl(options.root, sourceUrl);
+        const cached = await withDestinationLock(cachePath, () => readAssetCache(cachePath, sourceUrl), options.root);
+        cacheWarning = cached.warning;
+        const cachedAsset = cached.entry
+            ? await existingCachedAsset(cached.entry, options.root)
+            : undefined;
+        const validatorsReusable = cached.entry !== undefined &&
+            cachedAsset !== undefined &&
+            cached.entry.finalUrl === sourceUrl &&
+            cached.entry.vary.length === 0 &&
+            !hasCredentialHeaders(http.headers) &&
+            !requestHasNoStore(http.headers);
+        const conditional = validatorsReusable
+            ? cached.entry?.etag
+                ? { etag: cached.entry.etag }
+                : cached.entry?.lastModified
+                    ? { lastModified: cached.entry.lastModified }
+                    : undefined
+            : undefined;
+        const response = await fetchResource(sourceUrl, {
+            ...http,
+            conditional: conditional ?? {},
+            ...(conditional ? { allowNotModified: true } : {})
+        });
+        if (response.notModified) {
+            const cachedEntry = cached.entry;
+            if (!cachedEntry || !cachedAsset || !conditional) {
+                throw new Error(`HTTP 304 without a matching cached asset: ${sourceUrl}`);
+            }
+            const vary = response.vary ? varyNames(response.vary) : cachedEntry.vary;
+            await withDestinationLock(cachePath, async () => {
+                const current = await readAssetCache(cachePath, sourceUrl);
+                cacheWarning ??= current.warning;
+                if (!sameCacheEntry(current.entry, cachedEntry)) {
+                    return;
+                }
+                if (vary.length === 0 && !hasNoStore(response.cacheControl)) {
+                    const refreshed = {
+                        ...cachedEntry,
+                        ...(response.etag ? { etag: response.etag } : {}),
+                        ...(response.lastModified
+                            ? { lastModified: response.lastModified }
+                            : {}),
+                        vary
+                    };
+                    await replaceFileAtomic(cachePath, serializeAssetCache(refreshed), {
+                        root: options.root
+                    });
+                }
+                else if (current.writable) {
+                    await removeAssetCache(cachePath);
+                }
+            }, options.root);
+            const replacement = path
+                .relative(path.dirname(options.markdownPath), cachedAsset)
+                .split(path.sep)
+                .join("/");
+            return {
+                asset: {
+                    sourceUrl,
+                    finalUrl: cachedEntry.finalUrl,
+                    path: cachedAsset,
+                    status: "reused"
+                },
+                replacement,
+                representative: sourceUrl === representativeImage,
+                ...(cacheWarning ? { warnings: [cacheWarning] } : {})
+            };
+        }
+        const urlExtension = path.posix.extname(new URL(response.finalUrl).pathname).toLowerCase();
+        if ((response.contentType && !response.contentType.startsWith("image/")) ||
+            (!response.contentType && !IMAGE_EXTENSIONS.has(urlExtension))) {
+            throw new Error(`Unsupported asset Content-Type: ${response.contentType || "(missing)"}`);
+        }
+        const digest = createHash("sha256").update(response.body).digest("hex");
+        const assetFile = `${digest}${assetExtension(response.contentType, response.finalUrl)}`;
+        const assetPath = path.join(options.root, "_assets", assetFile);
+        const status = await saveAsset(assetPath, response.body, options.root);
+        const vary = varyNames(response.vary);
+        const cacheable = !response.redirected &&
+            vary.length === 0 &&
+            !hasCredentialHeaders(http.headers) &&
+            !requestHasNoStore(http.headers) &&
+            !hasNoStore(response.cacheControl) &&
+            Boolean(response.etag || response.lastModified);
+        await withDestinationLock(cachePath, async () => {
+            const current = await readAssetCache(cachePath, sourceUrl);
+            cacheWarning ??= current.warning;
+            if (!sameCacheEntry(current.entry, cached.entry)) {
+                return;
+            }
+            if (cacheable && current.writable) {
+                const entry = {
+                    version: ASSET_CACHE_VERSION,
+                    sourceUrl,
+                    finalUrl: response.finalUrl,
+                    assetFile,
+                    contentType: response.contentType,
+                    ...(response.etag ? { etag: response.etag } : {}),
+                    ...(response.lastModified
+                        ? { lastModified: response.lastModified }
+                        : {}),
+                    vary
+                };
+                await replaceFileAtomic(cachePath, serializeAssetCache(entry), {
+                    root: options.root
+                });
+            }
+            else if (current.writable &&
+                !hasCredentialHeaders(http.headers) &&
+                !requestHasNoStore(http.headers)) {
+                await removeAssetCache(cachePath);
+            }
+        }, options.root);
+        const replacement = path
+            .relative(path.dirname(options.markdownPath), assetPath)
+            .split(path.sep)
+            .join("/");
+        return {
+            asset: {
+                sourceUrl,
+                finalUrl: response.finalUrl,
+                path: assetPath,
+                status
+            },
+            replacement,
+            representative: sourceUrl === representativeImage,
+            ...(cacheWarning ? { warnings: [cacheWarning] } : {})
+        };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+            asset: { sourceUrl, status: "failed", error: message },
+            representative: sourceUrl === representativeImage,
+            warnings: [
+                ...(cacheWarning ? [cacheWarning] : []),
+                { code: "ASSET_FETCH_FAILED", message, url: sourceUrl }
+            ]
+        };
+    }
+}
+export async function localizeAssets(options) {
+    const representativeCandidate = options.representativeImage
+        ? new URL(options.representativeImage, options.baseUrl)
+        : undefined;
+    const representativeImage = representativeCandidate?.protocol === "http:" ||
+        representativeCandidate?.protocol === "https:"
+        ? representativeCandidate.href
+        : undefined;
+    const urls = [...new Set([...options.imageUrls, ...(representativeImage ? [representativeImage] : [])])];
+    const replacements = new Map();
+    const assets = [];
+    let localRepresentative;
+    for (let offset = 0; offset < urls.length; offset += 6) {
+        const batch = await Promise.all(urls
+            .slice(offset, offset + 6)
+            .map((sourceUrl) => processAsset(sourceUrl, options, representativeImage)));
+        for (const result of batch) {
+            assets.push(result.asset);
+            for (const warning of result.warnings ?? []) {
+                options.warn(warning);
+            }
+            if (result.replacement) {
+                replacements.set(result.asset.sourceUrl, result.replacement);
+                if (result.representative) {
+                    localRepresentative = result.replacement;
+                }
+            }
+        }
+    }
+    const result = {
+        markdown: rewriteImageUrls(options.markdown, replacements),
+        assets
+    };
+    if (localRepresentative !== undefined) {
+        result.representativeImage = localRepresentative;
+    }
+    if (representativeImage !== undefined && localRepresentative !== undefined) {
+        result.representativeImageSource = representativeImage;
+    }
+    return result;
+}

package/dist/cli.d.ts

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+import { Command } from "commander";
+export interface CliIo {
+    stdout: Pick<NodeJS.WriteStream, "write">;
+    stderr: Pick<NodeJS.WriteStream, "write">;
+}
+export declare function createProgram(io?: CliIo): Command;
+export declare function runCli(argv?: string[], io?: CliIo): Promise<number>;

package/dist/cli.js

@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+import { Command, Option } from "commander";
+import { realpathSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { loadConfig, resolveRoot } from "./config/config.js";
+import { MdhqError } from "./errors.js";
+import { getPage } from "./get-page.js";
+import { listMarkdownFiles } from "./list-files.js";
+import { VERSION } from "./version.js";
+function collect(value, previous) {
+    return [...previous, value];
+}
+function parseHeaders(values) {
+    return values.map((value) => {
+        const separator = value.indexOf(":");
+        const name = value.slice(0, Math.max(separator, 0)).trim();
+        if (separator <= 0 ||
+            !/^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/u.test(name)) {
+            throw new MdhqError("INVALID_HEADER", `Invalid header: ${value}`);
+        }
+        return {
+            name,
+            value: value.slice(separator + 1).trim()
+        };
+    });
+}
+export function createProgram(io = process) {
+    const program = new Command()
+        .name("mdhq")
+        .version(VERSION)
+        .description("Save web pages as Markdown.");
+    program.configureOutput({
+        writeOut: (value) => {
+            io.stdout.write(value);
+        },
+        writeErr: (value) => {
+            io.stderr.write(value);
+        }
+    });
+    program
+        .command("get")
+        .description("Fetch and save one web page.")
+        .argument("<url>")
+        .option("--root <path>", "storage root")
+        .option("--no-assets", "do not download images")
+        .option("--update", "update an existing page")
+        .option("--user-agent <value>", "HTTP User-Agent")
+        .option("--header <header>", "additional HTTP header", collect, [])
+        .addOption(new Option("--json", "print a structured result"))
+        .action(async (url, options) => {
+        const result = await getPage({
+            url,
+            ...(options.root ? { root: options.root } : {}),
+            ...(options.assets === false ? { assets: false } : {}),
+            update: options.update ?? false,
+            ...(options.userAgent ? { userAgent: options.userAgent } : {}),
+            headers: parseHeaders(options.header),
+            onWarning: (warning) => io.stderr.write(`warning: ${warning.message}\n`)
+        });
+        io.stdout.write(options.json ? `${JSON.stringify(result, null, 2)}\n` : `${result.path}\n`);
+    });
+    program
+        .command("list")
+        .description("List saved Markdown files.")
+        .option("--root <path>", "storage root")
+        .option("-p, --full-path", "print full paths")
+        .action(async (options) => {
+        const files = await listMarkdownFiles({
+            ...(options.root ? { root: options.root } : {}),
+            fullPath: options.fullPath ?? false,
+            onWarning: (warning) => io.stderr.write(`warning: ${warning.message}\n`)
+        });
+        if (files.length > 0) {
+            io.stdout.write(`${files.join("\n")}\n`);
+        }
+    });
+    program
+        .command("root")
+        .description("Print the effective storage root.")
+        .option("--root <path>", "storage root")
+        .action(async (options) => {
+        const loaded = await loadConfig();
+        for (const warning of loaded.warnings) {
+            io.stderr.write(`warning: ${warning.message}\n`);
+        }
+        io.stdout.write(`${resolveRoot(options.root, loaded.config)}\n`);
+    });
+    return program;
+}
+export async function runCli(argv = process.argv, io = process) {
+    try {
+        await createProgram(io).parseAsync(argv);
+        return 0;
+    }
+    catch (error) {
+        const message = error instanceof MdhqError
+            ? `mdhq: ${error.message}`
+            : error instanceof Error
+                ? error.message
+                : String(error);
+        io.stderr.write(`${message}\n`);
+        return 1;
+    }
+}
+let isMain = false;
+if (process.argv[1] !== undefined) {
+    try {
+        isMain =
+            realpathSync(fileURLToPath(import.meta.url)) ===
+                realpathSync(path.resolve(process.argv[1]));
+    }
+    catch {
+        isMain = false;
+    }
+}
+if (isMain) {
+    process.exitCode = await runCli();
+}

package/dist/config/config.d.ts

@@ -0,0 +1,25 @@
+import type { DefuddleOptions } from "defuddle/node";
+import type { MdhqWarning } from "../types.js";
+import type { HostConfig } from "./match.js";
+export interface MdhqConfig {
+    root?: string;
+    userAgent?: string;
+    timeoutMs?: number;
+    maxResponseBytes?: number;
+    maxRedirects?: number;
+    assets?: boolean;
+    useAsync?: boolean;
+    defuddle?: Omit<DefuddleOptions, "fetch" | "markdown" | "separateMarkdown" | "url">;
+    frontmatter?: {
+        exclude?: string[];
+        values?: Record<string, string | number | boolean | null>;
+    };
+    hosts?: Record<string, HostConfig>;
+}
+export declare function defaultConfigPath(env?: NodeJS.ProcessEnv): string;
+export declare function defaultDataRoot(env?: NodeJS.ProcessEnv): string;
+export declare function resolveRoot(cliRoot: string | undefined, config: MdhqConfig, env?: NodeJS.ProcessEnv): string;
+export declare function loadConfig(configPath?: string): Promise<{
+    config: MdhqConfig;
+    warnings: MdhqWarning[];
+}>;