@songmu/mdhq 0.0.2 → 0.0.3

package/README.md

@@ -1,4 +1,4 @@
-# mdhq - Markdown headquarters (?)
+# mdhq - Markdown headquarters, probably
 
 `mdhq` saves web pages as Markdown in a [ghq](https://github.com/x-motemen/ghq)-inspired filesystem layout. It
 uses [Defuddle](https://defuddle.md/) for content extraction and keeps all state in Markdown and asset
@@ -18,6 +18,8 @@ npm install --global @songmu/mdhq
 
 ```sh
 mdhq get https://example.com/article
+mdhq get https://example.com/article https://example.com/another-article
+cat urls.txt | mdhq get
 mdhq get --update https://example.com/article
 mdhq get --no-assets https://example.com/article
 mdhq get --json --header 'Cookie: session=value' https://example.com/article
@@ -26,9 +28,12 @@ mdhq list --full-path
 mdhq root
 ```
 
-`mdhq get` prints the absolute Markdown path to stdout by default. Warnings
-are written to stderr. `--json` returns the requested URL, final source URL,
-Markdown path, status, downloaded assets, and warnings.
+`mdhq get` accepts multiple URLs as arguments or one URL per line on standard
+input; both sources are merged. It prints one absolute Markdown path per URL
+to stdout by default. Warnings are written to stderr. `--json` returns one
+result object for a single URL, or an array for multiple URLs. Requests are
+limited to eight in parallel, with requests to the same host serialized and
+spaced one second apart; image downloads use the same limits.
 
 `mdhq list` recursively lists `.md` files below the storage root, one per
 line, in sorted root-relative form. Use `-p` or `--full-path` to print absolute
@@ -103,6 +108,14 @@ Saved frontmatter uses Obsidian Web Clipper-compatible names such as `title`,
 conditional updates. It does not add `type` or `tags` by default; use
 `frontmatter.values` to opt into values such as `"type": "clip"`.
 
+`source` is normalized after redirects. mdhq accepts an HTML canonical URL
+only when its normalized origin and pathname match the fetched page; otherwise
+it removes known tracking parameters with `urlpurify`, while retaining
+functional WordPress preview parameters. Fragments are removed.
+Queryless sources use the normal ghq-inspired path. Sources with a query use a
+configured `entryQueryKey` value when available, or a deterministic MD5
+filename derived from the final path segment and ordered query string.
+
 An update returns `updated` when the normalized Markdown body or user-facing
 frontmatter changes and `unchanged` when HTTP returns 304 or the fetched note
 content is unchanged.

package/dist/cli.d.ts

@@ -3,6 +3,7 @@ import { Command } from "commander";
 export interface CliIo {
     stdout: Pick<NodeJS.WriteStream, "write">;
     stderr: Pick<NodeJS.WriteStream, "write">;
+    stdin?: NodeJS.ReadStream;
 }
 export declare function createProgram(io?: CliIo): Command;
 export declare function runCli(argv?: string[], io?: CliIo): Promise<number>;

package/dist/cli.js

@@ -8,6 +8,7 @@ import { MdhqError } from "./errors.js";
 import { getPage } from "./get-page.js";
 import { listMarkdownFiles } from "./list-files.js";
 import { VERSION } from "./version.js";
+import { RequestScheduler } from "./http/scheduler.js";
 function collect(value, previous) {
     return [...previous, value];
 }
@@ -25,6 +26,27 @@ function parseHeaders(values) {
         };
     });
 }
+async function readStdinUrls(stdin) {
+    if (!stdin || stdin.isTTY) {
+        return [];
+    }
+    const urls = [];
+    let remainder = "";
+    const decoder = new TextDecoder();
+    for await (const chunk of stdin) {
+        const text = typeof chunk === "string"
+            ? chunk
+            : decoder.decode(chunk, { stream: true });
+        const lines = `${remainder}${text}`.split(/\r?\n/u);
+        remainder = lines.pop() ?? "";
+        urls.push(...lines.map((line) => line.trim()).filter(Boolean));
+    }
+    remainder += decoder.decode();
+    if (remainder.trim()) {
+        urls.push(remainder.trim());
+    }
+    return urls;
+}
 export function createProgram(io = process) {
     const program = new Command()
         .name("mdhq")
@@ -40,25 +62,49 @@ export function createProgram(io = process) {
     });
     program
         .command("get")
-        .description("Fetch and save one web page.")
-        .argument("<url>")
+        .description("Fetch and save web pages.")
+        .argument("[urls...]")
         .option("--root <path>", "storage root")
         .option("--no-assets", "do not download images")
         .option("--update", "update an existing page")
         .option("--user-agent <value>", "HTTP User-Agent")
         .option("--header <header>", "additional HTTP header", collect, [])
         .addOption(new Option("--json", "print a structured result"))
-        .action(async (url, options) => {
-        const result = await getPage({
-            url,
-            ...(options.root ? { root: options.root } : {}),
-            ...(options.assets === false ? { assets: false } : {}),
-            update: options.update ?? false,
-            ...(options.userAgent ? { userAgent: options.userAgent } : {}),
-            headers: parseHeaders(options.header),
-            onWarning: (warning) => io.stderr.write(`warning: ${warning.message}\n`)
-        });
-        io.stdout.write(options.json ? `${JSON.stringify(result, null, 2)}\n` : `${result.path}\n`);
+        .action(async (urls, options) => {
+        const inputUrls = await readStdinUrls(io.stdin);
+        const requestedUrls = [...(urls ?? []), ...inputUrls]
+            .map((url) => url.trim())
+            .filter(Boolean);
+        if (requestedUrls.length === 0) {
+            throw new MdhqError("INVALID_URL", "At least one URL is required");
+        }
+        const scheduler = new RequestScheduler();
+        const results = [];
+        let nextIndex = 0;
+        const worker = async () => {
+            while (nextIndex < requestedUrls.length) {
+                const index = nextIndex;
+                nextIndex += 1;
+                const url = requestedUrls[index];
+                if (url === undefined) {
+                    continue;
+                }
+                results[index] = await getPage({
+                    url,
+                    ...(options.root ? { root: options.root } : {}),
+                    ...(options.assets === false ? { assets: false } : {}),
+                    update: options.update ?? false,
+                    ...(options.userAgent ? { userAgent: options.userAgent } : {}),
+                    headers: parseHeaders(options.header),
+                    scheduler,
+                    onWarning: (warning) => io.stderr.write(`warning: ${warning.message}\n`)
+                });
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(8, requestedUrls.length) }, () => worker()));
+        io.stdout.write(options.json
+            ? `${JSON.stringify(results.length === 1 ? results[0] : results, null, 2)}\n`
+            : `${results.map((result) => result.path).join("\n")}\n`);
     });
     program
         .command("list")
@@ -115,5 +161,5 @@ if (process.argv[1] !== undefined) {
     }
 }
 if (isMain) {
-    process.exitCode = await runCli();
+    process.exitCode = await runCli(process.argv, process);
 }

package/dist/frontmatter/frontmatter.d.ts

@@ -5,13 +5,9 @@ export interface FrontmatterOptions {
     sourceUrl: string;
     requestedUrl: string;
     created: Date | string;
-    modified: Date;
-    contentDigest: string;
+    modified: Date | string;
     etag?: string;
     lastModified?: string;
-    vary?: string[];
-    image?: string;
-    imageSource?: string;
     config?: MdhqConfig["frontmatter"];
 }
 interface ControlledFrontmatterOptions {
@@ -19,11 +15,9 @@ interface ControlledFrontmatterOptions {
     sourceUrl: string;
     requestedUrl: string;
     created: Date | string;
-    modified: Date;
-    contentDigest: string;
+    modified: Date | string;
     etag?: string;
     lastModified?: string;
-    vary?: string[];
     config?: MdhqConfig["frontmatter"];
 }
 export declare function buildFrontmatter(options: FrontmatterOptions): Record<string, unknown>;

package/dist/frontmatter/frontmatter.js

@@ -1,9 +1,13 @@
 import { createHash } from "node:crypto";
 import { parse, stringify } from "yaml";
 import { formatLocalRfc3339 } from "../date.js";
+const REMOVED_DEFAULT_FIELDS = ["site", "domain", "image", "image_source", "word_count"];
 function applyControlledFields(options) {
     const fields = { ...options.fields };
     delete fields.type;
+    for (const key of REMOVED_DEFAULT_FIELDS) {
+        delete fields[key];
+    }
     for (const key of options.config?.exclude ?? []) {
         delete fields[key];
     }
@@ -21,8 +25,11 @@ function applyControlledFields(options) {
         typeof options.created === "string"
             ? options.created
             : formatLocalRfc3339(options.created);
-    fields.modified = formatLocalRfc3339(options.modified);
-    fields.content_digest = options.contentDigest;
+    fields.modified =
+        typeof options.modified === "string"
+            ? options.modified
+            : formatLocalRfc3339(options.modified);
+    delete fields.content_digest;
     if (options.etag) {
         fields.etag = options.etag;
     }
@@ -35,12 +42,7 @@ function applyControlledFields(options) {
     else {
         delete fields.last_modified;
     }
-    if (options.vary) {
-        fields.vary = options.vary;
-    }
-    else {
-        delete fields.vary;
-    }
+    delete fields.vary;
     return fields;
 }
 export function buildFrontmatter(options) {
@@ -51,25 +53,13 @@ export function buildFrontmatter(options) {
         ["author", options.metadata.author],
         ["published", options.metadata.published],
         ["updated", options.metadata.updated],
-        ["site", options.metadata.site],
-        ["domain", options.metadata.domain],
-        ["language", options.metadata.language],
-        ["word_count", options.metadata.wordCount]
+        ["language", options.metadata.language]
     ];
     for (const [key, value] of metadataFields) {
         if (value !== undefined && value !== "") {
             fields[key] = value;
         }
     }
-    if (options.image) {
-        fields.image = options.image;
-    }
-    else if (options.metadata.image) {
-        fields.image = options.metadata.image;
-    }
-    if (options.imageSource) {
-        fields.image_source = options.imageSource;
-    }
     return applyControlledFields({ ...options, fields });
 }
 export function refreshFrontmatter(existing, options) {

package/dist/get-page.js

@@ -9,7 +9,8 @@ import { fetchHtml, fetchWithEnvProxy } from "./http/fetch.js";
 import { transformMarkdown } from "./markdown/transform.js";
 import { storagePathForUrl } from "./path/storage-path.js";
 import { inspectDestination, saveDocument } from "./storage/save.js";
-import { normalizeHost, parseHttpUrl, sameHttpTarget } from "./url/identity.js";
+import { normalizeHost, sameHttpTarget } from "./url/identity.js";
+import { normalizeRequestedUrl, normalizeSourceUrl, normalizeSourceUrlWithoutCanonical } from "./url/normalize.js";
 function varyNames(value) {
     return [
         ...new Set((value ?? "")
@@ -24,8 +25,37 @@ function hasCredentialHeaders(headers) {
         return name === "authorization" || name === "cookie";
     });
 }
+const NOTE_BOOKKEEPING_FIELDS = new Set([
+    "created",
+    "modified",
+    "etag",
+    "last_modified",
+    "content_digest",
+    "vary"
+]);
+function normalizeComparable(value) {
+    if (Array.isArray(value)) {
+        return value.map(normalizeComparable);
+    }
+    if (value && typeof value === "object") {
+        return Object.fromEntries(Object.entries(value)
+            .sort(([left], [right]) => left.localeCompare(right))
+            .map(([key, entry]) => [key, normalizeComparable(entry)]));
+    }
+    return value;
+}
+function userFacingFrontmatter(frontmatter) {
+    return Object.fromEntries(Object.entries(frontmatter)
+        .filter(([key]) => !NOTE_BOOKKEEPING_FIELDS.has(key))
+        .map(([key, value]) => [key, normalizeComparable(value)])
+        .sort(([left], [right]) => left.localeCompare(right)));
+}
+function sameUserFacingFrontmatter(left, right) {
+    return (JSON.stringify(userFacingFrontmatter(left)) ===
+        JSON.stringify(userFacingFrontmatter(right)));
+}
 export async function getPage(options) {
-    const requestedUrl = parseHttpUrl(options.url).href;
+    const requestedUrl = normalizeRequestedUrl(options.url);
     const loaded = await loadConfig(options.configPath);
     const warnings = [];
     const warn = (warning) => {
@@ -40,7 +70,8 @@ export async function getPage(options) {
 }
 async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeadersAllowed) {
     const { options, requestedUrl, loaded, warnings, warn, root } = context;
-    const requested = new URL(fetchUrl);
+    const requestedCandidateUrl = normalizeSourceUrlWithoutCanonical(fetchUrl);
+    const requested = new URL(requestedCandidateUrl);
     const requestedConfig = resolveHostConfig(normalizeHost(requested), requested.pathname, loaded.config.hosts ?? {});
     const requestedEntryKey = requestedConfig?.entryQueryKey ?? undefined;
     const requestedPath = storagePathForUrl({
@@ -48,7 +79,7 @@ async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeaders
         url: requested,
         ...(requestedEntryKey ? { entryQueryKey: requestedEntryKey } : {})
     });
-    const requestedExisting = await inspectDestination(requestedPath, fetchUrl, requestedEntryKey, root);
+    const requestedExisting = await inspectDestination(requestedPath, requestedCandidateUrl, requestedEntryKey, root);
     if (requestedExisting && !options.update) {
         return {
             requestedUrl,
@@ -59,6 +90,11 @@ async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeaders
             warnings
         };
     }
+    if (options.update &&
+        requestedExisting &&
+        !sameHttpTarget(requestedCandidateUrl, fetchUrl)) {
+        return getPageAttempt(context, requestedCandidateUrl, retriesRemaining, callerHeadersAllowed);
+    }
     const headers = callerHeadersAllowed ? options.headers : [];
     const userAgent = options.userAgent ?? loaded.config.userAgent;
     const timeoutMs = options.timeoutMs ?? loaded.config.timeoutMs;
@@ -69,12 +105,12 @@ async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeaders
         ...(userAgent ? { userAgent } : {}),
         ...(timeoutMs !== undefined ? { timeoutMs } : {}),
         ...(maxResponseBytes !== undefined ? { maxResponseBytes } : {}),
-        ...(maxRedirects !== undefined ? { maxRedirects } : {})
+        ...(maxRedirects !== undefined ? { maxRedirects } : {}),
+        ...(options.scheduler ? { scheduler: options.scheduler } : {})
     };
     let conditional;
     if (options.update &&
         requestedExisting &&
-        requestedExisting.vary?.length === 0 &&
         !hasCredentialHeaders(headers) &&
         sameHttpTarget(requestedExisting.sourceUrl, fetchUrl)) {
         if (requestedExisting.etag) {
@@ -126,17 +162,11 @@ async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeaders
             created: isRfc3339DateTime(requestedExisting.created)
                 ? requestedExisting.created
                 : now,
-            modified: now,
-            contentDigest: requestedExisting.contentDigest,
+            modified: requestedExisting.modified ?? now,
             ...(etag ? { etag } : {}),
             ...(reusableLastModified
                 ? { lastModified: reusableLastModified }
                 : {}),
-            ...(vary.length > 0
-                ? { vary }
-                : etag || reusableLastModified
-                    ? { vary: [] }
-                    : {}),
             ...(loaded.config.frontmatter ? { config: loaded.config.frontmatter } : {})
         });
         const content = serializeDocument(frontmatter, requestedExisting.markdown);
@@ -159,32 +189,34 @@ async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeaders
             requestedUrl,
             sourceUrl: requestedExisting.sourceUrl,
             path: requestedPath,
-            status: storageStatus === "updated" ? "unchanged" : storageStatus,
+            status: storageStatus === "saved" ? "saved" : "unchanged",
             assets: [],
             warnings
         };
     }
-    const finalUrl = new URL(fetched.finalUrl);
-    const matchedConfig = resolveHostConfig(normalizeHost(finalUrl), finalUrl.pathname, loaded.config.hosts ?? {});
+    const finalResponseUrl = new URL(fetched.finalUrl);
+    const sourceUrl = normalizeSourceUrl(fetched.html, finalResponseUrl);
+    const normalizedSource = new URL(sourceUrl);
+    const matchedConfig = resolveHostConfig(normalizeHost(normalizedSource), normalizedSource.pathname, loaded.config.hosts ?? {});
     const entryQueryKey = matchedConfig?.entryQueryKey ?? undefined;
     const markdownPath = storagePathForUrl({
         root,
-        url: finalUrl,
+        url: normalizedSource,
         ...(entryQueryKey ? { entryQueryKey } : {})
     });
-    const existing = await inspectDestination(markdownPath, finalUrl.href, entryQueryKey, root);
+    const existing = await inspectDestination(markdownPath, sourceUrl, entryQueryKey, root);
     if (options.update &&
         existing &&
         markdownPath !== requestedPath) {
         if (retriesRemaining === 0) {
             throw new MdhqError("STORAGE_ERROR", `Redirect destination changed repeatedly while updating ${markdownPath}`);
         }
-        return getPageAttempt(context, finalUrl.href, retriesRemaining - 1, responseHeadersAllowed);
+        return getPageAttempt(context, sourceUrl, retriesRemaining - 1, responseHeadersAllowed);
     }
     if (existing && !options.update) {
         return {
             requestedUrl,
-            sourceUrl: finalUrl.href,
+            sourceUrl,
             path: markdownPath,
             status: "skipped",
             assets: [],
@@ -195,7 +227,7 @@ async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeaders
     const responseCredentialed = responseHeadersAllowed && hasCredentialHeaders(http.headers);
     const converted = await convertHtml({
         html: fetched.html,
-        url: finalUrl,
+        url: finalResponseUrl,
         defuddle: {
             ...loaded.config.defuddle,
             fetch: fetchWithEnvProxy,
@@ -208,7 +240,7 @@ async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeaders
     let metadata = converted.metadata;
     if (converted.metadata.image) {
         try {
-            const image = new URL(converted.metadata.image, finalUrl);
+            const image = new URL(converted.metadata.image, finalResponseUrl);
             if (image.protocol !== "http:" && image.protocol !== "https:") {
                 throw new TypeError(`Unsupported image URL scheme: ${image.protocol}`);
             }
@@ -223,11 +255,11 @@ async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeaders
             warn({
                 code: "INVALID_IMAGE_URL",
                 message: `Invalid representative image URL: ${converted.metadata.image}`,
-                url: finalUrl.href
+                url: finalResponseUrl.href
             });
         }
     }
-    const transformed = transformMarkdown(converted.markdown, finalUrl.href);
+    const transformed = transformMarkdown(converted.markdown, finalResponseUrl.href);
     const assetsEnabled = options.assets ?? loaded.config.assets ?? true;
     const localized = assetsEnabled
         ? await localizeAssets({
@@ -236,7 +268,7 @@ async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeaders
             ...(metadata.image ? { representativeImage: metadata.image } : {}),
             markdownPath,
             root,
-            baseUrl: finalUrl.href,
+            baseUrl: finalResponseUrl.href,
             http: responseHeadersAllowed ? http : { ...http, headers: [] },
             warn
         })
@@ -251,36 +283,36 @@ async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeaders
     const contentDigest = markdownContentDigest(localized.markdown);
     const lastModified = normalizeLastModified(fetched.lastModified);
     const vary = varyNames(fetched.vary);
-    const validatorsReusable = vary.length === 0 && !responseCredentialed;
+    const validatorsReusable = vary.length === 0 &&
+        !responseCredentialed &&
+        sameHttpTarget(sourceUrl, finalResponseUrl);
     const etag = validatorsReusable ? fetched.etag : undefined;
     const reusableLastModified = validatorsReusable ? lastModified : undefined;
-    const frontmatter = buildFrontmatter({
+    const nextFrontmatterOptions = {
         metadata,
-        sourceUrl: finalUrl.href,
+        sourceUrl,
         requestedUrl,
         created,
         modified: now,
-        contentDigest,
         ...(etag ? { etag } : {}),
         ...(reusableLastModified ? { lastModified: reusableLastModified } : {}),
-        ...(vary.length > 0
-            ? { vary }
-            : etag || reusableLastModified
-                ? { vary: [] }
-                : {}),
-        ...(localized.representativeImage
-            ? { image: localized.representativeImage }
-            : {}),
-        ...(localized.representativeImageSource
-            ? { imageSource: localized.representativeImageSource }
-            : {}),
         ...(loaded.config.frontmatter ? { config: loaded.config.frontmatter } : {})
-    });
+    };
+    const nextFrontmatter = buildFrontmatter(nextFrontmatterOptions);
+    const noteChanged = !expectedExisting ||
+        expectedExisting.contentDigest !== contentDigest ||
+        !sameUserFacingFrontmatter(expectedExisting.frontmatter, nextFrontmatter);
+    const frontmatter = noteChanged
+        ? nextFrontmatter
+        : buildFrontmatter({
+            ...nextFrontmatterOptions,
+            modified: expectedExisting.modified ?? now
+        });
     const content = serializeDocument(frontmatter, localized.markdown);
     const storageStatus = await saveDocument({
         path: markdownPath,
         content,
-        sourceUrl: finalUrl.href,
+        sourceUrl,
         update: options.update ?? false,
         ...(options.update
             ? { expectedContent: expectedExisting?.content ?? null }
@@ -296,10 +328,11 @@ async function getPageAttempt(context, fetchUrl, retriesRemaining, callerHeaders
     }
     return {
         requestedUrl,
-        sourceUrl: finalUrl.href,
+        sourceUrl,
         path: markdownPath,
-        status: storageStatus === "updated" &&
-            expectedExisting?.contentDigest === contentDigest
+        status: expectedExisting &&
+            (storageStatus === "updated" || storageStatus === "skipped") &&
+            !noteChanged
             ? "unchanged"
             : storageStatus,
         assets: localized.assets,

package/dist/http/fetch.d.ts

@@ -1,4 +1,5 @@
 import type { HeaderValue } from "../types.js";
+import type { RequestScheduler } from "./scheduler.js";
 export interface FetchResourceOptions {
     headers?: HeaderValue[];
     userAgent?: string;
@@ -11,6 +12,7 @@ export interface FetchResourceOptions {
         etag?: string;
         lastModified?: string;
     };
+    scheduler?: RequestScheduler;
 }
 export interface FetchedResource {
     body: Uint8Array;

package/dist/http/fetch.js

@@ -73,12 +73,13 @@ export async function fetchResource(input, options = {}) {
     for (let redirects = 0;; redirects += 1) {
         let response;
         try {
-            response = (await undiciFetch(url, {
+            const fetch = () => undiciFetch(url, {
                 dispatcher: proxyAgent,
                 headers: requestHeaders(options, customHeadersAllowed, redirects === 0),
                 redirect: "manual",
                 signal: AbortSignal.timeout(timeoutMs)
-            }));
+            });
+            response = options.scheduler ? await options.scheduler.run(url, fetch) : await fetch();
         }
         catch (error) {
             throw new MdhqError("FETCH_FAILED", `Failed to fetch ${url.href}`, { cause: error });

package/dist/http/scheduler.d.ts

@@ -0,0 +1,12 @@
+export declare class RequestScheduler {
+    private readonly maxConcurrent;
+    private readonly hostIntervalMs;
+    private active;
+    private readonly pending;
+    private readonly activeHosts;
+    private readonly lastStarted;
+    private timer;
+    constructor(maxConcurrent?: number, hostIntervalMs?: number);
+    run<T>(url: string | URL, task: () => Promise<T>): Promise<T>;
+    private pump;
+}

package/dist/http/scheduler.js

@@ -0,0 +1,69 @@
+export class RequestScheduler {
+    maxConcurrent;
+    hostIntervalMs;
+    active = 0;
+    pending = [];
+    activeHosts = new Set();
+    lastStarted = new Map();
+    timer;
+    constructor(maxConcurrent = 8, hostIntervalMs = 1_000) {
+        this.maxConcurrent = maxConcurrent;
+        this.hostIntervalMs = hostIntervalMs;
+    }
+    async run(url, task) {
+        const host = new URL(url).hostname.toLowerCase();
+        await new Promise((resolve) => {
+            this.pending.push({ host, run: resolve });
+            this.pump();
+        });
+        try {
+            return await task();
+        }
+        finally {
+            this.active -= 1;
+            this.activeHosts.delete(host);
+            this.pump();
+        }
+    }
+    pump() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = undefined;
+        }
+        if (this.active >= this.maxConcurrent) {
+            return;
+        }
+        const now = Date.now();
+        let waitMs;
+        const index = this.pending.findIndex(({ host }) => {
+            if (this.activeHosts.has(host)) {
+                return false;
+            }
+            const remaining = (this.lastStarted.get(host) ?? 0) + this.hostIntervalMs - now;
+            if (remaining > 0) {
+                waitMs = waitMs === undefined ? remaining : Math.min(waitMs, remaining);
+                return false;
+            }
+            return true;
+        });
+        if (index < 0) {
+            if (waitMs !== undefined) {
+                this.timer = setTimeout(() => {
+                    this.timer = undefined;
+                    this.pump();
+                }, waitMs);
+            }
+            return;
+        }
+        const item = this.pending.splice(index, 1)[0];
+        if (!item) {
+            return;
+        }
+        const { host, run } = item;
+        this.active += 1;
+        this.activeHosts.add(host);
+        this.lastStarted.set(host, Date.now());
+        run();
+        this.pump();
+    }
+}

package/dist/path/storage-path.js

@@ -1,7 +1,7 @@
 import { createHash } from "node:crypto";
 import path from "node:path";
 import { MdhqError } from "../errors.js";
-import { createUrlIdentity, parseHttpUrl } from "../url/identity.js";
+import { createUrlIdentity, parseHttpUrl, queryTailHash } from "../url/identity.js";
 import { decodeUrlPathSegment, storageBasename } from "../url/pathname.js";
 const INVALID_CHARACTERS = /[\/\\:*?"<>|\u0000-\u001f\u007f]/u;
 const WINDOWS_DEVICES = /^(con|prn|aux|nul|com(?:[1-9]|[¹²³])|lpt(?:[1-9]|[¹²³]))(?:\.|$)/iu;
@@ -66,13 +66,32 @@ export function storagePathForUrl(options) {
     const url = parseHttpUrl(options.url);
     const identity = createUrlIdentity(url, options.entryQueryKey);
     const rawSegments = url.pathname.split("/").filter(Boolean);
-    const directories = rawSegments.slice(0, -1).map((segment) => fitSegment(segment));
+    const queryHash = identity.entryValue === undefined ? queryTailHash(url) : undefined;
+    const directorySegments = (queryHash && url.pathname.endsWith("/")
+        ? rawSegments
+        : rawSegments.slice(0, -1));
+    if (queryHash && url.pathname.endsWith("/")) {
+        if (directorySegments.length === 0) {
+            directorySegments.push("index");
+        }
+        else {
+            const finalIndex = directorySegments.length - 1;
+            const finalSegment = directorySegments[finalIndex];
+            if (finalSegment !== undefined) {
+                directorySegments[finalIndex] = encodeURIComponent(storageBasename(finalSegment));
+            }
+        }
+    }
+    const directories = directorySegments.map((segment) => fitSegment(segment));
     let filename;
     if (identity.entryValue !== undefined) {
         const pageSegment = rawSegments.at(-1) ?? "index";
         directories.push(fitSegment(pageSegment));
         filename = `${fitSegment(encodeURIComponent(identity.entryValue), ".md")}.md`;
     }
+    else if (queryHash) {
+        filename = `${queryHash}.md`;
+    }
     else if (rawSegments.length === 0) {
         filename = "index.md";
     }

package/dist/storage/save.d.ts

@@ -14,9 +14,9 @@ export interface ExistingDocument {
     markdown: string;
     contentDigest: string;
     created?: string;
+    modified?: string;
     etag?: string;
     lastModified?: string;
-    vary?: string[];
 }
 export declare function readExistingDocument(filePath: string): Promise<ExistingDocument | undefined>;
 export declare function inspectDestination(filePath: string, sourceUrl: string, entryQueryKey?: string, root?: string): Promise<ExistingDocument | undefined>;

package/dist/storage/save.js

@@ -29,13 +29,12 @@ export async function readExistingDocument(filePath) {
         markdown: parsed.markdown,
         contentDigest: markdownContentDigest(parsed.markdown),
         ...(typeof frontmatter.created === "string" ? { created: frontmatter.created } : {}),
+        ...(typeof frontmatter.modified === "string"
+            ? { modified: frontmatter.modified }
+            : {}),
         ...(typeof frontmatter.etag === "string" ? { etag: frontmatter.etag } : {}),
         ...(typeof frontmatter.last_modified === "string"
             ? { lastModified: frontmatter.last_modified }
-            : {}),
-        ...(Array.isArray(frontmatter.vary) &&
-            frontmatter.vary.every((value) => typeof value === "string")
-            ? { vary: frontmatter.vary }
             : {})
     };
 }
@@ -81,6 +80,9 @@ export async function saveDocument(options) {
                 if (!options.update) {
                     return "skipped";
                 }
+                if (existing.content === options.content) {
+                    return "skipped";
+                }
             }
             if (existing) {
                 await replaceFileAtomic(options.path, options.content, {

package/dist/types.d.ts

@@ -1,4 +1,5 @@
 import type { DefuddleOptions } from "defuddle/node";
+import type { RequestScheduler } from "./http/scheduler.js";
 export interface MdhqWarning {
     code: string;
     message: string;
@@ -44,6 +45,7 @@ export interface GetPageOptions {
     useAsync?: boolean;
     now?: () => Date;
     onWarning?: (warning: MdhqWarning) => void;
+    scheduler?: RequestScheduler;
 }
 export interface AssetResult {
     sourceUrl: string;

package/dist/url/identity.d.ts

@@ -3,7 +3,10 @@ export interface UrlIdentity {
     pathname: string;
     entryKey?: string;
     entryValue?: string;
+    queryHash?: string;
 }
+export declare function queryTail(url: URL): string;
+export declare function queryTailHash(url: URL): string | undefined;
 export declare function parseHttpUrl(input: string | URL): URL;
 export declare function normalizeHost(url: URL): string;
 export declare function createUrlIdentity(input: string | URL, entryQueryKey?: string): UrlIdentity;

package/dist/url/identity.js

@@ -1,6 +1,22 @@
+import { createHash } from "node:crypto";
 import { domainToASCII } from "node:url";
 import { MdhqError } from "../errors.js";
 import { canonicalPathname } from "./pathname.js";
+export function queryTail(url) {
+    if (!url.search) {
+        return "";
+    }
+    if (url.pathname.endsWith("/")) {
+        return url.search;
+    }
+    return `${url.pathname.split("/").at(-1) ?? ""}${url.search}`;
+}
+export function queryTailHash(url) {
+    const tail = queryTail(url);
+    return tail
+        ? createHash("md5").update(tail, "utf8").digest("hex")
+        : undefined;
+}
 export function parseHttpUrl(input) {
     let url;
     try {
@@ -23,8 +39,10 @@ export function normalizeHost(url) {
 }
 export function createUrlIdentity(input, entryQueryKey) {
     const url = parseHttpUrl(input);
-    const entryValue = entryQueryKey ? url.searchParams.get(entryQueryKey) : null;
-    const hasEntryValue = entryValue !== null && entryValue !== "";
+    const entryValue = entryQueryKey
+        ? url.searchParams.getAll(entryQueryKey).find((value) => value !== "")
+        : undefined;
+    const hasEntryValue = entryValue !== undefined;
     const identity = {
         host: normalizeHost(url),
         pathname: canonicalPathname(url.pathname || "/", hasEntryValue)
@@ -33,12 +51,20 @@ export function createUrlIdentity(input, entryQueryKey) {
         identity.entryKey = entryQueryKey;
         identity.entryValue = entryValue.normalize("NFC");
     }
+    else {
+        const hash = queryTailHash(url);
+        if (hash) {
+            identity.queryHash = hash;
+        }
+    }
     return identity;
 }
 export function serializeUrlIdentity(identity) {
     const entry = identity.entryKey && identity.entryValue
-        ? `?${encodeURIComponent(identity.entryKey)}=${encodeURIComponent(identity.entryValue)}`
-        : "";
+        ? `?entry-key=${encodeURIComponent(identity.entryKey)}&entry-value=${encodeURIComponent(identity.entryValue)}`
+        : identity.queryHash
+            ? `?query-md5=${identity.queryHash}`
+            : "";
     return `//${identity.host}${identity.pathname}${entry}`;
 }
 export function sameUrlIdentity(left, right, entryQueryKey) {

package/dist/url/normalize.d.ts

@@ -0,0 +1,3 @@
+export declare function normalizeRequestedUrl(input: string | URL): string;
+export declare function normalizeSourceUrlWithoutCanonical(input: string | URL): string;
+export declare function normalizeSourceUrl(html: string, finalUrlInput: string | URL): string;

package/dist/url/normalize.js

@@ -0,0 +1,83 @@
+import { parseHTML } from "linkedom";
+import { defaultTrackingParams, stripTrackingParams } from "urlpurify";
+import { canonicalPathname } from "./pathname.js";
+import { normalizeHost, parseHttpUrl } from "./identity.js";
+const FUNCTIONAL_PAGE_PARAMS = new Set([
+    "preview",
+    "preview_id",
+    "preview_nonce"
+]);
+const PAGE_TRACKING_PARAMS = defaultTrackingParams.filter((parameter) => typeof parameter !== "string" ||
+    !FUNCTIONAL_PAGE_PARAMS.has(parameter.toLowerCase()));
+function withoutFragment(url) {
+    const normalized = new URL(url.href);
+    normalized.hash = "";
+    return normalized;
+}
+function normalizedOrigin(url) {
+    return `${url.protocol}//${normalizeHost(url)}`;
+}
+function canonicalUrl(html, finalUrl) {
+    let document;
+    try {
+        ({ document } = parseHTML(html));
+    }
+    catch {
+        return undefined;
+    }
+    const canonicalLinks = [...document.querySelectorAll("link[href]")].filter((link) => (link.getAttribute("rel") ?? "")
+        .split(/\s+/u)
+        .some((token) => token.toLowerCase() === "canonical"));
+    if (canonicalLinks.length !== 1) {
+        return undefined;
+    }
+    let baseUrl = finalUrl;
+    const baseHref = document.querySelector("base[href]")?.getAttribute("href");
+    if (baseHref) {
+        try {
+            baseUrl = parseHttpUrl(new URL(baseHref, finalUrl));
+        }
+        catch {
+            baseUrl = finalUrl;
+        }
+    }
+    const href = canonicalLinks[0]?.getAttribute("href");
+    if (!href) {
+        return undefined;
+    }
+    try {
+        return parseHttpUrl(new URL(href, baseUrl));
+    }
+    catch {
+        return undefined;
+    }
+}
+function isCanonicalEquivalent(finalUrl, canonical) {
+    return (canonical.username === "" &&
+        canonical.password === "" &&
+        normalizedOrigin(finalUrl) === normalizedOrigin(canonical) &&
+        canonicalPathname(finalUrl.pathname || "/") ===
+            canonicalPathname(canonical.pathname || "/"));
+}
+export function normalizeRequestedUrl(input) {
+    return withoutFragment(parseHttpUrl(input)).href;
+}
+export function normalizeSourceUrlWithoutCanonical(input) {
+    const url = withoutFragment(parseHttpUrl(input));
+    return withoutFragment(parseHttpUrl(stripTrackingParams(url.href, PAGE_TRACKING_PARAMS))).href;
+}
+export function normalizeSourceUrl(html, finalUrlInput) {
+    const finalUrl = withoutFragment(parseHttpUrl(finalUrlInput));
+    const canonical = canonicalUrl(html, finalUrl);
+    if (canonical) {
+        try {
+            if (isCanonicalEquivalent(finalUrl, canonical)) {
+                return withoutFragment(canonical).href;
+            }
+        }
+        catch {
+            // Ignore malformed optional canonical metadata.
+        }
+    }
+    return normalizeSourceUrlWithoutCanonical(finalUrl);
+}

package/docs/configuration.md

@@ -120,6 +120,10 @@ Use the dedicated User-Agent option for replacement.
 credentials such as cookies or authorization tokens from being stored in the
 configuration file.
 
+After fetching and source normalization, host and path configuration is
+resolved again against the normalized source URL. This final match determines
+the effective `entryQueryKey` and storage destination.
+
 ## Defuddle options
 
 Supported `defuddle` fields:
@@ -219,6 +223,11 @@ Selection order:
 2. The matching glob with the greatest number of fixed literal characters.
 3. No host configuration.
 
+When the effective `entryQueryKey` has multiple values, mdhq uses the first
+non-empty value and ignores the remaining query parameters for storage
+identity. When no non-empty value exists, the URL uses the generic ordered
+query-string MD5 destination rather than the queryless destination.
+
 If multiple matching patterns have equal specificity, processing fails with
 `CONFIG_ERROR`. Multiple patterns that normalize to the same exact host are
 also an error.
@@ -238,5 +247,7 @@ Rules:
 - A matching path-level string overrides the host value.
 - A matching path-level `null` disables the host value.
 - A matching path object that omits `entryQueryKey` inherits the host value.
-- A missing or empty parameter value falls back to normal path-based storage.
-- All non-selected query parameters are ignored.
+- When at least one non-empty selected value exists, all non-selected query
+  parameters are ignored.
+- When no non-empty selected value exists, a remaining query falls back to the
+  generic ordered query-string MD5 destination.

package/docs/library-api.md

@@ -159,14 +159,16 @@ interface GetPageResult {
 }
 ```
 
-- `requestedUrl` is the input after WHATWG URL serialization. This lowercases
-  and IDNA-normalizes the host, removes an explicit default port, and retains
-  the query and fragment. This serialization is separate from URL identity,
-  which ignores the scheme, fragment, and unselected query parameters.
-- `sourceUrl` is the WHATWG-serialized final URL maintained by the redirect
-  loop. Its query and fragment are whatever remain on that final URL after
-  URL resolution. It is exactly the value written to frontmatter `source`.
-  For a pre-fetch skip, it is the existing document's stored `source` value.
+- `requestedUrl` is the input after WHATWG URL serialization and fragment
+  removal. This lowercases and IDNA-normalizes the host and removes an explicit
+  default port.
+- `sourceUrl` is the normalized source URL written to frontmatter `source`.
+  After redirects, mdhq accepts one HTML canonical URL when its normalized
+  origin and pathname match the final response URL. Invalid, ambiguous,
+  cross-origin, or path-divergent canonical URLs are ignored. Without an
+  accepted canonical, mdhq uses `urlpurify` to remove known tracking
+  parameters from the final response URL. Fragments are always removed.
+  For a pre-fetch skip, `sourceUrl` is the existing document's stored source.
 - `path` is the absolute Markdown path.
 - `assets` is empty when page processing is skipped.
 - `warnings` contains configuration and asset warnings.
@@ -187,6 +189,11 @@ for the same HTTP target and does not include credentials. `etag` and
 `last_modified` are stored in Markdown frontmatter only when the response is
 safe to revalidate; `Vary` and body digests are not serialized.
 
+The fast pre-fetch skip is retained. If the requested URL candidate already
+maps to an existing document and `update` is false, mdhq does not make a
+network request and therefore cannot observe remote redirect or canonical
+changes.
+
 ### Asset results
 
 ```ts

package/docs/specification.md

@@ -93,22 +93,31 @@ create it.
 
 1. Validate that the requested URL uses HTTP or HTTPS.
 2. Load the JSON configuration and collect unknown-key warnings.
-3. Resolve the storage root and host/path-specific entry query key.
-4. Check whether the requested URL already has a same-identity destination.
+3. Resolve the storage root and a pre-fetch host/path-specific entry query key.
+4. Remove known tracking parameters from the requested URL candidate and check
+   whether it already has a same-identity destination.
 5. Fetch the page when it cannot be skipped before network access.
-6. Resolve the final URL after redirects and recalculate configuration and
-   destination from that URL.
-7. Check the final destination for another same-identity skip.
-8. Convert the fetched HTML to Markdown with Defuddle.
-9. Normalize ordinary links and discover image links.
-10. Download supported images and rewrite successful image destinations.
-11. Normalize the Markdown body and calculate its SHA-256 content digest.
-12. Build YAML frontmatter.
-13. Save the Markdown with collision-safe create or update behavior.
-
-The final URL determines the destination and the `source` frontmatter field.
-The original URL is retained as `requested_url` only when it differs from the
-final URL.
+6. Resolve the final response URL after redirects.
+7. Derive the normalized source URL from an accepted HTML canonical URL or
+   `urlpurify` tracking-parameter removal.
+8. Recalculate configuration and destination from the normalized source and
+   check the final destination for another same-identity skip.
+9. Convert the fetched HTML to Markdown with Defuddle using the final response
+   URL as its base URL.
+10. Normalize ordinary links and discover image links using the final response
+    URL.
+11. Download supported images and rewrite successful image destinations.
+12. Normalize the Markdown body and calculate its SHA-256 content digest.
+13. Build YAML frontmatter.
+14. Save the Markdown with collision-safe create or update behavior.
+
+The normalized source determines the destination and the `source` frontmatter
+field. The fragment-free WHATWG-serialized original URL is retained as
+`requested_url` only when it differs from the normalized source. Intermediate
+redirect URLs are not stored.
+
+The pre-fetch skip is intentionally retained. When it succeeds, mdhq performs
+no request and cannot discover later redirect or canonical changes.
 
 ## HTTP behavior
 
@@ -136,8 +145,8 @@ With `update`, mdhq uses validators from a recognized existing destination:
 5. Otherwise perform an ordinary GET.
 
 Storage identity is intentionally broader than HTTP validator scope. A
-same-identity URL with a different scheme, query, or path alias is fetched
-without stored validators.
+same-identity URL with a different scheme, ignored secondary query parameter
+under an entry key, or path alias is fetched without stored validators.
 
 Automatic validators are sent only on the first request and are not forwarded
 after redirects. When a stored validator is available, it replaces any
@@ -271,12 +280,14 @@ Identity includes:
 - normalized host
 - canonical pathname
 - one configured entry query key and value, when present and non-empty
+- otherwise, the MD5 digest of the ordered query tail when a query remains
 
 Identity ignores:
 
 - the difference between HTTP and HTTPS
 - fragments
-- all query parameters except the selected entry query key
+- query parameters other than a selected non-empty entry key when such a key
+  is used
 
 Host normalization:
 
@@ -298,8 +309,42 @@ Path normalization:
 - removes a final recognized HTML extension for identity comparison
 - applies canonical percent encoding for identity comparison
 
-The first value returned for the configured entry query key is used. A missing
-or empty value causes the URL to use its normal path-based identity.
+The first non-empty value returned for the configured entry query key is used.
+When no non-empty value exists, a remaining query uses the generic query-tail
+MD5 identity.
+
+## Source URL normalization
+
+After redirects, mdhq examines `link` elements whose whitespace-separated
+`rel` tokens contain `canonical`, case-insensitively.
+
+- Exactly one canonical link must be present.
+- A relative canonical is resolved against the document base URL. A valid
+  `<base href>` is honored; an invalid base falls back to the final response
+  URL.
+- The resolved canonical must use HTTP or HTTPS.
+- Canonical URLs containing a username or password are rejected.
+- Its normalized origin and canonical pathname must equal those of the final
+  response URL. Scheme, normalized host, port, and pathname aliases therefore
+  cannot point to another content location.
+- An accepted canonical is WHATWG-serialized and has its fragment removed. It
+  is not passed through `urlpurify`.
+
+When canonical is absent or rejected, mdhq applies
+`urlpurify.stripTrackingParams()` to the final response URL. URL wrapper
+unwrapping is not used. The result is validated as HTTP(S), WHATWG-serialized,
+and stripped of its fragment. Functional WordPress preview parameters
+(`preview`, `preview_id`, and `preview_nonce`) are retained because mdhq
+accepts arbitrary page URLs rather than feed URLs only.
+
+The normalized source is used for host/path configuration, storage identity,
+destination, and frontmatter. The final response URL is used as the base for
+Defuddle conversion, ordinary links, images, and assets, as well as for HTTP
+redirect and credential handling. This allows storage-equivalent canonical
+aliases such as `/article` and `/article/` without changing relative URL
+targets.
+
+Re-fetching a path-divergent canonical target is not currently supported.
 
 ## Storage paths
 
@@ -347,13 +392,48 @@ https://example.com/blog/blog.php?entry_id=123
 -> example.com/blog/blog.php/123.md
 ```
 
-Other query parameters and the fragment do not affect the destination.
+Other query parameters and the fragment do not affect the destination when a
+non-empty entry value is selected.
 
-The query value comes from `URLSearchParams.get`, so percent escapes are
-decoded and `+` is interpreted as a space. The resulting value then uses the
-same NFC normalization, unsafe-character encoding, reserved-name handling,
-240-byte limit, and MD5 fallback as a URL path segment. A slash in the query
-value becomes `%2F` inside the filename and never creates another directory.
+Entry values are inspected in URL order and the first non-empty value is used,
+so percent escapes are decoded and `+` is interpreted as a space. The
+resulting value then uses the same NFC normalization, unsafe-character
+encoding, reserved-name handling, 240-byte limit, and MD5 fallback as a URL
+path segment. A slash in the query value becomes `%2F` inside the filename and
+never creates another directory.
+
+### Generic query destinations
+
+When a normalized source retains a query and has no usable configured entry
+value, mdhq creates a lowercase 32-character MD5 digest from UTF-8 bytes. Query
+parameter order is preserved.
+
+For a pathname not ending in `/`, the digest input is the final raw pathname
+segment followed by the serialized query including `?`:
+
+```text
+https://example.com/path/to.php?id=123
+MD5 input: to.php?id=123
+-> example.com/path/<md5>.md
+```
+
+For a pathname ending in `/`, the digest input is only the serialized query:
+
+```text
+https://example.com/path/to/?id=123
+MD5 input: ?id=123
+-> example.com/path/to/<md5>.md
+```
+
+The final directory segment uses the same recognized HTML-extension
+normalization as queryless storage, so `/to/?id=123` and
+`/to.html/?id=123` resolve to the same destination.
+
+Root queries use `index` as their directory, so `/?id=123` and
+`/index.html/?id=123` both resolve to `index/<md5>.md`.
+
+If canonical selection or tracking cleanup removes the complete query, the
+normal queryless destination rules apply.
 
 ### Safe path segments
 
@@ -410,14 +490,14 @@ Defuddle's exact whitespace.
 
 For ordinary links:
 
-- relative URLs are resolved against the final page URL
+- relative URLs are resolved against the final response URL
 - absolute URLs are retained
 - fragment-only links are retained
 - reference-style link definitions are resolved and normalized
 
 For images:
 
-- relative destinations are resolved against the final page URL
+- relative destinations are resolved against the final response URL
 - reference-style image definitions are resolved and localized
 - absolute image URLs are collected for asset localization
 - duplicate source URLs are fetched once
@@ -552,8 +632,9 @@ Metadata fields are emitted when non-empty:
 
 mdhq-controlled fields:
 
-- `source`: final page URL
-- `requested_url`: original URL, only when different from `source`
+- `source`: normalized source URL
+- `requested_url`: fragment-free serialized original URL, only when different
+  from `source`
 - `created`: initial local acquisition time
 - `modified`: time of the latest meaningful Markdown note change
 - `etag`: HTTP ETag stored verbatim, when supplied

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@songmu/mdhq",
-  "version": "0.0.2",
+  "version": "0.0.3",
   "description": "Save web pages as Markdown in a ghq-inspired filesystem layout.",
   "type": "module",
   "bin": {
@@ -61,6 +61,7 @@
     "temml": "^0.13.5",
     "turndown": "^7.2.0",
     "undici": "^8.10.0",
+    "urlpurify": "1.4.0",
     "yaml": "^2.8.1",
     "zod": "^4.5.2"
   },