@songmu/mdhq 0.0.2

package/dist/url/pathname.js

@@ -0,0 +1,46 @@
+import { MdhqError } from "../errors.js";
+export const HTML_EXTENSIONS = new Set([
+    ".html",
+    ".htm",
+    ".xhtml",
+    ".php",
+    ".asp",
+    ".aspx",
+    ".jsp",
+    ".jspx"
+]);
+export function decodeUrlPathSegment(segment) {
+    try {
+        return decodeURIComponent(segment.replace(/%(?![0-9a-f]{2})/giu, "%25")).normalize("NFC");
+    }
+    catch (error) {
+        throw new MdhqError("INVALID_URL", `Invalid UTF-8 in path segment: ${segment}`, {
+            cause: error
+        });
+    }
+}
+export function storageBasename(segment) {
+    const normalized = decodeUrlPathSegment(segment);
+    const match = normalized.match(/(\.[^.]+)$/u);
+    const extension = match?.[1];
+    return extension && HTML_EXTENSIONS.has(extension.toLowerCase())
+        ? normalized.slice(0, -extension.length)
+        : normalized;
+}
+export function canonicalPathname(pathname, hasEntryValue = false) {
+    const segments = pathname
+        .split("/")
+        .filter(Boolean)
+        .map((segment) => encodeURIComponent(decodeUrlPathSegment(segment)));
+    if (!hasEntryValue) {
+        if (segments.length === 0) {
+            return "/index";
+        }
+        const finalIndex = segments.length - 1;
+        const finalSegment = pathname.split("/").filter(Boolean).at(-1);
+        if (finalSegment !== undefined) {
+            segments[finalIndex] = encodeURIComponent(storageBasename(finalSegment));
+        }
+    }
+    return `/${segments.join("/")}`;
+}

package/dist/version.d.ts

@@ -0,0 +1,3 @@
+export declare const VERSION: string;
+export declare const PROJECT_URL = "https://github.com/Songmu/mdhq";
+export declare const DEFAULT_USER_AGENT: string;

package/dist/version.js

@@ -0,0 +1,6 @@
+import { createRequire } from "node:module";
+const require = createRequire(import.meta.url);
+const packageJson = require("../package.json");
+export const VERSION = packageJson.version;
+export const PROJECT_URL = "https://github.com/Songmu/mdhq";
+export const DEFAULT_USER_AGENT = `mdhq/${VERSION} (+${PROJECT_URL})`;

package/docs/README.md

@@ -0,0 +1,14 @@
+# mdhq documentation
+
+This directory documents the behavior implemented in mdhq `0.0.0`.
+
+- [Current specification](specification.md) describes CLI behavior, URL
+  identity, HTTP fetching, storage paths, Markdown processing, assets,
+  frontmatter, and concurrent writes.
+- [Configuration reference](configuration.md) describes the JSON
+  configuration file, XDG paths, option precedence, and host/path matching.
+- [Library API reference](library-api.md) describes the public exports,
+  options, results, warnings, and errors.
+
+These documents describe the current implementation. They are not a roadmap
+for unimplemented features.

package/docs/configuration.md

@@ -0,0 +1,242 @@
+# Configuration reference
+
+mdhq reads one optional JSON configuration file.
+
+Default location:
+
+```text
+$XDG_CONFIG_HOME/mdhq/config.json
+```
+
+When `XDG_CONFIG_HOME` is unset:
+
+```text
+~/.config/mdhq/config.json
+```
+
+The library can select another file with `GetPageOptions.configPath`.
+The CLI does not currently expose a configuration-path option.
+
+A missing configuration file is accepted. An unreadable file, malformed
+JSON, a non-object top-level value, or an invalid known value is a fatal
+`CONFIG_ERROR`.
+
+Unknown keys are accepted and reported with `UNKNOWN_CONFIG_KEY` warnings.
+This applies at the top level and within `defuddle`, `frontmatter`, host
+configuration, and path configuration objects.
+
+## Complete example
+
+```json
+{
+  "root": "/data/mdhq",
+  "userAgent": "my-clipper/1.0",
+  "timeoutMs": 30000,
+  "maxResponseBytes": 20971520,
+  "maxRedirects": 10,
+  "assets": false,
+  "useAsync": true,
+  "defuddle": {
+    "removeSmallImages": true,
+    "standardize": true,
+    "language": "en"
+  },
+  "frontmatter": {
+    "exclude": ["description"],
+    "values": {
+      "collection": "reading",
+      "reviewed": false
+    }
+  },
+  "hosts": {
+    "*.example.com": {
+      "entryQueryKey": "entry_id",
+      "paths": {
+        "/articles/*.php": {
+          "entryQueryKey": "id"
+        },
+        "/search/*": {
+          "entryQueryKey": null
+        }
+      }
+    }
+  }
+}
+```
+
+## Top-level fields
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `root` | string | Default storage root. |
+| `userAgent` | string | Default page and asset User-Agent. |
+| `timeoutMs` | positive integer | Timeout in milliseconds for each mdhq HTTP request attempt. |
+| `maxResponseBytes` | positive integer | Maximum buffered bytes for each page or asset response. |
+| `maxRedirects` | non-negative integer | Maximum redirect count for each page or asset request. |
+| `assets` | boolean | Download images into `_assets`; defaults to `true`. |
+| `useAsync` | boolean | Legacy fallback for Defuddle asynchronous extractors. |
+| `defuddle` | object | Defuddle extraction options. |
+| `frontmatter` | object | Frontmatter exclusions and configured values. |
+| `hosts` | object | Host and pathname-specific entry query configuration. |
+
+## Storage root precedence
+
+The effective root is selected in this order:
+
+1. CLI `--root` or library `GetPageOptions.root`
+2. `MDHQ_ROOT`
+3. configuration `root`
+4. `$XDG_DATA_HOME/mdhq`
+5. `~/.local/share/mdhq` when `XDG_DATA_HOME` is unset
+
+The selected root is converted to an absolute path.
+
+## HTTP option precedence
+
+For each setting, a defined `GetPageOptions` value overrides configuration.
+Otherwise configuration overrides the built-in default.
+
+| Library option | Configuration field | Built-in default |
+| --- | --- | --- |
+| `userAgent` | `userAgent` | mdhq version User-Agent |
+| `timeoutMs` | `timeoutMs` | `30000` |
+| `maxResponseBytes` | `maxResponseBytes` | `20971520` |
+| `maxRedirects` | `maxRedirects` | `10` |
+
+The CLI `--user-agent` option is passed as `GetPageOptions.userAgent` and
+therefore overrides configuration `userAgent`.
+
+The CLI `--no-assets` option passes `GetPageOptions.assets: false` and
+therefore overrides configuration `assets`. When asset localization is
+disabled, image destinations remain absolute URLs, the result contains no
+asset entries, and mdhq does not create `_assets`.
+
+Generic CLI `--header` values and library `headers` values are appended after
+mdhq creates its `Accept` and User-Agent headers. A generic `User-Agent` or
+`Accept` entry is combined with the existing value; it does not replace it.
+Use the dedicated User-Agent option for replacement.
+
+`GetPageOptions.headers` has no configuration equivalent. This prevents
+credentials such as cookies or authorization tokens from being stored in the
+configuration file.
+
+## Defuddle options
+
+Supported `defuddle` fields:
+
+| Field | Type |
+| --- | --- |
+| `debug` | boolean |
+| `removeExactSelectors` | boolean |
+| `removePartialSelectors` | boolean |
+| `removeImages` | boolean |
+| `useAsync` | boolean |
+| `removeHiddenElements` | boolean |
+| `removeLowScoring` | boolean |
+| `removeSmallImages` | boolean |
+| `standardize` | boolean |
+| `removeContentPatterns` | boolean |
+| `contentSelector` | string |
+| `language` | string |
+| `includeReplies` | boolean or `"extractors"` |
+| `profile` | boolean |
+
+mdhq always enables Defuddle Markdown output and supplies its own
+proxy-aware fetch implementation. Those values are not configurable through
+the JSON file.
+
+The supplied fetch implementation adds environment proxy handling only.
+Defuddle-internal asynchronous requests do not inherit mdhq generic
+headers, the configured mdhq User-Agent, or mdhq's timeout,
+response-size, and redirect limits.
+
+The effective asynchronous-extractor setting is:
+
+1. `GetPageOptions.useAsync`
+2. `defuddle.useAsync`
+3. top-level `useAsync`
+4. `true`
+
+## Frontmatter configuration
+
+`frontmatter.exclude` is an array of field names to remove from extracted or
+derived metadata.
+
+`frontmatter.values` is an object of scalar fixed values. Accepted values are
+strings, numbers, booleans, and `null`.
+
+Application order:
+
+1. Add extracted metadata and representative-image fields.
+2. Remove fields listed in `exclude`.
+3. Apply fields from `values`.
+4. Add protected mdhq fields.
+
+Protected fields are:
+
+- `source`
+- `requested_url`
+- `created`
+- `modified`
+- `content_digest`
+- `etag`
+- `last_modified`
+- `vary`
+
+`content_digest` and `vary` are reserved legacy names and are removed from
+serialized frontmatter even if configured in `frontmatter.values`.
+
+mdhq does not add a `type` property by default. Add one explicitly when
+desired:
+
+```json
+{
+  "frontmatter": {
+    "values": {
+      "type": "clip"
+    }
+  }
+}
+```
+
+Protected fields cannot be removed or overridden by configuration.
+
+## Host and path matching
+
+`hosts` is an object whose keys are exact host strings or minimatch glob
+patterns.
+
+Before matching, URL hosts are lowercased, converted to IDNA ASCII, and
+normalized for trailing DNS dots and ports. Host patterns are also lowercased
+and literal labels are converted to IDNA ASCII. Explicit ports in patterns
+are retained. A pattern such as `example.com:443` therefore matches an
+explicit non-standard HTTP port 443, while an HTTPS URL using its standard
+port matches `example.com`.
+
+Selection order:
+
+1. A normalized exact match.
+2. The matching glob with the greatest number of fixed literal characters.
+3. No host configuration.
+
+If multiple matching patterns have equal specificity, processing fails with
+`CONFIG_ERROR`. Multiple patterns that normalize to the same exact host are
+also an error.
+
+After selecting a host, `paths` is matched against the URL's encoded
+`pathname` using the same exact-then-most-specific policy. Path patterns are
+not IDNA- or Unicode-normalized.
+
+## Entry query key inheritance
+
+`entryQueryKey` selects the only query parameter that contributes to URL
+identity and the storage path.
+
+Rules:
+
+- A host-level value applies to all paths by default.
+- A matching path-level string overrides the host value.
+- A matching path-level `null` disables the host value.
+- A matching path object that omits `entryQueryKey` inherits the host value.
+- A missing or empty parameter value falls back to normal path-based storage.
+- All non-selected query parameters are ignored.

package/docs/library-api.md

@@ -0,0 +1,275 @@
+# Library API reference
+
+The package is an ECMAScript module:
+
+```ts
+import {
+  convertHtml,
+  getPage,
+  MdhqError
+} from "@songmu/mdhq";
+```
+
+Runtime exports:
+
+- `convertHtml`
+- `getPage`
+- `MdhqError`
+
+Type-only exports:
+
+- `MdhqConfig`
+- `MdhqErrorCode`
+- `AssetResult`
+- `ConvertedPage`
+- `ConvertHtmlOptions`
+- `GetPageOptions`
+- `GetPageResult`
+- `HeaderValue`
+- `MdhqWarning`
+- `PageMetadata`
+
+TypeScript callers should import types with `import type`:
+
+```ts
+import { getPage, MdhqError } from "@songmu/mdhq";
+import type { GetPageOptions, GetPageResult, MdhqConfig } from "@songmu/mdhq";
+```
+
+## `convertHtml`
+
+```ts
+function convertHtml(options: ConvertHtmlOptions): Promise<ConvertedPage>
+```
+
+This is the low-level conversion API. It does not fetch the page, download
+assets, add frontmatter, or write files.
+
+### Options
+
+```ts
+interface ConvertHtmlOptions {
+  html: string;
+  url: string | URL;
+  defuddle?: Omit<DefuddleOptions, "markdown" | "url">;
+}
+```
+
+- `html` is the complete input HTML string.
+- `url` is an absolute base URL. Unlike `getPage`, it is not restricted to
+  HTTP or HTTPS; for example, a `file:` URL is accepted.
+- `defuddle` passes options to Defuddle.
+
+mdhq forces `markdown: true`. `useAsync` defaults to `true` when it is not
+provided.
+
+### Result
+
+```ts
+interface ConvertedPage {
+  markdown: string;
+  metadata: PageMetadata;
+}
+```
+
+```ts
+interface PageMetadata {
+  title?: string;
+  description?: string;
+  author?: string;
+  published?: string;
+  updated?: string;
+  site?: string;
+  domain?: string;
+  language?: string;
+  image?: string;
+  favicon?: string;
+  wordCount?: number;
+}
+```
+
+Empty string metadata is omitted. `wordCount` is omitted unless it is greater
+than zero.
+
+## `getPage`
+
+```ts
+function getPage(options: GetPageOptions): Promise<GetPageResult>
+```
+
+This is the high-level fetch, convert, localize, and save API.
+
+### Options
+
+```ts
+interface GetPageOptions {
+  url: string | URL;
+  root?: string;
+  configPath?: string;
+  assets?: boolean;
+  update?: boolean;
+  headers?: HeaderValue[];
+  userAgent?: string;
+  timeoutMs?: number;
+  maxResponseBytes?: number;
+  maxRedirects?: number;
+  useAsync?: boolean;
+  now?: () => Date;
+  onWarning?: (warning: MdhqWarning) => void;
+}
+```
+
+| Field | Behavior |
+| --- | --- |
+| `url` | Required HTTP or HTTPS page URL. |
+| `root` | Highest-precedence library storage root. |
+| `configPath` | Override the default XDG configuration path. |
+| `assets` | Download and localize images when `true`; overrides configuration and defaults to `true`. When `false`, image destinations remain absolute and `_assets` is not created. |
+| `update` | Replace an existing same-identity document when `true`; defaults to `false`. |
+| `headers` | Additional page headers. Sent only while requests remain on the initial page origin. A cross-origin page redirect permanently disables them for later page and asset requests in that operation. |
+| `userAgent` | Override the configured or built-in User-Agent. |
+| `timeoutMs` | Override the configured or built-in per-request timeout. |
+| `maxResponseBytes` | Override the configured or built-in response-size limit. |
+| `maxRedirects` | Override the configured or built-in redirect limit. |
+| `useAsync` | Highest-precedence Defuddle asynchronous-extractor setting. |
+| `now` | Clock injection used for `created` and `modified`; intended for deterministic callers and tests. Both fields are written on initial acquisition. |
+| `onWarning` | Called once for each warning as it is produced. |
+
+### Headers
+
+```ts
+interface HeaderValue {
+  name: string;
+  value: string;
+}
+```
+
+Multiple entries with the same name are appended.
+
+### Result
+
+```ts
+interface GetPageResult {
+  requestedUrl: string;
+  sourceUrl: string;
+  path: string;
+  status: "saved" | "updated" | "unchanged" | "skipped";
+  assets: AssetResult[];
+  warnings: MdhqWarning[];
+}
+```
+
+- `requestedUrl` is the input after WHATWG URL serialization. This lowercases
+  and IDNA-normalizes the host, removes an explicit default port, and retains
+  the query and fragment. This serialization is separate from URL identity,
+  which ignores the scheme, fragment, and unselected query parameters.
+- `sourceUrl` is the WHATWG-serialized final URL maintained by the redirect
+  loop. Its query and fragment are whatever remain on that final URL after
+  URL resolution. It is exactly the value written to frontmatter `source`.
+  For a pre-fetch skip, it is the existing document's stored `source` value.
+- `path` is the absolute Markdown path.
+- `assets` is empty when page processing is skipped.
+- `warnings` contains configuration and asset warnings.
+
+Status meanings:
+
+- `saved`: a new document was created.
+- `updated`: an existing document's normalized Markdown body or user-facing
+  frontmatter changed.
+- `unchanged`: HTTP returned 304, or a 200 response produced the same
+  normalized Markdown body and user-facing frontmatter. HTTP validators may
+  still be updated.
+- `skipped`: an existing same-identity document was found without `update`.
+
+With `update`, `getPage` sends the stored `etag` as `If-None-Match`, or falls
+back to the stored `last_modified` as `If-Modified-Since`, when the request is
+for the same HTTP target and does not include credentials. `etag` and
+`last_modified` are stored in Markdown frontmatter only when the response is
+safe to revalidate; `Vary` and body digests are not serialized.
+
+### Asset results
+
+```ts
+interface AssetResult {
+  sourceUrl: string;
+  finalUrl?: string;
+  path?: string;
+  status: "saved" | "reused" | "failed";
+  error?: string;
+}
+```
+
+- `sourceUrl` is the absolute pre-fetch asset URL discovered in Markdown or
+  representative-image metadata.
+- `path` is the absolute local asset path.
+- `saved` means a new immutable content-addressed asset file was created.
+- `reused` means the deterministic asset path already existed with identical
+  bytes.
+- `failed` means the Markdown operation continued without localizing that
+  asset, including the unlikely case of differing bytes at the same digest
+  and extension path.
+- `finalUrl` and `path` are present for successful asset operations.
+- `error` is present for failed operations.
+
+The `assets` array follows first-discovery order. Markdown images are listed
+in document traversal order, followed by the representative image when it was
+not already discovered. Exact duplicate source URLs produce one result.
+Different source URLs that redirect to the same final URL produce separate
+results but reuse the same deterministic file.
+
+### Warnings
+
+```ts
+interface MdhqWarning {
+  code: string;
+  message: string;
+  url?: string;
+}
+```
+
+Current warning codes:
+
+| Code | Meaning |
+| --- | --- |
+| `UNKNOWN_CONFIG_KEY` | A JSON configuration key is not recognized but processing continues. |
+| `ASSET_FETCH_FAILED` | An individual asset could not be fetched, validated, or saved. |
+| `INVALID_IMAGE_URL` | Defuddle returned an invalid or non-HTTP(S) representative-image URL; the page is saved without that metadata field. |
+| `INVALID_LAST_MODIFIED` | A response contained an invalid `Last-Modified` value; the page is saved without that validator. |
+
+Warnings are both accumulated in the result and delivered to `onWarning`.
+
+## Error model
+
+Fatal operational errors are represented by:
+
+```ts
+class MdhqError extends Error {
+  readonly code: MdhqErrorCode;
+  readonly cause?: unknown;
+}
+```
+
+Current error codes:
+
+| Code | Meaning |
+| --- | --- |
+| `INVALID_URL` | A URL or URL path encoding is invalid. |
+| `INVALID_HEADER` | A CLI `--header` value does not use `Name: value` syntax. |
+| `UNSUPPORTED_SCHEME` | The high-level operation received a non-HTTP(S) URL or a redirect used one. |
+| `UNSUPPORTED_CONTENT_TYPE` | A page response was not HTML or XHTML. |
+| `RESPONSE_TOO_LARGE` | A page exceeded the configured byte limit. The same condition for an individual asset is converted to an `ASSET_FETCH_FAILED` warning and failed asset result by `getPage`. |
+| `TOO_MANY_REDIRECTS` | The configured redirect limit was exceeded. |
+| `FETCH_FAILED` | Network, timeout, HTTP-status, or redirect metadata failure. |
+| `CONVERSION_FAILED` | Defuddle failed or produced no usable Markdown. |
+| `CONFIG_ERROR` | Configuration reading, syntax, type validation, or pattern selection failed. |
+| `PATH_COLLISION` | A destination belongs to a different URL identity or cannot be identified safely. |
+| `PATH_TOO_LONG` | The destination cannot fit after path-segment hashing. |
+| `STORAGE_ERROR` | A Markdown read, create, temporary write, or replacement failed. |
+
+Callers should branch on `error.code` rather than parsing `error.message`.
+
+Errors produced while processing an individual asset do not escape
+`getPage`. They are converted into `ASSET_FETCH_FAILED` warnings and
+`AssetResult` objects with `status: "failed"`. The error table describes
+fatal page-level and Markdown-storage failures unless a row explicitly says
+otherwise.