@songmu/mdhq 0.0.3 → 0.0.5

package/README.md

@@ -21,6 +21,7 @@ mdhq get https://example.com/article
 mdhq get https://example.com/article https://example.com/another-article
 cat urls.txt | mdhq get
 mdhq get --update https://example.com/article
+mdhq get --assets https://example.com/article
 mdhq get --no-assets https://example.com/article
 mdhq get --json --header 'Cookie: session=value' https://example.com/article
 mdhq list
@@ -30,10 +31,13 @@ mdhq root
 
 `mdhq get` accepts multiple URLs as arguments or one URL per line on standard
 input; both sources are merged. It prints one absolute Markdown path per URL
-to stdout by default. Warnings are written to stderr. `--json` returns one
-result object for a single URL, or an array for multiple URLs. Requests are
-limited to eight in parallel, with requests to the same host serialized and
-spaced one second apart; image downloads use the same limits.
+to stdout by default. Warnings are written to stderr. `--json` writes one
+compact JSON result object per URL as JSON Lines. Results are written as soon
+as each URL finishes, so parallel requests can produce output in completion
+order rather than input order. If a later request fails, results already
+written to stdout remain available.
+Requests are limited to eight in parallel, with requests to the same host
+serialized and spaced one second apart; image downloads use the same limits.
 
 `mdhq list` recursively lists `.md` files below the storage root, one per
 line, in sorted root-relative form. Use `-p` or `--full-path` to print absolute
@@ -99,7 +103,8 @@ const saved = await getPage({
 
 `convertHtml` performs extraction without fetching or writing files.
 `getPage` fetches, converts, optionally localizes images, adds frontmatter,
-and saves the document. Set `assets: false` or use `--no-assets` to keep
+and saves the document. Use `--assets` or `--no-assets` to override the
+configured asset behavior. Set `assets: false` or use `--no-assets` to keep
 absolute image URLs without creating `_assets`.
 
 Saved frontmatter uses Obsidian Web Clipper-compatible names such as `title`,

package/dist/cli.js

@@ -65,11 +65,12 @@ export function createProgram(io = process) {
         .description("Fetch and save web pages.")
         .argument("[urls...]")
         .option("--root <path>", "storage root")
+        .option("--assets", "download images")
         .option("--no-assets", "do not download images")
         .option("--update", "update an existing page")
         .option("--user-agent <value>", "HTTP User-Agent")
         .option("--header <header>", "additional HTTP header", collect, [])
-        .addOption(new Option("--json", "print a structured result"))
+        .addOption(new Option("--json", "print results as JSON Lines"))
         .action(async (urls, options) => {
         const inputUrls = await readStdinUrls(io.stdin);
         const requestedUrls = [...(urls ?? []), ...inputUrls]
@@ -79,7 +80,6 @@ export function createProgram(io = process) {
             throw new MdhqError("INVALID_URL", "At least one URL is required");
         }
         const scheduler = new RequestScheduler();
-        const results = [];
         let nextIndex = 0;
         const worker = async () => {
             while (nextIndex < requestedUrls.length) {
@@ -89,22 +89,31 @@ export function createProgram(io = process) {
                 if (url === undefined) {
                     continue;
                 }
-                results[index] = await getPage({
+                const result = await getPage({
                     url,
                     ...(options.root ? { root: options.root } : {}),
-                    ...(options.assets === false ? { assets: false } : {}),
+                    ...(options.assets !== undefined ? { assets: options.assets } : {}),
                     update: options.update ?? false,
                     ...(options.userAgent ? { userAgent: options.userAgent } : {}),
                     headers: parseHeaders(options.header),
                     scheduler,
                     onWarning: (warning) => io.stderr.write(`warning: ${warning.message}\n`)
                 });
+                io.stdout.write(`${options.json ? JSON.stringify(result) : result.path}\n`);
             }
         };
-        await Promise.all(Array.from({ length: Math.min(8, requestedUrls.length) }, () => worker()));
-        io.stdout.write(options.json
-            ? `${JSON.stringify(results.length === 1 ? results[0] : results, null, 2)}\n`
-            : `${results.map((result) => result.path).join("\n")}\n`);
+        const failures = [];
+        await Promise.all(Array.from({ length: Math.min(8, requestedUrls.length) }, async () => {
+            try {
+                await worker();
+            }
+            catch (error) {
+                failures.push(error);
+            }
+        }));
+        if (failures.length > 0) {
+            throw failures[0];
+        }
     });
     program
         .command("list")

package/docs/configuration.md

@@ -106,10 +106,11 @@ Otherwise configuration overrides the built-in default.
 The CLI `--user-agent` option is passed as `GetPageOptions.userAgent` and
 therefore overrides configuration `userAgent`.
 
-The CLI `--no-assets` option passes `GetPageOptions.assets: false` and
-therefore overrides configuration `assets`. When asset localization is
-disabled, image destinations remain absolute URLs, the result contains no
-asset entries, and mdhq does not create `_assets`.
+The CLI `--assets` and `--no-assets` options pass
+`GetPageOptions.assets: true` and `GetPageOptions.assets: false`,
+respectively, and therefore override configuration `assets`. When asset
+localization is disabled, image destinations remain absolute URLs, the result
+contains no asset entries, and mdhq does not create `_assets`.
 
 Generic CLI `--header` values and library `headers` values are appended after
 mdhq creates its `Accept` and User-Agent headers. A generic `User-Agent` or

package/docs/specification.md

@@ -21,13 +21,13 @@ Runtime requirements:
 The executable provides three subcommands:
 
 ```text
-mdhq get [options] <url>
+mdhq get [options] [urls...]
 mdhq list [options]
 mdhq root [options]
 ```
 
-`mdhq get` accepts exactly one URL per invocation. Parallel or multi-URL
-processing is delegated to external tools such as `xargs`.
+`mdhq get` accepts multiple URL arguments and one URL per line from standard
+input. Both sources are merged, and up to eight URLs are processed in parallel.
 
 ### `get` options
 
@@ -37,22 +37,19 @@ processing is delegated to external tools such as `xargs`.
 | `--update` | Fetch and replace an existing document with the same URL identity. |
 | `--user-agent <value>` | Override the default HTTP User-Agent. |
 | `--header <header>` | Add an HTTP header. The option is repeatable and uses `Name: value` syntax. |
-| `--json` | Write a structured result instead of only the Markdown path. |
-
-On success without `--json`, stdout contains exactly one absolute Markdown
-path followed by a newline. Warnings are written to stderr.
-
-With `--json`, stdout contains an object with this shape:
-
-```json
-{
-  "requestedUrl": "https://example.com/start",
-  "sourceUrl": "https://example.com/article",
-  "path": "/data/mdhq/example.com/article.md",
-  "status": "saved",
-  "assets": [],
-  "warnings": []
-}
+| `--json` | Write one result per line as JSON Lines instead of only the Markdown path. |
+
+Without `--json`, stdout receives one absolute Markdown path per requested URL,
+with each path followed by a newline. Each result is written as soon as that
+URL finishes, so parallel requests can produce output in completion order
+rather than input order.
+Warnings are written to stderr.
+
+With `--json`, stdout contains one compact JSON object per requested URL,
+also written as soon as the URL finishes. Each line has this shape:
+
+```jsonl
+{"requestedUrl":"https://example.com/start","sourceUrl":"https://example.com/article","path":"/data/mdhq/example.com/article.md","status":"saved","assets":[],"warnings":[]}
 ```
 
 `status` is one of:
@@ -63,7 +60,8 @@ With `--json`, stdout contains an object with this shape:
   body, including an HTTP 304 response.
 - `skipped`: an existing same-identity file was kept.
 
-An error is written to stderr and causes exit status `1`.
+An error is written to stderr and causes exit status `1`. Results written to
+stdout before the error remain available to downstream consumers.
 
 ### `list`
 
@@ -508,10 +506,11 @@ For images:
 
 Asset localization is enabled by default. It can be disabled with the
 top-level configuration field `assets: false`, library option
-`GetPageOptions.assets: false`, or CLI option `--no-assets`. The library
-option and CLI option take precedence over configuration. When disabled,
-HTTP(S) image destinations are kept as absolute URLs, the result `assets`
-array is empty, and `_assets` is not created.
+`GetPageOptions.assets: false`, or CLI option `--no-assets`. The CLI options
+`--assets` and `--no-assets` explicitly enable or disable localization and
+take precedence over configuration, as does the library option. When
+disabled, HTTP(S) image destinations are kept as absolute URLs, the result
+`assets` array is empty, and `_assets` is not created.
 
 mdhq does not rewrite ordinary links to other locally stored Markdown
 files.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@songmu/mdhq",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "description": "Save web pages as Markdown in a ghq-inspired filesystem layout.",
   "type": "module",
   "bin": {
@@ -69,6 +69,6 @@
     "@types/node": "^26.4.0",
     "@types/proper-lockfile": "4.1.4",
     "typescript": "^7.0.2",
-    "vitest": "^4.1.11"
+    "vitest": "^5.0.0"
   }
 }