@songmu/mdhq 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/cli.js +20 -3
- package/docs/configuration.md +5 -4
- package/docs/specification.md +12 -10
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -21,6 +21,7 @@ mdhq get https://example.com/article
|
|
|
21
21
|
mdhq get https://example.com/article https://example.com/another-article
|
|
22
22
|
cat urls.txt | mdhq get
|
|
23
23
|
mdhq get --update https://example.com/article
|
|
24
|
+
mdhq get --assets https://example.com/article
|
|
24
25
|
mdhq get --no-assets https://example.com/article
|
|
25
26
|
mdhq get --json --header 'Cookie: session=value' https://example.com/article
|
|
26
27
|
mdhq list
|
|
@@ -29,8 +30,8 @@ mdhq root
|
|
|
29
30
|
```
|
|
30
31
|
|
|
31
32
|
`mdhq get` accepts multiple URLs as arguments or one URL per line on standard
|
|
32
|
-
input; both sources are merged. It prints one
|
|
33
|
-
to stdout by default. Warnings are written to stderr. `--json` writes one
|
|
33
|
+
input; both sources are merged. It prints one root-relative Markdown path per
|
|
34
|
+
URL to stdout by default. Warnings are written to stderr. `--json` writes one
|
|
34
35
|
compact JSON result object per URL as JSON Lines. Results are written as soon
|
|
35
36
|
as each URL finishes, so parallel requests can produce output in completion
|
|
36
37
|
order rather than input order. If a later request fails, results already
|
|
@@ -102,7 +103,8 @@ const saved = await getPage({
|
|
|
102
103
|
|
|
103
104
|
`convertHtml` performs extraction without fetching or writing files.
|
|
104
105
|
`getPage` fetches, converts, optionally localizes images, adds frontmatter,
|
|
105
|
-
and saves the document.
|
|
106
|
+
and saves the document. Use `--assets` or `--no-assets` to override the
|
|
107
|
+
configured asset behavior. Set `assets: false` or use `--no-assets` to keep
|
|
106
108
|
absolute image URLs without creating `_assets`.
|
|
107
109
|
|
|
108
110
|
Saved frontmatter uses Obsidian Web Clipper-compatible names such as `title`,
|
package/dist/cli.js
CHANGED
|
@@ -26,6 +26,13 @@ function parseHeaders(values) {
|
|
|
26
26
|
};
|
|
27
27
|
});
|
|
28
28
|
}
|
|
29
|
+
function toRootRelative(root, target) {
|
|
30
|
+
const relative = path.relative(root, target);
|
|
31
|
+
if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) {
|
|
32
|
+
return target;
|
|
33
|
+
}
|
|
34
|
+
return relative;
|
|
35
|
+
}
|
|
29
36
|
async function readStdinUrls(stdin) {
|
|
30
37
|
if (!stdin || stdin.isTTY) {
|
|
31
38
|
return [];
|
|
@@ -65,6 +72,7 @@ export function createProgram(io = process) {
|
|
|
65
72
|
.description("Fetch and save web pages.")
|
|
66
73
|
.argument("[urls...]")
|
|
67
74
|
.option("--root <path>", "storage root")
|
|
75
|
+
.option("--assets", "download images")
|
|
68
76
|
.option("--no-assets", "do not download images")
|
|
69
77
|
.option("--update", "update an existing page")
|
|
70
78
|
.option("--user-agent <value>", "HTTP User-Agent")
|
|
@@ -78,6 +86,8 @@ export function createProgram(io = process) {
|
|
|
78
86
|
if (requestedUrls.length === 0) {
|
|
79
87
|
throw new MdhqError("INVALID_URL", "At least one URL is required");
|
|
80
88
|
}
|
|
89
|
+
const loaded = await loadConfig();
|
|
90
|
+
const root = resolveRoot(options.root, loaded.config);
|
|
81
91
|
const scheduler = new RequestScheduler();
|
|
82
92
|
let nextIndex = 0;
|
|
83
93
|
const worker = async () => {
|
|
@@ -90,15 +100,22 @@ export function createProgram(io = process) {
|
|
|
90
100
|
}
|
|
91
101
|
const result = await getPage({
|
|
92
102
|
url,
|
|
93
|
-
|
|
94
|
-
...(options.assets
|
|
103
|
+
root,
|
|
104
|
+
...(options.assets !== undefined ? { assets: options.assets } : {}),
|
|
95
105
|
update: options.update ?? false,
|
|
96
106
|
...(options.userAgent ? { userAgent: options.userAgent } : {}),
|
|
97
107
|
headers: parseHeaders(options.header),
|
|
98
108
|
scheduler,
|
|
99
109
|
onWarning: (warning) => io.stderr.write(`warning: ${warning.message}\n`)
|
|
100
110
|
});
|
|
101
|
-
|
|
111
|
+
const relativeResult = {
|
|
112
|
+
...result,
|
|
113
|
+
path: toRootRelative(root, result.path),
|
|
114
|
+
assets: result.assets.map((asset) => asset.path === undefined
|
|
115
|
+
? asset
|
|
116
|
+
: { ...asset, path: toRootRelative(root, asset.path) })
|
|
117
|
+
};
|
|
118
|
+
io.stdout.write(`${options.json ? JSON.stringify(relativeResult) : relativeResult.path}\n`);
|
|
102
119
|
}
|
|
103
120
|
};
|
|
104
121
|
const failures = [];
|
package/docs/configuration.md
CHANGED
|
@@ -106,10 +106,11 @@ Otherwise configuration overrides the built-in default.
|
|
|
106
106
|
The CLI `--user-agent` option is passed as `GetPageOptions.userAgent` and
|
|
107
107
|
therefore overrides configuration `userAgent`.
|
|
108
108
|
|
|
109
|
-
The CLI `--
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
109
|
+
The CLI `--assets` and `--no-assets` options pass
|
|
110
|
+
`GetPageOptions.assets: true` and `GetPageOptions.assets: false`,
|
|
111
|
+
respectively, and therefore override configuration `assets`. When asset
|
|
112
|
+
localization is disabled, image destinations remain absolute URLs, the result
|
|
113
|
+
contains no asset entries, and mdhq does not create `_assets`.
|
|
113
114
|
|
|
114
115
|
Generic CLI `--header` values and library `headers` values are appended after
|
|
115
116
|
mdhq creates its `Accept` and User-Agent headers. A generic `User-Agent` or
|
package/docs/specification.md
CHANGED
|
@@ -39,17 +39,18 @@ input. Both sources are merged, and up to eight URLs are processed in parallel.
|
|
|
39
39
|
| `--header <header>` | Add an HTTP header. The option is repeatable and uses `Name: value` syntax. |
|
|
40
40
|
| `--json` | Write one result per line as JSON Lines instead of only the Markdown path. |
|
|
41
41
|
|
|
42
|
-
Without `--json`, stdout receives one
|
|
43
|
-
with each path followed by a newline. Each
|
|
44
|
-
URL finishes, so parallel requests can
|
|
45
|
-
rather than input order.
|
|
42
|
+
Without `--json`, stdout receives one Markdown path per requested URL, relative
|
|
43
|
+
to the effective storage root, with each path followed by a newline. Each
|
|
44
|
+
result is written as soon as that URL finishes, so parallel requests can
|
|
45
|
+
produce output in completion order rather than input order.
|
|
46
46
|
Warnings are written to stderr.
|
|
47
47
|
|
|
48
48
|
With `--json`, stdout contains one compact JSON object per requested URL,
|
|
49
|
-
also written as soon as the URL finishes.
|
|
49
|
+
also written as soon as the URL finishes. Markdown and asset paths are relative
|
|
50
|
+
to the effective storage root. Each line has this shape:
|
|
50
51
|
|
|
51
52
|
```jsonl
|
|
52
|
-
{"requestedUrl":"https://example.com/start","sourceUrl":"https://example.com/article","path":"
|
|
53
|
+
{"requestedUrl":"https://example.com/start","sourceUrl":"https://example.com/article","path":"example.com/article.md","status":"saved","assets":[],"warnings":[]}
|
|
53
54
|
```
|
|
54
55
|
|
|
55
56
|
`status` is one of:
|
|
@@ -506,10 +507,11 @@ For images:
|
|
|
506
507
|
|
|
507
508
|
Asset localization is enabled by default. It can be disabled with the
|
|
508
509
|
top-level configuration field `assets: false`, library option
|
|
509
|
-
`GetPageOptions.assets: false`, or CLI option `--no-assets`. The
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
510
|
+
`GetPageOptions.assets: false`, or CLI option `--no-assets`. The CLI options
|
|
511
|
+
`--assets` and `--no-assets` explicitly enable or disable localization and
|
|
512
|
+
take precedence over configuration, as does the library option. When
|
|
513
|
+
disabled, HTTP(S) image destinations are kept as absolute URLs, the result
|
|
514
|
+
`assets` array is empty, and `_assets` is not created.
|
|
513
515
|
|
|
514
516
|
mdhq does not rewrite ordinary links to other locally stored Markdown
|
|
515
517
|
files.
|