@songmu/mdhq 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +126 -0
- package/dist/assets/localize.d.ts +19 -0
- package/dist/assets/localize.js +364 -0
- package/dist/cli.d.ts +8 -0
- package/dist/cli.js +119 -0
- package/dist/config/config.d.ts +25 -0
- package/dist/config/config.js +170 -0
- package/dist/config/match.d.ts +7 -0
- package/dist/config/match.js +101 -0
- package/dist/convert/article-date.d.ts +20 -0
- package/dist/convert/article-date.js +255 -0
- package/dist/convert/convert-html.d.ts +2 -0
- package/dist/convert/convert-html.js +89 -0
- package/dist/convert/extract-published.d.ts +12 -0
- package/dist/convert/extract-published.js +24 -0
- package/dist/convert/extract-updated.d.ts +8 -0
- package/dist/convert/extract-updated.js +20 -0
- package/dist/date.d.ts +18 -0
- package/dist/date.js +448 -0
- package/dist/errors.d.ts +8 -0
- package/dist/errors.js +10 -0
- package/dist/frontmatter/frontmatter.d.ts +40 -0
- package/dist/frontmatter/frontmatter.js +114 -0
- package/dist/get-page.d.ts +2 -0
- package/dist/get-page.js +308 -0
- package/dist/http/fetch.d.ts +46 -0
- package/dist/http/fetch.js +195 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +3 -0
- package/dist/list-files.d.ts +8 -0
- package/dist/list-files.js +35 -0
- package/dist/markdown/transform.d.ts +6 -0
- package/dist/markdown/transform.js +129 -0
- package/dist/path/storage-path.d.ts +7 -0
- package/dist/path/storage-path.js +110 -0
- package/dist/storage/atomic.d.ts +8 -0
- package/dist/storage/atomic.js +84 -0
- package/dist/storage/path-safety.d.ts +1 -0
- package/dist/storage/path-safety.js +55 -0
- package/dist/storage/save.d.ts +23 -0
- package/dist/storage/save.js +118 -0
- package/dist/types.d.ts +62 -0
- package/dist/types.js +1 -0
- package/dist/url/identity.d.ts +12 -0
- package/dist/url/identity.js +54 -0
- package/dist/url/pathname.d.ts +4 -0
- package/dist/url/pathname.js +46 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.js +6 -0
- package/docs/README.md +14 -0
- package/docs/configuration.md +242 -0
- package/docs/library-api.md +275 -0
- package/docs/specification.md +730 -0
- package/package.json +73 -0
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { gfmFromMarkdown, gfmToMarkdown } from "mdast-util-gfm";
|
|
2
|
+
import { fromMarkdown } from "mdast-util-from-markdown";
|
|
3
|
+
import { toMarkdown } from "mdast-util-to-markdown";
|
|
4
|
+
import { gfm } from "micromark-extension-gfm";
|
|
5
|
+
function walk(node, visitor) {
|
|
6
|
+
visitor(node);
|
|
7
|
+
const children = node.children;
|
|
8
|
+
if (Array.isArray(children)) {
|
|
9
|
+
for (const child of children) {
|
|
10
|
+
walk(child, visitor);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
function absoluteUrl(value, baseUrl) {
|
|
15
|
+
if (value.startsWith("#")) {
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
try {
|
|
19
|
+
return new URL(value, baseUrl).href;
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return value;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
function parseMarkdown(markdown) {
|
|
26
|
+
return fromMarkdown(markdown, {
|
|
27
|
+
extensions: [gfm()],
|
|
28
|
+
mdastExtensions: [gfmFromMarkdown()]
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
function serializeMarkdown(tree) {
|
|
32
|
+
return toMarkdown(tree, { extensions: [gfmToMarkdown()] }).trimEnd();
|
|
33
|
+
}
|
|
34
|
+
function isDownloadableImage(value) {
|
|
35
|
+
try {
|
|
36
|
+
const protocol = new URL(value).protocol;
|
|
37
|
+
return protocol === "http:" || protocol === "https:";
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
export function transformMarkdown(markdown, baseUrl) {
|
|
44
|
+
const tree = parseMarkdown(markdown);
|
|
45
|
+
const definitions = new Map();
|
|
46
|
+
walk(tree, (node) => {
|
|
47
|
+
if (node.type === "definition") {
|
|
48
|
+
if (!definitions.has(node.identifier)) {
|
|
49
|
+
definitions.set(node.identifier, node);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
const imageUrls = [];
|
|
54
|
+
const seenImages = new Set();
|
|
55
|
+
const addImage = (url) => {
|
|
56
|
+
if (isDownloadableImage(url) && !seenImages.has(url)) {
|
|
57
|
+
seenImages.add(url);
|
|
58
|
+
imageUrls.push(url);
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
walk(tree, (node) => {
|
|
62
|
+
if (node.type === "link") {
|
|
63
|
+
const link = node;
|
|
64
|
+
link.url = absoluteUrl(link.url, baseUrl);
|
|
65
|
+
}
|
|
66
|
+
else if (node.type === "image") {
|
|
67
|
+
const image = node;
|
|
68
|
+
image.url = absoluteUrl(image.url, baseUrl);
|
|
69
|
+
addImage(image.url);
|
|
70
|
+
}
|
|
71
|
+
else if (node.type === "linkReference" || node.type === "imageReference") {
|
|
72
|
+
const definition = definitions.get(node.identifier);
|
|
73
|
+
if (definition) {
|
|
74
|
+
definition.url = absoluteUrl(definition.url, baseUrl);
|
|
75
|
+
if (node.type === "imageReference") {
|
|
76
|
+
addImage(definition.url);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
});
|
|
81
|
+
return { markdown: serializeMarkdown(tree), imageUrls };
|
|
82
|
+
}
|
|
83
|
+
export function rewriteImageUrls(markdown, replacements) {
|
|
84
|
+
const tree = parseMarkdown(markdown);
|
|
85
|
+
const definitions = new Map();
|
|
86
|
+
const linkReferences = new Set();
|
|
87
|
+
walk(tree, (node) => {
|
|
88
|
+
if (node.type === "definition") {
|
|
89
|
+
if (!definitions.has(node.identifier)) {
|
|
90
|
+
definitions.set(node.identifier, node);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
else if (node.type === "linkReference") {
|
|
94
|
+
linkReferences.add(node.identifier);
|
|
95
|
+
}
|
|
96
|
+
else if (node.type === "image") {
|
|
97
|
+
const image = node;
|
|
98
|
+
image.url = replacements.get(image.url) ?? image.url;
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
const rewriteReferences = (parent) => {
|
|
102
|
+
parent.children = parent.children.map((node) => {
|
|
103
|
+
if (node.type === "imageReference") {
|
|
104
|
+
const imageReference = node;
|
|
105
|
+
const definition = definitions.get(imageReference.identifier);
|
|
106
|
+
const replacement = definition
|
|
107
|
+
? replacements.get(definition.url)
|
|
108
|
+
: undefined;
|
|
109
|
+
if (definition && replacement) {
|
|
110
|
+
if (linkReferences.has(imageReference.identifier)) {
|
|
111
|
+
return {
|
|
112
|
+
type: "image",
|
|
113
|
+
url: replacement,
|
|
114
|
+
alt: imageReference.alt,
|
|
115
|
+
title: definition.title
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
definition.url = replacement;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
if ("children" in node && Array.isArray(node.children)) {
|
|
122
|
+
rewriteReferences(node);
|
|
123
|
+
}
|
|
124
|
+
return node;
|
|
125
|
+
});
|
|
126
|
+
};
|
|
127
|
+
rewriteReferences(tree);
|
|
128
|
+
return serializeMarkdown(tree);
|
|
129
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { MdhqError } from "../errors.js";
|
|
4
|
+
import { createUrlIdentity, parseHttpUrl } from "../url/identity.js";
|
|
5
|
+
import { decodeUrlPathSegment, storageBasename } from "../url/pathname.js";
|
|
6
|
+
const INVALID_CHARACTERS = /[\/\\:*?"<>|\u0000-\u001f\u007f]/u;
|
|
7
|
+
const WINDOWS_DEVICES = /^(con|prn|aux|nul|com(?:[1-9]|[¹²³])|lpt(?:[1-9]|[¹²³]))(?:\.|$)/iu;
|
|
8
|
+
const MAX_SEGMENT_BYTES = 240;
|
|
9
|
+
const MAX_ABSOLUTE_PATH_LENGTH = process.platform === "win32" ? 240 : 1000;
|
|
10
|
+
function md5(value) {
|
|
11
|
+
return createHash("md5").update(value).digest("hex");
|
|
12
|
+
}
|
|
13
|
+
function absolutePathLength(value) {
|
|
14
|
+
return process.platform === "win32" ? value.length : Buffer.byteLength(value);
|
|
15
|
+
}
|
|
16
|
+
function ensureWithinRoot(root, target) {
|
|
17
|
+
const relative = path.relative(root, target);
|
|
18
|
+
if (relative === ".." ||
|
|
19
|
+
relative.startsWith(`..${path.sep}`) ||
|
|
20
|
+
path.isAbsolute(relative)) {
|
|
21
|
+
throw new MdhqError("PATH_COLLISION", `Storage path escapes its root: ${target}`);
|
|
22
|
+
}
|
|
23
|
+
return target;
|
|
24
|
+
}
|
|
25
|
+
function encodeCharacter(character) {
|
|
26
|
+
return [...Buffer.from(character)].map((byte) => `%${byte.toString(16).toUpperCase().padStart(2, "0")}`).join("");
|
|
27
|
+
}
|
|
28
|
+
export function sanitizePathSegment(rawSegment) {
|
|
29
|
+
const normalized = decodeUrlPathSegment(rawSegment);
|
|
30
|
+
if (normalized === "." || normalized === ".." || WINDOWS_DEVICES.test(normalized)) {
|
|
31
|
+
return [...normalized].map(encodeCharacter).join("");
|
|
32
|
+
}
|
|
33
|
+
const characters = [...normalized];
|
|
34
|
+
return characters
|
|
35
|
+
.map((character, index) => {
|
|
36
|
+
const trailingUnsafe = index === characters.length - 1 && (character === " " || character === ".");
|
|
37
|
+
return INVALID_CHARACTERS.test(character) || character === "%" || trailingUnsafe
|
|
38
|
+
? encodeCharacter(character)
|
|
39
|
+
: character;
|
|
40
|
+
})
|
|
41
|
+
.join("");
|
|
42
|
+
}
|
|
43
|
+
function fitSegment(rawSegment, suffix = "") {
|
|
44
|
+
const sanitized = sanitizePathSegment(rawSegment);
|
|
45
|
+
return Buffer.byteLength(`${sanitized}${suffix}`) <= MAX_SEGMENT_BYTES
|
|
46
|
+
? sanitized
|
|
47
|
+
: md5(decodeUrlPathSegment(rawSegment));
|
|
48
|
+
}
|
|
49
|
+
function markdownFilename(rawSegment) {
|
|
50
|
+
const safe = fitSegment(encodeURIComponent(storageBasename(rawSegment)), ".md");
|
|
51
|
+
return `${safe}.md`;
|
|
52
|
+
}
|
|
53
|
+
function hostDirectory(url) {
|
|
54
|
+
const identity = createUrlIdentity(url);
|
|
55
|
+
const ipv6 = identity.host.match(/^\[([^\]]+)\](?::(\d+))?$/u);
|
|
56
|
+
const host = ipv6
|
|
57
|
+
? `[${ipv6[1]?.replaceAll(":", "_")}]${ipv6[2] ? `_${ipv6[2]}` : ""}`
|
|
58
|
+
: identity.host.replaceAll(":", "_");
|
|
59
|
+
const safeHost = fitSegment(encodeURIComponent(host));
|
|
60
|
+
if (safeHost.toLowerCase() === "_assets") {
|
|
61
|
+
throw new MdhqError("PATH_COLLISION", "The normalized host conflicts with _assets");
|
|
62
|
+
}
|
|
63
|
+
return safeHost;
|
|
64
|
+
}
|
|
65
|
+
export function storagePathForUrl(options) {
|
|
66
|
+
const url = parseHttpUrl(options.url);
|
|
67
|
+
const identity = createUrlIdentity(url, options.entryQueryKey);
|
|
68
|
+
const rawSegments = url.pathname.split("/").filter(Boolean);
|
|
69
|
+
const directories = rawSegments.slice(0, -1).map((segment) => fitSegment(segment));
|
|
70
|
+
let filename;
|
|
71
|
+
if (identity.entryValue !== undefined) {
|
|
72
|
+
const pageSegment = rawSegments.at(-1) ?? "index";
|
|
73
|
+
directories.push(fitSegment(pageSegment));
|
|
74
|
+
filename = `${fitSegment(encodeURIComponent(identity.entryValue), ".md")}.md`;
|
|
75
|
+
}
|
|
76
|
+
else if (rawSegments.length === 0) {
|
|
77
|
+
filename = "index.md";
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
filename = markdownFilename(rawSegments.at(-1) ?? "index");
|
|
81
|
+
}
|
|
82
|
+
const allSegments = [hostDirectory(url), ...directories, filename];
|
|
83
|
+
const resolvedRoot = path.resolve(options.root);
|
|
84
|
+
let result = path.resolve(resolvedRoot, ...allSegments);
|
|
85
|
+
if (absolutePathLength(result) <= MAX_ABSOLUTE_PATH_LENGTH) {
|
|
86
|
+
return ensureWithinRoot(resolvedRoot, result);
|
|
87
|
+
}
|
|
88
|
+
const candidates = allSegments
|
|
89
|
+
.map((segment, index) => ({
|
|
90
|
+
index,
|
|
91
|
+
length: absolutePathLength(segment),
|
|
92
|
+
replacementLength: segment.endsWith(".md") ? 35 : 32
|
|
93
|
+
}))
|
|
94
|
+
.filter((candidate) => candidate.length > candidate.replacementLength)
|
|
95
|
+
.sort((a, b) => b.length - a.length);
|
|
96
|
+
for (const candidate of candidates) {
|
|
97
|
+
const current = allSegments[candidate.index];
|
|
98
|
+
if (!current) {
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
allSegments[candidate.index] = current.endsWith(".md")
|
|
102
|
+
? `${md5(current.slice(0, -3))}.md`
|
|
103
|
+
: md5(current);
|
|
104
|
+
result = path.resolve(resolvedRoot, ...allSegments);
|
|
105
|
+
if (absolutePathLength(result) <= MAX_ABSOLUTE_PATH_LENGTH) {
|
|
106
|
+
return ensureWithinRoot(resolvedRoot, result);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
throw new MdhqError("PATH_TOO_LONG", `Storage path is too long for ${url.href}: ${result}`);
|
|
110
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
type FileContent = string | Uint8Array;
|
|
2
|
+
export declare function withDestinationLock<T>(targetPath: string, operation: () => Promise<T>, root?: string): Promise<T>;
|
|
3
|
+
export declare function publishFileExclusive(targetPath: string, content: FileContent, root?: string): Promise<boolean>;
|
|
4
|
+
export declare function replaceFileAtomic(targetPath: string, content: FileContent, options?: {
|
|
5
|
+
root?: string;
|
|
6
|
+
beforeCommit?: () => Promise<void>;
|
|
7
|
+
}): Promise<void>;
|
|
8
|
+
export {};
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { link, mkdir, rename, unlink, writeFile } from "node:fs/promises";
|
|
4
|
+
import { lock } from "proper-lockfile";
|
|
5
|
+
import { assertSafeDestination } from "./path-safety.js";
|
|
6
|
+
const LOCK_STALE_MS = 300_000;
|
|
7
|
+
const LOCK_RETRY_INTERVAL_MS = 100;
|
|
8
|
+
const LOCK_WAIT_MS = 60_000;
|
|
9
|
+
function temporaryPath(targetPath) {
|
|
10
|
+
return path.join(path.dirname(targetPath), `.${path.basename(targetPath)}.${randomUUID()}.tmp`);
|
|
11
|
+
}
|
|
12
|
+
export async function withDestinationLock(targetPath, operation, root) {
|
|
13
|
+
const lockPath = `${targetPath}.lock`;
|
|
14
|
+
if (root) {
|
|
15
|
+
await assertSafeDestination(root, lockPath);
|
|
16
|
+
}
|
|
17
|
+
await mkdir(path.dirname(lockPath), { recursive: true });
|
|
18
|
+
const release = await lock(targetPath, {
|
|
19
|
+
realpath: false,
|
|
20
|
+
lockfilePath: lockPath,
|
|
21
|
+
stale: LOCK_STALE_MS,
|
|
22
|
+
update: LOCK_STALE_MS / 3,
|
|
23
|
+
retries: {
|
|
24
|
+
retries: LOCK_WAIT_MS / LOCK_RETRY_INTERVAL_MS,
|
|
25
|
+
factor: 1,
|
|
26
|
+
minTimeout: LOCK_RETRY_INTERVAL_MS,
|
|
27
|
+
maxTimeout: LOCK_RETRY_INTERVAL_MS,
|
|
28
|
+
randomize: false
|
|
29
|
+
}
|
|
30
|
+
});
|
|
31
|
+
try {
|
|
32
|
+
return await operation();
|
|
33
|
+
}
|
|
34
|
+
finally {
|
|
35
|
+
await release();
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export async function publishFileExclusive(targetPath, content, root) {
|
|
39
|
+
if (root) {
|
|
40
|
+
await assertSafeDestination(root, targetPath);
|
|
41
|
+
}
|
|
42
|
+
await mkdir(path.dirname(targetPath), { recursive: true });
|
|
43
|
+
if (root) {
|
|
44
|
+
await assertSafeDestination(root, targetPath);
|
|
45
|
+
}
|
|
46
|
+
const tempPath = temporaryPath(targetPath);
|
|
47
|
+
try {
|
|
48
|
+
await writeFile(tempPath, content, { flag: "wx" });
|
|
49
|
+
try {
|
|
50
|
+
await link(tempPath, targetPath);
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
if (error.code === "EEXIST") {
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
throw error;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
await unlink(tempPath).catch(() => undefined);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
export async function replaceFileAtomic(targetPath, content, options = {}) {
|
|
65
|
+
if (options.root) {
|
|
66
|
+
await assertSafeDestination(options.root, targetPath);
|
|
67
|
+
}
|
|
68
|
+
await mkdir(path.dirname(targetPath), { recursive: true });
|
|
69
|
+
if (options.root) {
|
|
70
|
+
await assertSafeDestination(options.root, targetPath);
|
|
71
|
+
}
|
|
72
|
+
const tempPath = temporaryPath(targetPath);
|
|
73
|
+
try {
|
|
74
|
+
await writeFile(tempPath, content, { flag: "wx" });
|
|
75
|
+
await options.beforeCommit?.();
|
|
76
|
+
if (options.root) {
|
|
77
|
+
await assertSafeDestination(options.root, targetPath);
|
|
78
|
+
}
|
|
79
|
+
await rename(tempPath, targetPath);
|
|
80
|
+
}
|
|
81
|
+
finally {
|
|
82
|
+
await unlink(tempPath).catch(() => undefined);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function assertSafeDestination(root: string, target: string): Promise<void>;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { lstat, realpath } from "node:fs/promises";
|
|
3
|
+
import { MdhqError } from "../errors.js";
|
|
4
|
+
function isWithin(root, target) {
|
|
5
|
+
const relative = path.relative(root, target);
|
|
6
|
+
return (relative !== ".." &&
|
|
7
|
+
!relative.startsWith(`..${path.sep}`) &&
|
|
8
|
+
!path.isAbsolute(relative));
|
|
9
|
+
}
|
|
10
|
+
export async function assertSafeDestination(root, target) {
|
|
11
|
+
const resolvedRoot = path.resolve(root);
|
|
12
|
+
const resolvedTarget = path.resolve(target);
|
|
13
|
+
if (!isWithin(resolvedRoot, resolvedTarget)) {
|
|
14
|
+
throw new MdhqError("PATH_COLLISION", `Storage path escapes its root: ${resolvedTarget}`);
|
|
15
|
+
}
|
|
16
|
+
let rootMetadata;
|
|
17
|
+
try {
|
|
18
|
+
rootMetadata = await lstat(resolvedRoot);
|
|
19
|
+
}
|
|
20
|
+
catch (error) {
|
|
21
|
+
if (error.code === "ENOENT") {
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
throw error;
|
|
25
|
+
}
|
|
26
|
+
if (!rootMetadata.isDirectory() && !rootMetadata.isSymbolicLink()) {
|
|
27
|
+
throw new MdhqError("PATH_COLLISION", `Storage root is not a directory: ${resolvedRoot}`);
|
|
28
|
+
}
|
|
29
|
+
const realRoot = await realpath(resolvedRoot);
|
|
30
|
+
const relative = path.relative(resolvedRoot, path.dirname(resolvedTarget));
|
|
31
|
+
let current = resolvedRoot;
|
|
32
|
+
for (const segment of relative.split(path.sep).filter(Boolean)) {
|
|
33
|
+
current = path.join(current, segment);
|
|
34
|
+
let metadata;
|
|
35
|
+
try {
|
|
36
|
+
metadata = await lstat(current);
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
if (error.code === "ENOENT") {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
throw error;
|
|
43
|
+
}
|
|
44
|
+
if (metadata.isSymbolicLink()) {
|
|
45
|
+
throw new MdhqError("PATH_COLLISION", `Storage directory is a symbolic link: ${current}`);
|
|
46
|
+
}
|
|
47
|
+
if (!metadata.isDirectory()) {
|
|
48
|
+
throw new MdhqError("PATH_COLLISION", `Storage path component is not a directory: ${current}`);
|
|
49
|
+
}
|
|
50
|
+
const realCurrent = await realpath(current);
|
|
51
|
+
if (!isWithin(realRoot, realCurrent) && realCurrent !== realRoot) {
|
|
52
|
+
throw new MdhqError("PATH_COLLISION", `Storage directory escapes its root: ${current}`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export interface SaveDocumentOptions {
|
|
2
|
+
path: string;
|
|
3
|
+
content: string;
|
|
4
|
+
sourceUrl: string;
|
|
5
|
+
update: boolean;
|
|
6
|
+
expectedContent?: string | null;
|
|
7
|
+
entryQueryKey?: string;
|
|
8
|
+
root?: string;
|
|
9
|
+
}
|
|
10
|
+
export interface ExistingDocument {
|
|
11
|
+
content: string;
|
|
12
|
+
sourceUrl: string;
|
|
13
|
+
frontmatter: Record<string, unknown>;
|
|
14
|
+
markdown: string;
|
|
15
|
+
contentDigest: string;
|
|
16
|
+
created?: string;
|
|
17
|
+
etag?: string;
|
|
18
|
+
lastModified?: string;
|
|
19
|
+
vary?: string[];
|
|
20
|
+
}
|
|
21
|
+
export declare function readExistingDocument(filePath: string): Promise<ExistingDocument | undefined>;
|
|
22
|
+
export declare function inspectDestination(filePath: string, sourceUrl: string, entryQueryKey?: string, root?: string): Promise<ExistingDocument | undefined>;
|
|
23
|
+
export declare function saveDocument(options: SaveDocumentOptions): Promise<"saved" | "updated" | "skipped" | "conflicted">;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { mkdir, readFile } from "node:fs/promises";
|
|
3
|
+
import { MdhqError } from "../errors.js";
|
|
4
|
+
import { markdownContentDigest, parseDocument } from "../frontmatter/frontmatter.js";
|
|
5
|
+
import { sameUrlIdentity } from "../url/identity.js";
|
|
6
|
+
import { publishFileExclusive, replaceFileAtomic, withDestinationLock } from "./atomic.js";
|
|
7
|
+
import { assertSafeDestination } from "./path-safety.js";
|
|
8
|
+
export async function readExistingDocument(filePath) {
|
|
9
|
+
let content;
|
|
10
|
+
try {
|
|
11
|
+
content = await readFile(filePath, "utf8");
|
|
12
|
+
}
|
|
13
|
+
catch (error) {
|
|
14
|
+
if (error.code === "ENOENT") {
|
|
15
|
+
return undefined;
|
|
16
|
+
}
|
|
17
|
+
throw new MdhqError("STORAGE_ERROR", `Failed to read ${filePath}`, { cause: error });
|
|
18
|
+
}
|
|
19
|
+
const parsed = parseDocument(content);
|
|
20
|
+
if (!parsed || typeof parsed.frontmatter.source !== "string") {
|
|
21
|
+
throw new MdhqError("PATH_COLLISION", `Existing file does not contain a valid source URL: ${filePath}`);
|
|
22
|
+
}
|
|
23
|
+
const frontmatter = parsed.frontmatter;
|
|
24
|
+
const sourceUrl = frontmatter.source;
|
|
25
|
+
return {
|
|
26
|
+
content,
|
|
27
|
+
sourceUrl,
|
|
28
|
+
frontmatter,
|
|
29
|
+
markdown: parsed.markdown,
|
|
30
|
+
contentDigest: markdownContentDigest(parsed.markdown),
|
|
31
|
+
...(typeof frontmatter.created === "string" ? { created: frontmatter.created } : {}),
|
|
32
|
+
...(typeof frontmatter.etag === "string" ? { etag: frontmatter.etag } : {}),
|
|
33
|
+
...(typeof frontmatter.last_modified === "string"
|
|
34
|
+
? { lastModified: frontmatter.last_modified }
|
|
35
|
+
: {}),
|
|
36
|
+
...(Array.isArray(frontmatter.vary) &&
|
|
37
|
+
frontmatter.vary.every((value) => typeof value === "string")
|
|
38
|
+
? { vary: frontmatter.vary }
|
|
39
|
+
: {})
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
function assertSameIdentity(existing, sourceUrl, entryQueryKey, filePath) {
|
|
43
|
+
let matches = false;
|
|
44
|
+
try {
|
|
45
|
+
matches =
|
|
46
|
+
existing !== undefined &&
|
|
47
|
+
sameUrlIdentity(existing.sourceUrl, sourceUrl, entryQueryKey);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
matches = false;
|
|
51
|
+
}
|
|
52
|
+
if (!matches) {
|
|
53
|
+
throw new MdhqError("PATH_COLLISION", `Storage path is already used by another URL: ${filePath}`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
export async function inspectDestination(filePath, sourceUrl, entryQueryKey, root) {
|
|
57
|
+
if (root) {
|
|
58
|
+
await assertSafeDestination(root, filePath);
|
|
59
|
+
}
|
|
60
|
+
const existing = await readExistingDocument(filePath);
|
|
61
|
+
if (existing) {
|
|
62
|
+
assertSameIdentity(existing, sourceUrl, entryQueryKey, filePath);
|
|
63
|
+
}
|
|
64
|
+
return existing;
|
|
65
|
+
}
|
|
66
|
+
export async function saveDocument(options) {
|
|
67
|
+
if (options.root) {
|
|
68
|
+
await assertSafeDestination(options.root, options.path);
|
|
69
|
+
}
|
|
70
|
+
await mkdir(path.dirname(options.path), { recursive: true });
|
|
71
|
+
try {
|
|
72
|
+
return await withDestinationLock(options.path, async () => {
|
|
73
|
+
const existing = await readExistingDocument(options.path);
|
|
74
|
+
const hasExpectation = options.expectedContent !== undefined;
|
|
75
|
+
if (hasExpectation &&
|
|
76
|
+
(existing?.content ?? null) !== options.expectedContent) {
|
|
77
|
+
return "conflicted";
|
|
78
|
+
}
|
|
79
|
+
if (existing) {
|
|
80
|
+
assertSameIdentity(existing, options.sourceUrl, options.entryQueryKey, options.path);
|
|
81
|
+
if (!options.update) {
|
|
82
|
+
return "skipped";
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (existing) {
|
|
86
|
+
await replaceFileAtomic(options.path, options.content, {
|
|
87
|
+
...(options.root ? { root: options.root } : {}),
|
|
88
|
+
beforeCommit: async () => {
|
|
89
|
+
const current = await readExistingDocument(options.path);
|
|
90
|
+
assertSameIdentity(current, options.sourceUrl, options.entryQueryKey, options.path);
|
|
91
|
+
if (hasExpectation &&
|
|
92
|
+
current?.content !== options.expectedContent) {
|
|
93
|
+
throw new DestinationChangedError();
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
return "updated";
|
|
98
|
+
}
|
|
99
|
+
if (await publishFileExclusive(options.path, options.content, options.root)) {
|
|
100
|
+
return "saved";
|
|
101
|
+
}
|
|
102
|
+
const current = await readExistingDocument(options.path);
|
|
103
|
+
assertSameIdentity(current, options.sourceUrl, options.entryQueryKey, options.path);
|
|
104
|
+
return hasExpectation ? "conflicted" : "skipped";
|
|
105
|
+
}, options.root);
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
if (error instanceof DestinationChangedError) {
|
|
109
|
+
return "conflicted";
|
|
110
|
+
}
|
|
111
|
+
if (error instanceof MdhqError) {
|
|
112
|
+
throw error;
|
|
113
|
+
}
|
|
114
|
+
throw new MdhqError("STORAGE_ERROR", `Failed to ${options.update ? "update" : "write"} ${options.path}`, { cause: error });
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
class DestinationChangedError extends Error {
|
|
118
|
+
}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { DefuddleOptions } from "defuddle/node";
|
|
2
|
+
export interface MdhqWarning {
|
|
3
|
+
code: string;
|
|
4
|
+
message: string;
|
|
5
|
+
url?: string;
|
|
6
|
+
}
|
|
7
|
+
export interface HeaderValue {
|
|
8
|
+
name: string;
|
|
9
|
+
value: string;
|
|
10
|
+
}
|
|
11
|
+
export interface ConvertHtmlOptions {
|
|
12
|
+
html: string;
|
|
13
|
+
url: string | URL;
|
|
14
|
+
defuddle?: Omit<DefuddleOptions, "markdown" | "url">;
|
|
15
|
+
}
|
|
16
|
+
export interface ConvertedPage {
|
|
17
|
+
markdown: string;
|
|
18
|
+
metadata: PageMetadata;
|
|
19
|
+
}
|
|
20
|
+
export interface PageMetadata {
|
|
21
|
+
title?: string;
|
|
22
|
+
description?: string;
|
|
23
|
+
author?: string;
|
|
24
|
+
published?: string;
|
|
25
|
+
updated?: string;
|
|
26
|
+
site?: string;
|
|
27
|
+
domain?: string;
|
|
28
|
+
language?: string;
|
|
29
|
+
image?: string;
|
|
30
|
+
favicon?: string;
|
|
31
|
+
wordCount?: number;
|
|
32
|
+
}
|
|
33
|
+
export interface GetPageOptions {
|
|
34
|
+
url: string | URL;
|
|
35
|
+
root?: string;
|
|
36
|
+
configPath?: string;
|
|
37
|
+
assets?: boolean;
|
|
38
|
+
update?: boolean;
|
|
39
|
+
headers?: HeaderValue[];
|
|
40
|
+
userAgent?: string;
|
|
41
|
+
timeoutMs?: number;
|
|
42
|
+
maxResponseBytes?: number;
|
|
43
|
+
maxRedirects?: number;
|
|
44
|
+
useAsync?: boolean;
|
|
45
|
+
now?: () => Date;
|
|
46
|
+
onWarning?: (warning: MdhqWarning) => void;
|
|
47
|
+
}
|
|
48
|
+
export interface AssetResult {
|
|
49
|
+
sourceUrl: string;
|
|
50
|
+
finalUrl?: string;
|
|
51
|
+
path?: string;
|
|
52
|
+
status: "saved" | "reused" | "failed";
|
|
53
|
+
error?: string;
|
|
54
|
+
}
|
|
55
|
+
export interface GetPageResult {
|
|
56
|
+
requestedUrl: string;
|
|
57
|
+
sourceUrl: string;
|
|
58
|
+
path: string;
|
|
59
|
+
status: "saved" | "updated" | "unchanged" | "skipped";
|
|
60
|
+
assets: AssetResult[];
|
|
61
|
+
warnings: MdhqWarning[];
|
|
62
|
+
}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export interface UrlIdentity {
|
|
2
|
+
host: string;
|
|
3
|
+
pathname: string;
|
|
4
|
+
entryKey?: string;
|
|
5
|
+
entryValue?: string;
|
|
6
|
+
}
|
|
7
|
+
export declare function parseHttpUrl(input: string | URL): URL;
|
|
8
|
+
export declare function normalizeHost(url: URL): string;
|
|
9
|
+
export declare function createUrlIdentity(input: string | URL, entryQueryKey?: string): UrlIdentity;
|
|
10
|
+
export declare function serializeUrlIdentity(identity: UrlIdentity): string;
|
|
11
|
+
export declare function sameUrlIdentity(left: string | URL, right: string | URL, entryQueryKey?: string): boolean;
|
|
12
|
+
export declare function sameHttpTarget(left: string | URL, right: string | URL): boolean;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { domainToASCII } from "node:url";
|
|
2
|
+
import { MdhqError } from "../errors.js";
|
|
3
|
+
import { canonicalPathname } from "./pathname.js";
|
|
4
|
+
export function parseHttpUrl(input) {
|
|
5
|
+
let url;
|
|
6
|
+
try {
|
|
7
|
+
url = input instanceof URL ? new URL(input.href) : new URL(input);
|
|
8
|
+
}
|
|
9
|
+
catch (error) {
|
|
10
|
+
throw new MdhqError("INVALID_URL", `Invalid URL: ${String(input)}`, { cause: error });
|
|
11
|
+
}
|
|
12
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
13
|
+
throw new MdhqError("UNSUPPORTED_SCHEME", `Unsupported URL scheme: ${url.protocol}`);
|
|
14
|
+
}
|
|
15
|
+
return url;
|
|
16
|
+
}
|
|
17
|
+
export function normalizeHost(url) {
|
|
18
|
+
const rawHostname = url.hostname.toLowerCase();
|
|
19
|
+
const hostname = domainToASCII(/^\.+$/u.test(rawHostname) ? rawHostname : rawHostname.replace(/\.+$/u, ""));
|
|
20
|
+
const standardPort = (url.protocol === "http:" && url.port === "80") ||
|
|
21
|
+
(url.protocol === "https:" && url.port === "443");
|
|
22
|
+
return url.port && !standardPort ? `${hostname}:${url.port}` : hostname;
|
|
23
|
+
}
|
|
24
|
+
export function createUrlIdentity(input, entryQueryKey) {
|
|
25
|
+
const url = parseHttpUrl(input);
|
|
26
|
+
const entryValue = entryQueryKey ? url.searchParams.get(entryQueryKey) : null;
|
|
27
|
+
const hasEntryValue = entryValue !== null && entryValue !== "";
|
|
28
|
+
const identity = {
|
|
29
|
+
host: normalizeHost(url),
|
|
30
|
+
pathname: canonicalPathname(url.pathname || "/", hasEntryValue)
|
|
31
|
+
};
|
|
32
|
+
if (entryQueryKey && hasEntryValue) {
|
|
33
|
+
identity.entryKey = entryQueryKey;
|
|
34
|
+
identity.entryValue = entryValue.normalize("NFC");
|
|
35
|
+
}
|
|
36
|
+
return identity;
|
|
37
|
+
}
|
|
38
|
+
export function serializeUrlIdentity(identity) {
|
|
39
|
+
const entry = identity.entryKey && identity.entryValue
|
|
40
|
+
? `?${encodeURIComponent(identity.entryKey)}=${encodeURIComponent(identity.entryValue)}`
|
|
41
|
+
: "";
|
|
42
|
+
return `//${identity.host}${identity.pathname}${entry}`;
|
|
43
|
+
}
|
|
44
|
+
export function sameUrlIdentity(left, right, entryQueryKey) {
|
|
45
|
+
return (serializeUrlIdentity(createUrlIdentity(left, entryQueryKey)) ===
|
|
46
|
+
serializeUrlIdentity(createUrlIdentity(right, entryQueryKey)));
|
|
47
|
+
}
|
|
48
|
+
export function sameHttpTarget(left, right) {
|
|
49
|
+
const leftUrl = parseHttpUrl(left);
|
|
50
|
+
const rightUrl = parseHttpUrl(right);
|
|
51
|
+
leftUrl.hash = "";
|
|
52
|
+
rightUrl.hash = "";
|
|
53
|
+
return leftUrl.href === rightUrl.href;
|
|
54
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare const HTML_EXTENSIONS: Set<string>;
|
|
2
|
+
export declare function decodeUrlPathSegment(segment: string): string;
|
|
3
|
+
export declare function storageBasename(segment: string): string;
|
|
4
|
+
export declare function canonicalPathname(pathname: string, hasEntryValue?: boolean): string;
|