@songmu/mdhq 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +126 -0
- package/dist/assets/localize.d.ts +19 -0
- package/dist/assets/localize.js +364 -0
- package/dist/cli.d.ts +8 -0
- package/dist/cli.js +119 -0
- package/dist/config/config.d.ts +25 -0
- package/dist/config/config.js +170 -0
- package/dist/config/match.d.ts +7 -0
- package/dist/config/match.js +101 -0
- package/dist/convert/article-date.d.ts +20 -0
- package/dist/convert/article-date.js +255 -0
- package/dist/convert/convert-html.d.ts +2 -0
- package/dist/convert/convert-html.js +89 -0
- package/dist/convert/extract-published.d.ts +12 -0
- package/dist/convert/extract-published.js +24 -0
- package/dist/convert/extract-updated.d.ts +8 -0
- package/dist/convert/extract-updated.js +20 -0
- package/dist/date.d.ts +18 -0
- package/dist/date.js +448 -0
- package/dist/errors.d.ts +8 -0
- package/dist/errors.js +10 -0
- package/dist/frontmatter/frontmatter.d.ts +40 -0
- package/dist/frontmatter/frontmatter.js +114 -0
- package/dist/get-page.d.ts +2 -0
- package/dist/get-page.js +308 -0
- package/dist/http/fetch.d.ts +46 -0
- package/dist/http/fetch.js +195 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +3 -0
- package/dist/list-files.d.ts +8 -0
- package/dist/list-files.js +35 -0
- package/dist/markdown/transform.d.ts +6 -0
- package/dist/markdown/transform.js +129 -0
- package/dist/path/storage-path.d.ts +7 -0
- package/dist/path/storage-path.js +110 -0
- package/dist/storage/atomic.d.ts +8 -0
- package/dist/storage/atomic.js +84 -0
- package/dist/storage/path-safety.d.ts +1 -0
- package/dist/storage/path-safety.js +55 -0
- package/dist/storage/save.d.ts +23 -0
- package/dist/storage/save.js +118 -0
- package/dist/types.d.ts +62 -0
- package/dist/types.js +1 -0
- package/dist/url/identity.d.ts +12 -0
- package/dist/url/identity.js +54 -0
- package/dist/url/pathname.d.ts +4 -0
- package/dist/url/pathname.js +46 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.js +6 -0
- package/docs/README.md +14 -0
- package/docs/configuration.md +242 -0
- package/docs/library-api.md +275 -0
- package/docs/specification.md +730 -0
- package/package.json +73 -0
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import os from "node:os";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { readFile } from "node:fs/promises";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { MdhqError } from "../errors.js";
|
|
6
|
+
const pathConfigSchema = z
|
|
7
|
+
.object({
|
|
8
|
+
entryQueryKey: z.string().nullable().optional()
|
|
9
|
+
})
|
|
10
|
+
.passthrough();
|
|
11
|
+
const hostConfigSchema = pathConfigSchema
|
|
12
|
+
.extend({
|
|
13
|
+
paths: z.record(z.string(), pathConfigSchema).optional()
|
|
14
|
+
})
|
|
15
|
+
.passthrough();
|
|
16
|
+
const frontmatterSchema = z
|
|
17
|
+
.object({
|
|
18
|
+
exclude: z.array(z.string()).optional(),
|
|
19
|
+
values: z.record(z.string(), z.union([z.string(), z.number(), z.boolean(), z.null()])).optional()
|
|
20
|
+
})
|
|
21
|
+
.passthrough();
|
|
22
|
+
const defuddleSchema = z
|
|
23
|
+
.object({
|
|
24
|
+
debug: z.boolean().optional(),
|
|
25
|
+
removeExactSelectors: z.boolean().optional(),
|
|
26
|
+
removePartialSelectors: z.boolean().optional(),
|
|
27
|
+
removeImages: z.boolean().optional(),
|
|
28
|
+
useAsync: z.boolean().optional(),
|
|
29
|
+
removeHiddenElements: z.boolean().optional(),
|
|
30
|
+
removeLowScoring: z.boolean().optional(),
|
|
31
|
+
removeSmallImages: z.boolean().optional(),
|
|
32
|
+
standardize: z.boolean().optional(),
|
|
33
|
+
removeContentPatterns: z.boolean().optional(),
|
|
34
|
+
contentSelector: z.string().optional(),
|
|
35
|
+
language: z.string().optional(),
|
|
36
|
+
includeReplies: z.union([z.boolean(), z.literal("extractors")]).optional(),
|
|
37
|
+
profile: z.boolean().optional()
|
|
38
|
+
})
|
|
39
|
+
.passthrough();
|
|
40
|
+
const configSchema = z
|
|
41
|
+
.object({
|
|
42
|
+
root: z.string().optional(),
|
|
43
|
+
userAgent: z.string().optional(),
|
|
44
|
+
timeoutMs: z.number().int().positive().optional(),
|
|
45
|
+
maxResponseBytes: z.number().int().positive().optional(),
|
|
46
|
+
maxRedirects: z.number().int().nonnegative().optional(),
|
|
47
|
+
assets: z.boolean().optional(),
|
|
48
|
+
useAsync: z.boolean().optional(),
|
|
49
|
+
defuddle: defuddleSchema.optional(),
|
|
50
|
+
frontmatter: frontmatterSchema.optional(),
|
|
51
|
+
hosts: z.record(z.string(), hostConfigSchema).optional()
|
|
52
|
+
})
|
|
53
|
+
.passthrough();
|
|
54
|
+
const KNOWN_TOP_LEVEL = new Set([
|
|
55
|
+
"root",
|
|
56
|
+
"userAgent",
|
|
57
|
+
"timeoutMs",
|
|
58
|
+
"maxResponseBytes",
|
|
59
|
+
"maxRedirects",
|
|
60
|
+
"assets",
|
|
61
|
+
"useAsync",
|
|
62
|
+
"defuddle",
|
|
63
|
+
"frontmatter",
|
|
64
|
+
"hosts"
|
|
65
|
+
]);
|
|
66
|
+
const KNOWN_FRONTMATTER = new Set(["exclude", "values"]);
|
|
67
|
+
const KNOWN_HOST = new Set(["entryQueryKey", "paths"]);
|
|
68
|
+
const KNOWN_PATH = new Set(["entryQueryKey"]);
|
|
69
|
+
const KNOWN_DEFUDDLE = new Set([
|
|
70
|
+
"debug",
|
|
71
|
+
"removeExactSelectors",
|
|
72
|
+
"removePartialSelectors",
|
|
73
|
+
"removeImages",
|
|
74
|
+
"useAsync",
|
|
75
|
+
"removeHiddenElements",
|
|
76
|
+
"removeLowScoring",
|
|
77
|
+
"removeSmallImages",
|
|
78
|
+
"standardize",
|
|
79
|
+
"removeContentPatterns",
|
|
80
|
+
"contentSelector",
|
|
81
|
+
"language",
|
|
82
|
+
"includeReplies",
|
|
83
|
+
"profile"
|
|
84
|
+
]);
|
|
85
|
+
function warnUnknown(object, known, location, warnings) {
|
|
86
|
+
for (const key of Object.keys(object)) {
|
|
87
|
+
if (!known.has(key)) {
|
|
88
|
+
warnings.push({
|
|
89
|
+
code: "UNKNOWN_CONFIG_KEY",
|
|
90
|
+
message: `Unknown configuration key: ${location}${key}`
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
function collectUnknownWarnings(value) {
|
|
96
|
+
const warnings = [];
|
|
97
|
+
warnUnknown(value, KNOWN_TOP_LEVEL, "", warnings);
|
|
98
|
+
const frontmatter = value.frontmatter;
|
|
99
|
+
if (frontmatter && typeof frontmatter === "object" && !Array.isArray(frontmatter)) {
|
|
100
|
+
warnUnknown(frontmatter, KNOWN_FRONTMATTER, "frontmatter.", warnings);
|
|
101
|
+
}
|
|
102
|
+
const defuddle = value.defuddle;
|
|
103
|
+
if (defuddle && typeof defuddle === "object" && !Array.isArray(defuddle)) {
|
|
104
|
+
warnUnknown(defuddle, KNOWN_DEFUDDLE, "defuddle.", warnings);
|
|
105
|
+
}
|
|
106
|
+
const hosts = value.hosts;
|
|
107
|
+
if (hosts && typeof hosts === "object" && !Array.isArray(hosts)) {
|
|
108
|
+
for (const [hostPattern, hostValue] of Object.entries(hosts)) {
|
|
109
|
+
if (!hostValue || typeof hostValue !== "object" || Array.isArray(hostValue)) {
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
const host = hostValue;
|
|
113
|
+
warnUnknown(host, KNOWN_HOST, `hosts.${hostPattern}.`, warnings);
|
|
114
|
+
const paths = host.paths;
|
|
115
|
+
if (paths && typeof paths === "object" && !Array.isArray(paths)) {
|
|
116
|
+
for (const [pathPattern, pathValue] of Object.entries(paths)) {
|
|
117
|
+
if (pathValue && typeof pathValue === "object" && !Array.isArray(pathValue)) {
|
|
118
|
+
warnUnknown(pathValue, KNOWN_PATH, `hosts.${hostPattern}.paths.${pathPattern}.`, warnings);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return warnings;
|
|
125
|
+
}
|
|
126
|
+
export function defaultConfigPath(env = process.env) {
|
|
127
|
+
const configHome = env.XDG_CONFIG_HOME || path.join(os.homedir(), ".config");
|
|
128
|
+
return path.join(configHome, "mdhq", "config.json");
|
|
129
|
+
}
|
|
130
|
+
export function defaultDataRoot(env = process.env) {
|
|
131
|
+
const dataHome = env.XDG_DATA_HOME || path.join(os.homedir(), ".local", "share");
|
|
132
|
+
return path.join(dataHome, "mdhq");
|
|
133
|
+
}
|
|
134
|
+
export function resolveRoot(cliRoot, config, env = process.env) {
|
|
135
|
+
return path.resolve(cliRoot || env.MDHQ_ROOT || config.root || defaultDataRoot(env));
|
|
136
|
+
}
|
|
137
|
+
export async function loadConfig(configPath = defaultConfigPath()) {
|
|
138
|
+
let source;
|
|
139
|
+
try {
|
|
140
|
+
source = await readFile(configPath, "utf8");
|
|
141
|
+
}
|
|
142
|
+
catch (error) {
|
|
143
|
+
if (error.code === "ENOENT") {
|
|
144
|
+
return { config: {}, warnings: [] };
|
|
145
|
+
}
|
|
146
|
+
throw new MdhqError("CONFIG_ERROR", `Failed to read configuration: ${configPath}`, {
|
|
147
|
+
cause: error
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
let raw;
|
|
151
|
+
try {
|
|
152
|
+
raw = JSON.parse(source);
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
throw new MdhqError("CONFIG_ERROR", `Invalid JSON configuration: ${configPath}`, {
|
|
156
|
+
cause: error
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
160
|
+
throw new MdhqError("CONFIG_ERROR", "Configuration must be a JSON object");
|
|
161
|
+
}
|
|
162
|
+
const parsed = configSchema.safeParse(raw);
|
|
163
|
+
if (!parsed.success) {
|
|
164
|
+
throw new MdhqError("CONFIG_ERROR", z.prettifyError(parsed.error));
|
|
165
|
+
}
|
|
166
|
+
return {
|
|
167
|
+
config: parsed.data,
|
|
168
|
+
warnings: collectUnknownWarnings(raw)
|
|
169
|
+
};
|
|
170
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export interface PathConfig {
|
|
2
|
+
entryQueryKey?: string | null;
|
|
3
|
+
}
|
|
4
|
+
export interface HostConfig extends PathConfig {
|
|
5
|
+
paths?: Record<string, PathConfig>;
|
|
6
|
+
}
|
|
7
|
+
export declare function resolveHostConfig(host: string, pathname: string, hosts: Record<string, HostConfig>): PathConfig | undefined;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { domainToASCII } from "node:url";
|
|
2
|
+
import { Minimatch } from "minimatch";
|
|
3
|
+
import { MdhqError } from "../errors.js";
|
|
4
|
+
function literalSpecificity(pattern) {
|
|
5
|
+
let count = 0;
|
|
6
|
+
let skippedUntil;
|
|
7
|
+
for (let index = 0; index < pattern.length; index += 1) {
|
|
8
|
+
const character = pattern[index];
|
|
9
|
+
if (character === undefined) {
|
|
10
|
+
continue;
|
|
11
|
+
}
|
|
12
|
+
if (character === "\\") {
|
|
13
|
+
if (skippedUntil === undefined && index + 1 < pattern.length) {
|
|
14
|
+
count += 1;
|
|
15
|
+
}
|
|
16
|
+
index += 1;
|
|
17
|
+
continue;
|
|
18
|
+
}
|
|
19
|
+
if (skippedUntil) {
|
|
20
|
+
if (character === skippedUntil) {
|
|
21
|
+
skippedUntil = undefined;
|
|
22
|
+
}
|
|
23
|
+
continue;
|
|
24
|
+
}
|
|
25
|
+
if (character === "[") {
|
|
26
|
+
skippedUntil = "]";
|
|
27
|
+
}
|
|
28
|
+
else if (character === "{") {
|
|
29
|
+
skippedUntil = "}";
|
|
30
|
+
}
|
|
31
|
+
else if ("!?+*@".includes(character) && pattern[index + 1] === "(") {
|
|
32
|
+
skippedUntil = ")";
|
|
33
|
+
index += 1;
|
|
34
|
+
}
|
|
35
|
+
else if (character !== "*" && character !== "?") {
|
|
36
|
+
count += 1;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return count;
|
|
40
|
+
}
|
|
41
|
+
function normalizeHostPattern(pattern) {
|
|
42
|
+
const lower = pattern.toLowerCase();
|
|
43
|
+
const portMatch = lower.match(/:(\d+)$/u);
|
|
44
|
+
const port = portMatch?.[1];
|
|
45
|
+
const hostname = portMatch ? lower.slice(0, -portMatch[0].length) : lower;
|
|
46
|
+
const normalized = hostname
|
|
47
|
+
.split(".")
|
|
48
|
+
.map((label) => /[*?[\]{}()!+@]/u.test(label) ? label : domainToASCII(label))
|
|
49
|
+
.join(".");
|
|
50
|
+
return port ? `${normalized}:${port}` : normalized;
|
|
51
|
+
}
|
|
52
|
+
function selectPattern(value, patterns, kind) {
|
|
53
|
+
const entries = Object.entries(patterns).map(([pattern, config]) => ({
|
|
54
|
+
pattern,
|
|
55
|
+
normalizedPattern: kind === "host" ? normalizeHostPattern(pattern) : pattern,
|
|
56
|
+
config
|
|
57
|
+
}));
|
|
58
|
+
const exact = entries.filter((entry) => entry.normalizedPattern === value);
|
|
59
|
+
if (exact.length > 1) {
|
|
60
|
+
throw new MdhqError("CONFIG_ERROR", `Ambiguous normalized ${kind} patterns: ${exact.map((entry) => entry.pattern).join(" and ")}`);
|
|
61
|
+
}
|
|
62
|
+
if (exact[0]) {
|
|
63
|
+
return exact[0].config;
|
|
64
|
+
}
|
|
65
|
+
const matches = entries
|
|
66
|
+
.filter((entry) => new Minimatch(entry.normalizedPattern, { dot: true }).match(value))
|
|
67
|
+
.map((entry) => ({
|
|
68
|
+
pattern: entry.pattern,
|
|
69
|
+
config: entry.config,
|
|
70
|
+
specificity: literalSpecificity(entry.normalizedPattern)
|
|
71
|
+
}))
|
|
72
|
+
.sort((a, b) => b.specificity - a.specificity);
|
|
73
|
+
if (matches.length < 1) {
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
const best = matches[0];
|
|
77
|
+
const ambiguous = matches.find((match, index) => index > 0 && match.specificity === best?.specificity);
|
|
78
|
+
if (ambiguous) {
|
|
79
|
+
throw new MdhqError("CONFIG_ERROR", `Ambiguous ${kind} patterns: ${best?.pattern} and ${ambiguous.pattern}`);
|
|
80
|
+
}
|
|
81
|
+
return best?.config;
|
|
82
|
+
}
|
|
83
|
+
export function resolveHostConfig(host, pathname, hosts) {
|
|
84
|
+
const hostConfig = selectPattern(host, hosts, "host");
|
|
85
|
+
if (!hostConfig) {
|
|
86
|
+
return undefined;
|
|
87
|
+
}
|
|
88
|
+
const pathConfig = hostConfig.paths
|
|
89
|
+
? selectPattern(pathname, hostConfig.paths, "path")
|
|
90
|
+
: undefined;
|
|
91
|
+
if (pathConfig) {
|
|
92
|
+
return pathConfig.entryQueryKey === undefined
|
|
93
|
+
? hostConfig.entryQueryKey === undefined
|
|
94
|
+
? {}
|
|
95
|
+
: { entryQueryKey: hostConfig.entryQueryKey }
|
|
96
|
+
: pathConfig;
|
|
97
|
+
}
|
|
98
|
+
return hostConfig.entryQueryKey === undefined
|
|
99
|
+
? {}
|
|
100
|
+
: { entryQueryKey: hostConfig.entryQueryKey };
|
|
101
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { parseHTML } from "linkedom";
|
|
2
|
+
export type ArticleDocument = ReturnType<typeof parseHTML>["document"];
|
|
3
|
+
export declare function extractArticleDateFromDocument(document: ArticleDocument, options: ArticleDateOptions, pageUrl?: string): string | undefined;
|
|
4
|
+
export interface ArticleDateOptions {
|
|
5
|
+
/** Schema.org JSON-LD property name, e.g. "datePublished" or "dateModified". */
|
|
6
|
+
schemaProperty: string;
|
|
7
|
+
/** `meta[property="..."]` names to check, in priority order. */
|
|
8
|
+
metaProperties: readonly string[];
|
|
9
|
+
/** `itemprop` microdata names to check, in priority order. */
|
|
10
|
+
itemprops: readonly string[];
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Extracts and normalizes a source-article date (published or updated) from
|
|
14
|
+
* Schema.org JSON-LD, Open Graph/article meta tags, and microdata. JSON-LD
|
|
15
|
+
* candidates preserve their original value (string, number, or JSON-LD
|
|
16
|
+
* `@value` object/array) so numeric or JSON-LD forms are not lost before
|
|
17
|
+
* `normalizeSourceDate` can inspect them; malformed JSON-LD blocks are
|
|
18
|
+
* ignored rather than throwing.
|
|
19
|
+
*/
|
|
20
|
+
export declare function extractArticleDate(html: string, options: ArticleDateOptions, pageUrl?: string): string | undefined;
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import { parseHTML } from "linkedom";
|
|
2
|
+
import { normalizeSourceDate } from "../date.js";
|
|
3
|
+
const ARTICLE_LIKE_TYPES = new Set([
|
|
4
|
+
"CreativeWork",
|
|
5
|
+
"APIReference",
|
|
6
|
+
"Report",
|
|
7
|
+
"WebPage"
|
|
8
|
+
]);
|
|
9
|
+
function types(value) {
|
|
10
|
+
if (typeof value === "string") {
|
|
11
|
+
return [value];
|
|
12
|
+
}
|
|
13
|
+
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
14
|
+
}
|
|
15
|
+
function schemaTypeName(value) {
|
|
16
|
+
for (const prefix of ["https://schema.org/", "http://schema.org/"]) {
|
|
17
|
+
if (value.startsWith(prefix)) {
|
|
18
|
+
return value.slice(prefix.length);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
}
|
|
23
|
+
function isArticleLikeType(value) {
|
|
24
|
+
const name = schemaTypeName(value);
|
|
25
|
+
return (ARTICLE_LIKE_TYPES.has(name) ||
|
|
26
|
+
name.endsWith("Article") ||
|
|
27
|
+
name.endsWith("Posting"));
|
|
28
|
+
}
|
|
29
|
+
function isPageType(value) {
|
|
30
|
+
const name = schemaTypeName(value);
|
|
31
|
+
return name === "WebPage" || name.endsWith("Page");
|
|
32
|
+
}
|
|
33
|
+
function collectJsonLdObjects(value, objects, seen = new Set()) {
|
|
34
|
+
if (Array.isArray(value)) {
|
|
35
|
+
for (const item of value) {
|
|
36
|
+
collectJsonLdObjects(item, objects, seen);
|
|
37
|
+
}
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
if (!value || typeof value !== "object") {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
const object = value;
|
|
44
|
+
if (seen.has(object)) {
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
seen.add(object);
|
|
48
|
+
objects.push(object);
|
|
49
|
+
if ("@graph" in object) {
|
|
50
|
+
collectJsonLdObjects(object["@graph"], objects, seen);
|
|
51
|
+
}
|
|
52
|
+
if ("mainEntity" in object) {
|
|
53
|
+
collectJsonLdObjects(object.mainEntity, objects, seen);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function referenceUrls(value, baseUrl) {
|
|
57
|
+
if (Array.isArray(value)) {
|
|
58
|
+
return value.flatMap((item) => referenceUrls(item, baseUrl));
|
|
59
|
+
}
|
|
60
|
+
if (typeof value === "string") {
|
|
61
|
+
try {
|
|
62
|
+
return [new URL(value, baseUrl).href];
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return [];
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
if (!value || typeof value !== "object") {
|
|
69
|
+
return [];
|
|
70
|
+
}
|
|
71
|
+
const object = value;
|
|
72
|
+
return [
|
|
73
|
+
...referenceUrls(object["@id"], baseUrl),
|
|
74
|
+
...referenceUrls(object.url, baseUrl)
|
|
75
|
+
];
|
|
76
|
+
}
|
|
77
|
+
function pageHref(pageUrl) {
|
|
78
|
+
if (!pageUrl) {
|
|
79
|
+
return undefined;
|
|
80
|
+
}
|
|
81
|
+
try {
|
|
82
|
+
const url = new URL(pageUrl);
|
|
83
|
+
url.hash = "";
|
|
84
|
+
return url.href;
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
function directlyMatchesPage(value, page, baseUrl) {
|
|
91
|
+
if (!page) {
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
return referenceUrls(value, baseUrl).some((reference) => {
|
|
95
|
+
const url = new URL(reference);
|
|
96
|
+
return !url.hash && url.href === page;
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
function identifiesPageDocument(value, page, baseUrl) {
|
|
100
|
+
if (!page) {
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
return referenceUrls(value, baseUrl).some((reference) => {
|
|
104
|
+
const url = new URL(reference);
|
|
105
|
+
url.hash = "";
|
|
106
|
+
return url.href === page;
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
function intersects(left, right) {
|
|
110
|
+
return left.some((value) => right.has(value));
|
|
111
|
+
}
|
|
112
|
+
function referencedObjects(value) {
|
|
113
|
+
if (Array.isArray(value)) {
|
|
114
|
+
return value.flatMap(referencedObjects);
|
|
115
|
+
}
|
|
116
|
+
return value && typeof value === "object"
|
|
117
|
+
? [value]
|
|
118
|
+
: [];
|
|
119
|
+
}
|
|
120
|
+
function hasDateCandidate(value) {
|
|
121
|
+
return value !== undefined && value !== null;
|
|
122
|
+
}
|
|
123
|
+
function jsonLdDates(objects, pageUrl, schemaProperty) {
|
|
124
|
+
const page = pageHref(pageUrl);
|
|
125
|
+
const pageEntityIds = new Set(page ? [page] : []);
|
|
126
|
+
const primaryEntityIds = new Set();
|
|
127
|
+
const primaryEntityObjects = new Set();
|
|
128
|
+
for (const object of objects) {
|
|
129
|
+
const pageObject = types(object["@type"]).some(isPageType);
|
|
130
|
+
if ((pageObject && identifiesPageDocument(object["@id"], page, pageUrl)) ||
|
|
131
|
+
directlyMatchesPage(object.url, page, pageUrl)) {
|
|
132
|
+
for (const identifier of referenceUrls(object["@id"], pageUrl)) {
|
|
133
|
+
pageEntityIds.add(identifier);
|
|
134
|
+
}
|
|
135
|
+
for (const identifier of referenceUrls(object.mainEntity, pageUrl)) {
|
|
136
|
+
primaryEntityIds.add(identifier);
|
|
137
|
+
}
|
|
138
|
+
for (const entity of referencedObjects(object.mainEntity)) {
|
|
139
|
+
primaryEntityObjects.add(entity);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return objects
|
|
144
|
+
.flatMap((object, index) => {
|
|
145
|
+
if (!hasDateCandidate(object[schemaProperty]) ||
|
|
146
|
+
!types(object["@type"]).some(isArticleLikeType)) {
|
|
147
|
+
return [];
|
|
148
|
+
}
|
|
149
|
+
const names = types(object["@type"]).map(schemaTypeName);
|
|
150
|
+
const concreteArticle = names.some((name) => name.endsWith("Article") || name.endsWith("Posting"));
|
|
151
|
+
const identifiers = [
|
|
152
|
+
...referenceUrls(object["@id"], pageUrl),
|
|
153
|
+
...referenceUrls(object.url, pageUrl)
|
|
154
|
+
];
|
|
155
|
+
const primary = identifiesPageDocument(object["@id"], page, pageUrl) ||
|
|
156
|
+
directlyMatchesPage(object.url, page, pageUrl) ||
|
|
157
|
+
primaryEntityObjects.has(object) ||
|
|
158
|
+
identifiesPageDocument(object.mainEntityOfPage, page, pageUrl) ||
|
|
159
|
+
intersects(referenceUrls(object.mainEntityOfPage, pageUrl), pageEntityIds) ||
|
|
160
|
+
intersects(identifiers, primaryEntityIds);
|
|
161
|
+
return [
|
|
162
|
+
{
|
|
163
|
+
date: object[schemaProperty],
|
|
164
|
+
score: (primary ? 100 : 0) + (concreteArticle ? 20 : 10),
|
|
165
|
+
index
|
|
166
|
+
}
|
|
167
|
+
];
|
|
168
|
+
})
|
|
169
|
+
.sort((left, right) => right.score - left.score || left.index - right.index)
|
|
170
|
+
.map((candidate) => candidate.date);
|
|
171
|
+
}
|
|
172
|
+
function attributeValue(element, attributes) {
|
|
173
|
+
for (const attribute of attributes) {
|
|
174
|
+
const value = element?.getAttribute(attribute)?.trim();
|
|
175
|
+
if (value) {
|
|
176
|
+
return value;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return undefined;
|
|
180
|
+
}
|
|
181
|
+
export function extractArticleDateFromDocument(document, options, pageUrl) {
|
|
182
|
+
const objects = [];
|
|
183
|
+
for (const script of document.querySelectorAll("script[type]")) {
|
|
184
|
+
const mediaType = script.getAttribute("type")?.split(";", 1)[0]?.trim().toLowerCase() ?? "";
|
|
185
|
+
if (mediaType !== "application/ld+json") {
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
try {
|
|
189
|
+
collectJsonLdObjects(parseJsonLd(script.textContent ?? ""), objects);
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
// Ignore malformed blocks and continue with other metadata sources.
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
const metaCandidates = options.metaProperties.map((property) => attributeValue(document.querySelector(`meta[property="${property}"]`), ["content"]));
|
|
196
|
+
const microdataCandidates = options.itemprops.map((itemprop) => attributeValue(document.querySelector(`[itemprop~="${itemprop}"]`), ["datetime", "content"]));
|
|
197
|
+
for (const candidate of [
|
|
198
|
+
...jsonLdDates(objects, pageUrl, options.schemaProperty),
|
|
199
|
+
...metaCandidates,
|
|
200
|
+
...microdataCandidates
|
|
201
|
+
]) {
|
|
202
|
+
const normalized = normalizeSourceDate(candidate);
|
|
203
|
+
if (normalized) {
|
|
204
|
+
return normalized;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
return undefined;
|
|
208
|
+
}
|
|
209
|
+
function parseJsonLd(source) {
|
|
210
|
+
let result = "";
|
|
211
|
+
let inString = false;
|
|
212
|
+
for (let index = 0; index < source.length; index += 1) {
|
|
213
|
+
const character = source.charAt(index);
|
|
214
|
+
if (character === '"') {
|
|
215
|
+
let backslashes = 0;
|
|
216
|
+
for (let previous = index - 1; previous >= 0 && source[previous] === "\\"; previous -= 1) {
|
|
217
|
+
backslashes += 1;
|
|
218
|
+
}
|
|
219
|
+
if (backslashes % 2 === 0) {
|
|
220
|
+
inString = !inString;
|
|
221
|
+
}
|
|
222
|
+
result += character;
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
if (inString) {
|
|
226
|
+
result += character;
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
if (character === "-" || /\d/u.test(character)) {
|
|
230
|
+
const match = source
|
|
231
|
+
.slice(index)
|
|
232
|
+
.match(/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/u);
|
|
233
|
+
if (match) {
|
|
234
|
+
const token = match[0];
|
|
235
|
+
result += /^-?\d+$/u.test(token) ? `"${token}"` : token;
|
|
236
|
+
index += token.length - 1;
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
result += character;
|
|
241
|
+
}
|
|
242
|
+
return JSON.parse(result);
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Extracts and normalizes a source-article date (published or updated) from
|
|
246
|
+
* Schema.org JSON-LD, Open Graph/article meta tags, and microdata. JSON-LD
|
|
247
|
+
* candidates preserve their original value (string, number, or JSON-LD
|
|
248
|
+
* `@value` object/array) so numeric or JSON-LD forms are not lost before
|
|
249
|
+
* `normalizeSourceDate` can inspect them; malformed JSON-LD blocks are
|
|
250
|
+
* ignored rather than throwing.
|
|
251
|
+
*/
|
|
252
|
+
export function extractArticleDate(html, options, pageUrl) {
|
|
253
|
+
const { document } = parseHTML(html);
|
|
254
|
+
return extractArticleDateFromDocument(document, options, pageUrl);
|
|
255
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { Defuddle } from "defuddle/node";
|
|
2
|
+
import { parseHTML } from "linkedom";
|
|
3
|
+
import { MdhqError } from "../errors.js";
|
|
4
|
+
import { normalizeSourceDate } from "../date.js";
|
|
5
|
+
import { extractPublishedDateFromDocument } from "./extract-published.js";
|
|
6
|
+
import { extractUpdatedDateFromDocument } from "./extract-updated.js";
|
|
7
|
+
function nonempty(value) {
|
|
8
|
+
return value?.trim() ? value.trim() : undefined;
|
|
9
|
+
}
|
|
10
|
+
function dateOnlyEvidence(document, expectedDate) {
|
|
11
|
+
const text = document.body?.textContent ?? "";
|
|
12
|
+
const candidates = [
|
|
13
|
+
...text.matchAll(/\b(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\.?\s+\d{1,2},?\s+\d{4}\b/giu),
|
|
14
|
+
...text.matchAll(/\b\d{1,2}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\.?,?\s+\d{4}\b/giu)
|
|
15
|
+
];
|
|
16
|
+
for (const candidate of candidates) {
|
|
17
|
+
const normalized = normalizeSourceDate(candidate[0]);
|
|
18
|
+
if (normalized === expectedDate) {
|
|
19
|
+
return normalized;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return undefined;
|
|
23
|
+
}
|
|
24
|
+
export async function convertHtml(options) {
|
|
25
|
+
let url;
|
|
26
|
+
try {
|
|
27
|
+
url = options.url instanceof URL ? new URL(options.url.href) : new URL(options.url);
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
throw new MdhqError("INVALID_URL", `Invalid base URL: ${String(options.url)}`, {
|
|
31
|
+
cause: error
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
try {
|
|
35
|
+
const { document } = parseHTML(options.html);
|
|
36
|
+
const updated = extractUpdatedDateFromDocument(document, url.href);
|
|
37
|
+
const publishedFromMetadata = extractPublishedDateFromDocument(document, url.href);
|
|
38
|
+
const result = await Defuddle(options.html, url.href, {
|
|
39
|
+
...options.defuddle,
|
|
40
|
+
markdown: true,
|
|
41
|
+
useAsync: options.defuddle?.useAsync ?? true
|
|
42
|
+
});
|
|
43
|
+
const markdown = result.content?.trim();
|
|
44
|
+
if (!markdown) {
|
|
45
|
+
throw new MdhqError("CONVERSION_FAILED", `Defuddle returned no content for ${url.href}`);
|
|
46
|
+
}
|
|
47
|
+
const publishedFromDefuddle = normalizeSourceDate(nonempty(result.published));
|
|
48
|
+
const synthesizedMidnightDate = publishedFromMetadata === undefined &&
|
|
49
|
+
publishedFromDefuddle?.match(/^\d{4}-\d{2}-\d{2}T00:00:00(?:Z|\+00:00)$/u)
|
|
50
|
+
? publishedFromDefuddle
|
|
51
|
+
: undefined;
|
|
52
|
+
const publishedDateOnlyEvidence = synthesizedMidnightDate === undefined
|
|
53
|
+
? undefined
|
|
54
|
+
: dateOnlyEvidence(document, synthesizedMidnightDate.slice(0, 10));
|
|
55
|
+
const published = publishedFromMetadata ??
|
|
56
|
+
(synthesizedMidnightDate !== undefined &&
|
|
57
|
+
publishedDateOnlyEvidence === synthesizedMidnightDate.slice(0, 10)
|
|
58
|
+
? publishedDateOnlyEvidence
|
|
59
|
+
: publishedFromDefuddle);
|
|
60
|
+
const metadata = {};
|
|
61
|
+
const stringFields = {
|
|
62
|
+
title: nonempty(result.title),
|
|
63
|
+
description: nonempty(result.description),
|
|
64
|
+
author: nonempty(result.author),
|
|
65
|
+
published,
|
|
66
|
+
updated,
|
|
67
|
+
site: nonempty(result.site),
|
|
68
|
+
domain: nonempty(result.domain),
|
|
69
|
+
language: nonempty(result.language),
|
|
70
|
+
image: nonempty(result.image),
|
|
71
|
+
favicon: nonempty(result.favicon)
|
|
72
|
+
};
|
|
73
|
+
for (const [key, value] of Object.entries(stringFields)) {
|
|
74
|
+
if (value !== undefined) {
|
|
75
|
+
metadata[key] = value;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
if (result.wordCount > 0) {
|
|
79
|
+
metadata.wordCount = result.wordCount;
|
|
80
|
+
}
|
|
81
|
+
return { markdown, metadata };
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
if (error instanceof MdhqError) {
|
|
85
|
+
throw error;
|
|
86
|
+
}
|
|
87
|
+
throw new MdhqError("CONVERSION_FAILED", `Failed to convert ${url.href}`, { cause: error });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type ArticleDocument } from "./article-date.js";
|
|
2
|
+
/**
|
|
3
|
+
* Extracts and normalizes the article publication ("published") date from
|
|
4
|
+
* Schema.org `datePublished` JSON-LD, `article:published_time` /
|
|
5
|
+
* `og:published_time` meta tags, or `itemprop="datePublished"` microdata.
|
|
6
|
+
*
|
|
7
|
+
* This is attempted before falling back to Defuddle's own (string-only)
|
|
8
|
+
* `published` extraction, because Defuddle drops numeric or JSON-LD
|
|
9
|
+
* `@value`-shaped `datePublished` values instead of stringifying them.
|
|
10
|
+
*/
|
|
11
|
+
export declare function extractPublishedDate(html: string, pageUrl?: string): string | undefined;
|
|
12
|
+
export declare function extractPublishedDateFromDocument(document: ArticleDocument, pageUrl?: string): string | undefined;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { extractArticleDate, extractArticleDateFromDocument } from "./article-date.js";
|
|
2
|
+
/**
|
|
3
|
+
* Extracts and normalizes the article publication ("published") date from
|
|
4
|
+
* Schema.org `datePublished` JSON-LD, `article:published_time` /
|
|
5
|
+
* `og:published_time` meta tags, or `itemprop="datePublished"` microdata.
|
|
6
|
+
*
|
|
7
|
+
* This is attempted before falling back to Defuddle's own (string-only)
|
|
8
|
+
* `published` extraction, because Defuddle drops numeric or JSON-LD
|
|
9
|
+
* `@value`-shaped `datePublished` values instead of stringifying them.
|
|
10
|
+
*/
|
|
11
|
+
export function extractPublishedDate(html, pageUrl) {
|
|
12
|
+
return extractArticleDate(html, {
|
|
13
|
+
schemaProperty: "datePublished",
|
|
14
|
+
metaProperties: ["article:published_time", "og:published_time"],
|
|
15
|
+
itemprops: ["datePublished"]
|
|
16
|
+
}, pageUrl);
|
|
17
|
+
}
|
|
18
|
+
export function extractPublishedDateFromDocument(document, pageUrl) {
|
|
19
|
+
return extractArticleDateFromDocument(document, {
|
|
20
|
+
schemaProperty: "datePublished",
|
|
21
|
+
metaProperties: ["article:published_time", "og:published_time"],
|
|
22
|
+
itemprops: ["datePublished"]
|
|
23
|
+
}, pageUrl);
|
|
24
|
+
}
|