@songmu/mdhq 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +126 -0
  3. package/dist/assets/localize.d.ts +19 -0
  4. package/dist/assets/localize.js +364 -0
  5. package/dist/cli.d.ts +8 -0
  6. package/dist/cli.js +119 -0
  7. package/dist/config/config.d.ts +25 -0
  8. package/dist/config/config.js +170 -0
  9. package/dist/config/match.d.ts +7 -0
  10. package/dist/config/match.js +101 -0
  11. package/dist/convert/article-date.d.ts +20 -0
  12. package/dist/convert/article-date.js +255 -0
  13. package/dist/convert/convert-html.d.ts +2 -0
  14. package/dist/convert/convert-html.js +89 -0
  15. package/dist/convert/extract-published.d.ts +12 -0
  16. package/dist/convert/extract-published.js +24 -0
  17. package/dist/convert/extract-updated.d.ts +8 -0
  18. package/dist/convert/extract-updated.js +20 -0
  19. package/dist/date.d.ts +18 -0
  20. package/dist/date.js +448 -0
  21. package/dist/errors.d.ts +8 -0
  22. package/dist/errors.js +10 -0
  23. package/dist/frontmatter/frontmatter.d.ts +40 -0
  24. package/dist/frontmatter/frontmatter.js +114 -0
  25. package/dist/get-page.d.ts +2 -0
  26. package/dist/get-page.js +308 -0
  27. package/dist/http/fetch.d.ts +46 -0
  28. package/dist/http/fetch.js +195 -0
  29. package/dist/index.d.ts +6 -0
  30. package/dist/index.js +3 -0
  31. package/dist/list-files.d.ts +8 -0
  32. package/dist/list-files.js +35 -0
  33. package/dist/markdown/transform.d.ts +6 -0
  34. package/dist/markdown/transform.js +129 -0
  35. package/dist/path/storage-path.d.ts +7 -0
  36. package/dist/path/storage-path.js +110 -0
  37. package/dist/storage/atomic.d.ts +8 -0
  38. package/dist/storage/atomic.js +84 -0
  39. package/dist/storage/path-safety.d.ts +1 -0
  40. package/dist/storage/path-safety.js +55 -0
  41. package/dist/storage/save.d.ts +23 -0
  42. package/dist/storage/save.js +118 -0
  43. package/dist/types.d.ts +62 -0
  44. package/dist/types.js +1 -0
  45. package/dist/url/identity.d.ts +12 -0
  46. package/dist/url/identity.js +54 -0
  47. package/dist/url/pathname.d.ts +4 -0
  48. package/dist/url/pathname.js +46 -0
  49. package/dist/version.d.ts +3 -0
  50. package/dist/version.js +6 -0
  51. package/docs/README.md +14 -0
  52. package/docs/configuration.md +242 -0
  53. package/docs/library-api.md +275 -0
  54. package/docs/specification.md +730 -0
  55. package/package.json +73 -0
@@ -0,0 +1,170 @@
1
+ import os from "node:os";
2
+ import path from "node:path";
3
+ import { readFile } from "node:fs/promises";
4
+ import { z } from "zod";
5
+ import { MdhqError } from "../errors.js";
6
+ const pathConfigSchema = z
7
+ .object({
8
+ entryQueryKey: z.string().nullable().optional()
9
+ })
10
+ .passthrough();
11
+ const hostConfigSchema = pathConfigSchema
12
+ .extend({
13
+ paths: z.record(z.string(), pathConfigSchema).optional()
14
+ })
15
+ .passthrough();
16
+ const frontmatterSchema = z
17
+ .object({
18
+ exclude: z.array(z.string()).optional(),
19
+ values: z.record(z.string(), z.union([z.string(), z.number(), z.boolean(), z.null()])).optional()
20
+ })
21
+ .passthrough();
22
+ const defuddleSchema = z
23
+ .object({
24
+ debug: z.boolean().optional(),
25
+ removeExactSelectors: z.boolean().optional(),
26
+ removePartialSelectors: z.boolean().optional(),
27
+ removeImages: z.boolean().optional(),
28
+ useAsync: z.boolean().optional(),
29
+ removeHiddenElements: z.boolean().optional(),
30
+ removeLowScoring: z.boolean().optional(),
31
+ removeSmallImages: z.boolean().optional(),
32
+ standardize: z.boolean().optional(),
33
+ removeContentPatterns: z.boolean().optional(),
34
+ contentSelector: z.string().optional(),
35
+ language: z.string().optional(),
36
+ includeReplies: z.union([z.boolean(), z.literal("extractors")]).optional(),
37
+ profile: z.boolean().optional()
38
+ })
39
+ .passthrough();
40
+ const configSchema = z
41
+ .object({
42
+ root: z.string().optional(),
43
+ userAgent: z.string().optional(),
44
+ timeoutMs: z.number().int().positive().optional(),
45
+ maxResponseBytes: z.number().int().positive().optional(),
46
+ maxRedirects: z.number().int().nonnegative().optional(),
47
+ assets: z.boolean().optional(),
48
+ useAsync: z.boolean().optional(),
49
+ defuddle: defuddleSchema.optional(),
50
+ frontmatter: frontmatterSchema.optional(),
51
+ hosts: z.record(z.string(), hostConfigSchema).optional()
52
+ })
53
+ .passthrough();
54
+ const KNOWN_TOP_LEVEL = new Set([
55
+ "root",
56
+ "userAgent",
57
+ "timeoutMs",
58
+ "maxResponseBytes",
59
+ "maxRedirects",
60
+ "assets",
61
+ "useAsync",
62
+ "defuddle",
63
+ "frontmatter",
64
+ "hosts"
65
+ ]);
66
+ const KNOWN_FRONTMATTER = new Set(["exclude", "values"]);
67
+ const KNOWN_HOST = new Set(["entryQueryKey", "paths"]);
68
+ const KNOWN_PATH = new Set(["entryQueryKey"]);
69
+ const KNOWN_DEFUDDLE = new Set([
70
+ "debug",
71
+ "removeExactSelectors",
72
+ "removePartialSelectors",
73
+ "removeImages",
74
+ "useAsync",
75
+ "removeHiddenElements",
76
+ "removeLowScoring",
77
+ "removeSmallImages",
78
+ "standardize",
79
+ "removeContentPatterns",
80
+ "contentSelector",
81
+ "language",
82
+ "includeReplies",
83
+ "profile"
84
+ ]);
85
+ function warnUnknown(object, known, location, warnings) {
86
+ for (const key of Object.keys(object)) {
87
+ if (!known.has(key)) {
88
+ warnings.push({
89
+ code: "UNKNOWN_CONFIG_KEY",
90
+ message: `Unknown configuration key: ${location}${key}`
91
+ });
92
+ }
93
+ }
94
+ }
95
+ function collectUnknownWarnings(value) {
96
+ const warnings = [];
97
+ warnUnknown(value, KNOWN_TOP_LEVEL, "", warnings);
98
+ const frontmatter = value.frontmatter;
99
+ if (frontmatter && typeof frontmatter === "object" && !Array.isArray(frontmatter)) {
100
+ warnUnknown(frontmatter, KNOWN_FRONTMATTER, "frontmatter.", warnings);
101
+ }
102
+ const defuddle = value.defuddle;
103
+ if (defuddle && typeof defuddle === "object" && !Array.isArray(defuddle)) {
104
+ warnUnknown(defuddle, KNOWN_DEFUDDLE, "defuddle.", warnings);
105
+ }
106
+ const hosts = value.hosts;
107
+ if (hosts && typeof hosts === "object" && !Array.isArray(hosts)) {
108
+ for (const [hostPattern, hostValue] of Object.entries(hosts)) {
109
+ if (!hostValue || typeof hostValue !== "object" || Array.isArray(hostValue)) {
110
+ continue;
111
+ }
112
+ const host = hostValue;
113
+ warnUnknown(host, KNOWN_HOST, `hosts.${hostPattern}.`, warnings);
114
+ const paths = host.paths;
115
+ if (paths && typeof paths === "object" && !Array.isArray(paths)) {
116
+ for (const [pathPattern, pathValue] of Object.entries(paths)) {
117
+ if (pathValue && typeof pathValue === "object" && !Array.isArray(pathValue)) {
118
+ warnUnknown(pathValue, KNOWN_PATH, `hosts.${hostPattern}.paths.${pathPattern}.`, warnings);
119
+ }
120
+ }
121
+ }
122
+ }
123
+ }
124
+ return warnings;
125
+ }
126
+ export function defaultConfigPath(env = process.env) {
127
+ const configHome = env.XDG_CONFIG_HOME || path.join(os.homedir(), ".config");
128
+ return path.join(configHome, "mdhq", "config.json");
129
+ }
130
+ export function defaultDataRoot(env = process.env) {
131
+ const dataHome = env.XDG_DATA_HOME || path.join(os.homedir(), ".local", "share");
132
+ return path.join(dataHome, "mdhq");
133
+ }
134
+ export function resolveRoot(cliRoot, config, env = process.env) {
135
+ return path.resolve(cliRoot || env.MDHQ_ROOT || config.root || defaultDataRoot(env));
136
+ }
137
+ export async function loadConfig(configPath = defaultConfigPath()) {
138
+ let source;
139
+ try {
140
+ source = await readFile(configPath, "utf8");
141
+ }
142
+ catch (error) {
143
+ if (error.code === "ENOENT") {
144
+ return { config: {}, warnings: [] };
145
+ }
146
+ throw new MdhqError("CONFIG_ERROR", `Failed to read configuration: ${configPath}`, {
147
+ cause: error
148
+ });
149
+ }
150
+ let raw;
151
+ try {
152
+ raw = JSON.parse(source);
153
+ }
154
+ catch (error) {
155
+ throw new MdhqError("CONFIG_ERROR", `Invalid JSON configuration: ${configPath}`, {
156
+ cause: error
157
+ });
158
+ }
159
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
160
+ throw new MdhqError("CONFIG_ERROR", "Configuration must be a JSON object");
161
+ }
162
+ const parsed = configSchema.safeParse(raw);
163
+ if (!parsed.success) {
164
+ throw new MdhqError("CONFIG_ERROR", z.prettifyError(parsed.error));
165
+ }
166
+ return {
167
+ config: parsed.data,
168
+ warnings: collectUnknownWarnings(raw)
169
+ };
170
+ }
@@ -0,0 +1,7 @@
1
+ export interface PathConfig {
2
+ entryQueryKey?: string | null;
3
+ }
4
+ export interface HostConfig extends PathConfig {
5
+ paths?: Record<string, PathConfig>;
6
+ }
7
+ export declare function resolveHostConfig(host: string, pathname: string, hosts: Record<string, HostConfig>): PathConfig | undefined;
@@ -0,0 +1,101 @@
1
+ import { domainToASCII } from "node:url";
2
+ import { Minimatch } from "minimatch";
3
+ import { MdhqError } from "../errors.js";
4
+ function literalSpecificity(pattern) {
5
+ let count = 0;
6
+ let skippedUntil;
7
+ for (let index = 0; index < pattern.length; index += 1) {
8
+ const character = pattern[index];
9
+ if (character === undefined) {
10
+ continue;
11
+ }
12
+ if (character === "\\") {
13
+ if (skippedUntil === undefined && index + 1 < pattern.length) {
14
+ count += 1;
15
+ }
16
+ index += 1;
17
+ continue;
18
+ }
19
+ if (skippedUntil) {
20
+ if (character === skippedUntil) {
21
+ skippedUntil = undefined;
22
+ }
23
+ continue;
24
+ }
25
+ if (character === "[") {
26
+ skippedUntil = "]";
27
+ }
28
+ else if (character === "{") {
29
+ skippedUntil = "}";
30
+ }
31
+ else if ("!?+*@".includes(character) && pattern[index + 1] === "(") {
32
+ skippedUntil = ")";
33
+ index += 1;
34
+ }
35
+ else if (character !== "*" && character !== "?") {
36
+ count += 1;
37
+ }
38
+ }
39
+ return count;
40
+ }
41
+ function normalizeHostPattern(pattern) {
42
+ const lower = pattern.toLowerCase();
43
+ const portMatch = lower.match(/:(\d+)$/u);
44
+ const port = portMatch?.[1];
45
+ const hostname = portMatch ? lower.slice(0, -portMatch[0].length) : lower;
46
+ const normalized = hostname
47
+ .split(".")
48
+ .map((label) => /[*?[\]{}()!+@]/u.test(label) ? label : domainToASCII(label))
49
+ .join(".");
50
+ return port ? `${normalized}:${port}` : normalized;
51
+ }
52
+ function selectPattern(value, patterns, kind) {
53
+ const entries = Object.entries(patterns).map(([pattern, config]) => ({
54
+ pattern,
55
+ normalizedPattern: kind === "host" ? normalizeHostPattern(pattern) : pattern,
56
+ config
57
+ }));
58
+ const exact = entries.filter((entry) => entry.normalizedPattern === value);
59
+ if (exact.length > 1) {
60
+ throw new MdhqError("CONFIG_ERROR", `Ambiguous normalized ${kind} patterns: ${exact.map((entry) => entry.pattern).join(" and ")}`);
61
+ }
62
+ if (exact[0]) {
63
+ return exact[0].config;
64
+ }
65
+ const matches = entries
66
+ .filter((entry) => new Minimatch(entry.normalizedPattern, { dot: true }).match(value))
67
+ .map((entry) => ({
68
+ pattern: entry.pattern,
69
+ config: entry.config,
70
+ specificity: literalSpecificity(entry.normalizedPattern)
71
+ }))
72
+ .sort((a, b) => b.specificity - a.specificity);
73
+ if (matches.length < 1) {
74
+ return undefined;
75
+ }
76
+ const best = matches[0];
77
+ const ambiguous = matches.find((match, index) => index > 0 && match.specificity === best?.specificity);
78
+ if (ambiguous) {
79
+ throw new MdhqError("CONFIG_ERROR", `Ambiguous ${kind} patterns: ${best?.pattern} and ${ambiguous.pattern}`);
80
+ }
81
+ return best?.config;
82
+ }
83
+ export function resolveHostConfig(host, pathname, hosts) {
84
+ const hostConfig = selectPattern(host, hosts, "host");
85
+ if (!hostConfig) {
86
+ return undefined;
87
+ }
88
+ const pathConfig = hostConfig.paths
89
+ ? selectPattern(pathname, hostConfig.paths, "path")
90
+ : undefined;
91
+ if (pathConfig) {
92
+ return pathConfig.entryQueryKey === undefined
93
+ ? hostConfig.entryQueryKey === undefined
94
+ ? {}
95
+ : { entryQueryKey: hostConfig.entryQueryKey }
96
+ : pathConfig;
97
+ }
98
+ return hostConfig.entryQueryKey === undefined
99
+ ? {}
100
+ : { entryQueryKey: hostConfig.entryQueryKey };
101
+ }
@@ -0,0 +1,20 @@
1
+ import { parseHTML } from "linkedom";
2
+ export type ArticleDocument = ReturnType<typeof parseHTML>["document"];
3
+ export declare function extractArticleDateFromDocument(document: ArticleDocument, options: ArticleDateOptions, pageUrl?: string): string | undefined;
4
+ export interface ArticleDateOptions {
5
+ /** Schema.org JSON-LD property name, e.g. "datePublished" or "dateModified". */
6
+ schemaProperty: string;
7
+ /** `meta[property="..."]` names to check, in priority order. */
8
+ metaProperties: readonly string[];
9
+ /** `itemprop` microdata names to check, in priority order. */
10
+ itemprops: readonly string[];
11
+ }
12
+ /**
13
+ * Extracts and normalizes a source-article date (published or updated) from
14
+ * Schema.org JSON-LD, Open Graph/article meta tags, and microdata. JSON-LD
15
+ * candidates preserve their original value (string, number, or JSON-LD
16
+ * `@value` object/array) so numeric or JSON-LD forms are not lost before
17
+ * `normalizeSourceDate` can inspect them; malformed JSON-LD blocks are
18
+ * ignored rather than throwing.
19
+ */
20
+ export declare function extractArticleDate(html: string, options: ArticleDateOptions, pageUrl?: string): string | undefined;
@@ -0,0 +1,255 @@
1
+ import { parseHTML } from "linkedom";
2
+ import { normalizeSourceDate } from "../date.js";
3
+ const ARTICLE_LIKE_TYPES = new Set([
4
+ "CreativeWork",
5
+ "APIReference",
6
+ "Report",
7
+ "WebPage"
8
+ ]);
9
+ function types(value) {
10
+ if (typeof value === "string") {
11
+ return [value];
12
+ }
13
+ return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
14
+ }
15
+ function schemaTypeName(value) {
16
+ for (const prefix of ["https://schema.org/", "http://schema.org/"]) {
17
+ if (value.startsWith(prefix)) {
18
+ return value.slice(prefix.length);
19
+ }
20
+ }
21
+ return value;
22
+ }
23
+ function isArticleLikeType(value) {
24
+ const name = schemaTypeName(value);
25
+ return (ARTICLE_LIKE_TYPES.has(name) ||
26
+ name.endsWith("Article") ||
27
+ name.endsWith("Posting"));
28
+ }
29
+ function isPageType(value) {
30
+ const name = schemaTypeName(value);
31
+ return name === "WebPage" || name.endsWith("Page");
32
+ }
33
+ function collectJsonLdObjects(value, objects, seen = new Set()) {
34
+ if (Array.isArray(value)) {
35
+ for (const item of value) {
36
+ collectJsonLdObjects(item, objects, seen);
37
+ }
38
+ return;
39
+ }
40
+ if (!value || typeof value !== "object") {
41
+ return;
42
+ }
43
+ const object = value;
44
+ if (seen.has(object)) {
45
+ return;
46
+ }
47
+ seen.add(object);
48
+ objects.push(object);
49
+ if ("@graph" in object) {
50
+ collectJsonLdObjects(object["@graph"], objects, seen);
51
+ }
52
+ if ("mainEntity" in object) {
53
+ collectJsonLdObjects(object.mainEntity, objects, seen);
54
+ }
55
+ }
56
+ function referenceUrls(value, baseUrl) {
57
+ if (Array.isArray(value)) {
58
+ return value.flatMap((item) => referenceUrls(item, baseUrl));
59
+ }
60
+ if (typeof value === "string") {
61
+ try {
62
+ return [new URL(value, baseUrl).href];
63
+ }
64
+ catch {
65
+ return [];
66
+ }
67
+ }
68
+ if (!value || typeof value !== "object") {
69
+ return [];
70
+ }
71
+ const object = value;
72
+ return [
73
+ ...referenceUrls(object["@id"], baseUrl),
74
+ ...referenceUrls(object.url, baseUrl)
75
+ ];
76
+ }
77
+ function pageHref(pageUrl) {
78
+ if (!pageUrl) {
79
+ return undefined;
80
+ }
81
+ try {
82
+ const url = new URL(pageUrl);
83
+ url.hash = "";
84
+ return url.href;
85
+ }
86
+ catch {
87
+ return undefined;
88
+ }
89
+ }
90
+ function directlyMatchesPage(value, page, baseUrl) {
91
+ if (!page) {
92
+ return false;
93
+ }
94
+ return referenceUrls(value, baseUrl).some((reference) => {
95
+ const url = new URL(reference);
96
+ return !url.hash && url.href === page;
97
+ });
98
+ }
99
+ function identifiesPageDocument(value, page, baseUrl) {
100
+ if (!page) {
101
+ return false;
102
+ }
103
+ return referenceUrls(value, baseUrl).some((reference) => {
104
+ const url = new URL(reference);
105
+ url.hash = "";
106
+ return url.href === page;
107
+ });
108
+ }
109
+ function intersects(left, right) {
110
+ return left.some((value) => right.has(value));
111
+ }
112
+ function referencedObjects(value) {
113
+ if (Array.isArray(value)) {
114
+ return value.flatMap(referencedObjects);
115
+ }
116
+ return value && typeof value === "object"
117
+ ? [value]
118
+ : [];
119
+ }
120
+ function hasDateCandidate(value) {
121
+ return value !== undefined && value !== null;
122
+ }
123
+ function jsonLdDates(objects, pageUrl, schemaProperty) {
124
+ const page = pageHref(pageUrl);
125
+ const pageEntityIds = new Set(page ? [page] : []);
126
+ const primaryEntityIds = new Set();
127
+ const primaryEntityObjects = new Set();
128
+ for (const object of objects) {
129
+ const pageObject = types(object["@type"]).some(isPageType);
130
+ if ((pageObject && identifiesPageDocument(object["@id"], page, pageUrl)) ||
131
+ directlyMatchesPage(object.url, page, pageUrl)) {
132
+ for (const identifier of referenceUrls(object["@id"], pageUrl)) {
133
+ pageEntityIds.add(identifier);
134
+ }
135
+ for (const identifier of referenceUrls(object.mainEntity, pageUrl)) {
136
+ primaryEntityIds.add(identifier);
137
+ }
138
+ for (const entity of referencedObjects(object.mainEntity)) {
139
+ primaryEntityObjects.add(entity);
140
+ }
141
+ }
142
+ }
143
+ return objects
144
+ .flatMap((object, index) => {
145
+ if (!hasDateCandidate(object[schemaProperty]) ||
146
+ !types(object["@type"]).some(isArticleLikeType)) {
147
+ return [];
148
+ }
149
+ const names = types(object["@type"]).map(schemaTypeName);
150
+ const concreteArticle = names.some((name) => name.endsWith("Article") || name.endsWith("Posting"));
151
+ const identifiers = [
152
+ ...referenceUrls(object["@id"], pageUrl),
153
+ ...referenceUrls(object.url, pageUrl)
154
+ ];
155
+ const primary = identifiesPageDocument(object["@id"], page, pageUrl) ||
156
+ directlyMatchesPage(object.url, page, pageUrl) ||
157
+ primaryEntityObjects.has(object) ||
158
+ identifiesPageDocument(object.mainEntityOfPage, page, pageUrl) ||
159
+ intersects(referenceUrls(object.mainEntityOfPage, pageUrl), pageEntityIds) ||
160
+ intersects(identifiers, primaryEntityIds);
161
+ return [
162
+ {
163
+ date: object[schemaProperty],
164
+ score: (primary ? 100 : 0) + (concreteArticle ? 20 : 10),
165
+ index
166
+ }
167
+ ];
168
+ })
169
+ .sort((left, right) => right.score - left.score || left.index - right.index)
170
+ .map((candidate) => candidate.date);
171
+ }
172
+ function attributeValue(element, attributes) {
173
+ for (const attribute of attributes) {
174
+ const value = element?.getAttribute(attribute)?.trim();
175
+ if (value) {
176
+ return value;
177
+ }
178
+ }
179
+ return undefined;
180
+ }
181
+ export function extractArticleDateFromDocument(document, options, pageUrl) {
182
+ const objects = [];
183
+ for (const script of document.querySelectorAll("script[type]")) {
184
+ const mediaType = script.getAttribute("type")?.split(";", 1)[0]?.trim().toLowerCase() ?? "";
185
+ if (mediaType !== "application/ld+json") {
186
+ continue;
187
+ }
188
+ try {
189
+ collectJsonLdObjects(parseJsonLd(script.textContent ?? ""), objects);
190
+ }
191
+ catch {
192
+ // Ignore malformed blocks and continue with other metadata sources.
193
+ }
194
+ }
195
+ const metaCandidates = options.metaProperties.map((property) => attributeValue(document.querySelector(`meta[property="${property}"]`), ["content"]));
196
+ const microdataCandidates = options.itemprops.map((itemprop) => attributeValue(document.querySelector(`[itemprop~="${itemprop}"]`), ["datetime", "content"]));
197
+ for (const candidate of [
198
+ ...jsonLdDates(objects, pageUrl, options.schemaProperty),
199
+ ...metaCandidates,
200
+ ...microdataCandidates
201
+ ]) {
202
+ const normalized = normalizeSourceDate(candidate);
203
+ if (normalized) {
204
+ return normalized;
205
+ }
206
+ }
207
+ return undefined;
208
+ }
209
+ function parseJsonLd(source) {
210
+ let result = "";
211
+ let inString = false;
212
+ for (let index = 0; index < source.length; index += 1) {
213
+ const character = source.charAt(index);
214
+ if (character === '"') {
215
+ let backslashes = 0;
216
+ for (let previous = index - 1; previous >= 0 && source[previous] === "\\"; previous -= 1) {
217
+ backslashes += 1;
218
+ }
219
+ if (backslashes % 2 === 0) {
220
+ inString = !inString;
221
+ }
222
+ result += character;
223
+ continue;
224
+ }
225
+ if (inString) {
226
+ result += character;
227
+ continue;
228
+ }
229
+ if (character === "-" || /\d/u.test(character)) {
230
+ const match = source
231
+ .slice(index)
232
+ .match(/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/u);
233
+ if (match) {
234
+ const token = match[0];
235
+ result += /^-?\d+$/u.test(token) ? `"${token}"` : token;
236
+ index += token.length - 1;
237
+ continue;
238
+ }
239
+ }
240
+ result += character;
241
+ }
242
+ return JSON.parse(result);
243
+ }
244
+ /**
245
+ * Extracts and normalizes a source-article date (published or updated) from
246
+ * Schema.org JSON-LD, Open Graph/article meta tags, and microdata. JSON-LD
247
+ * candidates preserve their original value (string, number, or JSON-LD
248
+ * `@value` object/array) so numeric or JSON-LD forms are not lost before
249
+ * `normalizeSourceDate` can inspect them; malformed JSON-LD blocks are
250
+ * ignored rather than throwing.
251
+ */
252
+ export function extractArticleDate(html, options, pageUrl) {
253
+ const { document } = parseHTML(html);
254
+ return extractArticleDateFromDocument(document, options, pageUrl);
255
+ }
@@ -0,0 +1,2 @@
1
+ import type { ConvertedPage, ConvertHtmlOptions } from "../types.js";
2
+ export declare function convertHtml(options: ConvertHtmlOptions): Promise<ConvertedPage>;
@@ -0,0 +1,89 @@
1
+ import { Defuddle } from "defuddle/node";
2
+ import { parseHTML } from "linkedom";
3
+ import { MdhqError } from "../errors.js";
4
+ import { normalizeSourceDate } from "../date.js";
5
+ import { extractPublishedDateFromDocument } from "./extract-published.js";
6
+ import { extractUpdatedDateFromDocument } from "./extract-updated.js";
7
+ function nonempty(value) {
8
+ return value?.trim() ? value.trim() : undefined;
9
+ }
10
+ function dateOnlyEvidence(document, expectedDate) {
11
+ const text = document.body?.textContent ?? "";
12
+ const candidates = [
13
+ ...text.matchAll(/\b(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\.?\s+\d{1,2},?\s+\d{4}\b/giu),
14
+ ...text.matchAll(/\b\d{1,2}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\.?,?\s+\d{4}\b/giu)
15
+ ];
16
+ for (const candidate of candidates) {
17
+ const normalized = normalizeSourceDate(candidate[0]);
18
+ if (normalized === expectedDate) {
19
+ return normalized;
20
+ }
21
+ }
22
+ return undefined;
23
+ }
24
+ export async function convertHtml(options) {
25
+ let url;
26
+ try {
27
+ url = options.url instanceof URL ? new URL(options.url.href) : new URL(options.url);
28
+ }
29
+ catch (error) {
30
+ throw new MdhqError("INVALID_URL", `Invalid base URL: ${String(options.url)}`, {
31
+ cause: error
32
+ });
33
+ }
34
+ try {
35
+ const { document } = parseHTML(options.html);
36
+ const updated = extractUpdatedDateFromDocument(document, url.href);
37
+ const publishedFromMetadata = extractPublishedDateFromDocument(document, url.href);
38
+ const result = await Defuddle(options.html, url.href, {
39
+ ...options.defuddle,
40
+ markdown: true,
41
+ useAsync: options.defuddle?.useAsync ?? true
42
+ });
43
+ const markdown = result.content?.trim();
44
+ if (!markdown) {
45
+ throw new MdhqError("CONVERSION_FAILED", `Defuddle returned no content for ${url.href}`);
46
+ }
47
+ const publishedFromDefuddle = normalizeSourceDate(nonempty(result.published));
48
+ const synthesizedMidnightDate = publishedFromMetadata === undefined &&
49
+ publishedFromDefuddle?.match(/^\d{4}-\d{2}-\d{2}T00:00:00(?:Z|\+00:00)$/u)
50
+ ? publishedFromDefuddle
51
+ : undefined;
52
+ const publishedDateOnlyEvidence = synthesizedMidnightDate === undefined
53
+ ? undefined
54
+ : dateOnlyEvidence(document, synthesizedMidnightDate.slice(0, 10));
55
+ const published = publishedFromMetadata ??
56
+ (synthesizedMidnightDate !== undefined &&
57
+ publishedDateOnlyEvidence === synthesizedMidnightDate.slice(0, 10)
58
+ ? publishedDateOnlyEvidence
59
+ : publishedFromDefuddle);
60
+ const metadata = {};
61
+ const stringFields = {
62
+ title: nonempty(result.title),
63
+ description: nonempty(result.description),
64
+ author: nonempty(result.author),
65
+ published,
66
+ updated,
67
+ site: nonempty(result.site),
68
+ domain: nonempty(result.domain),
69
+ language: nonempty(result.language),
70
+ image: nonempty(result.image),
71
+ favicon: nonempty(result.favicon)
72
+ };
73
+ for (const [key, value] of Object.entries(stringFields)) {
74
+ if (value !== undefined) {
75
+ metadata[key] = value;
76
+ }
77
+ }
78
+ if (result.wordCount > 0) {
79
+ metadata.wordCount = result.wordCount;
80
+ }
81
+ return { markdown, metadata };
82
+ }
83
+ catch (error) {
84
+ if (error instanceof MdhqError) {
85
+ throw error;
86
+ }
87
+ throw new MdhqError("CONVERSION_FAILED", `Failed to convert ${url.href}`, { cause: error });
88
+ }
89
+ }
@@ -0,0 +1,12 @@
1
+ import { type ArticleDocument } from "./article-date.js";
2
+ /**
3
+ * Extracts and normalizes the article publication ("published") date from
4
+ * Schema.org `datePublished` JSON-LD, `article:published_time` /
5
+ * `og:published_time` meta tags, or `itemprop="datePublished"` microdata.
6
+ *
7
+ * This is attempted before falling back to Defuddle's own (string-only)
8
+ * `published` extraction, because Defuddle drops numeric or JSON-LD
9
+ * `@value`-shaped `datePublished` values instead of stringifying them.
10
+ */
11
+ export declare function extractPublishedDate(html: string, pageUrl?: string): string | undefined;
12
+ export declare function extractPublishedDateFromDocument(document: ArticleDocument, pageUrl?: string): string | undefined;
@@ -0,0 +1,24 @@
1
+ import { extractArticleDate, extractArticleDateFromDocument } from "./article-date.js";
2
+ /**
3
+ * Extracts and normalizes the article publication ("published") date from
4
+ * Schema.org `datePublished` JSON-LD, `article:published_time` /
5
+ * `og:published_time` meta tags, or `itemprop="datePublished"` microdata.
6
+ *
7
+ * This is attempted before falling back to Defuddle's own (string-only)
8
+ * `published` extraction, because Defuddle drops numeric or JSON-LD
9
+ * `@value`-shaped `datePublished` values instead of stringifying them.
10
+ */
11
+ export function extractPublishedDate(html, pageUrl) {
12
+ return extractArticleDate(html, {
13
+ schemaProperty: "datePublished",
14
+ metaProperties: ["article:published_time", "og:published_time"],
15
+ itemprops: ["datePublished"]
16
+ }, pageUrl);
17
+ }
18
+ export function extractPublishedDateFromDocument(document, pageUrl) {
19
+ return extractArticleDateFromDocument(document, {
20
+ schemaProperty: "datePublished",
21
+ metaProperties: ["article:published_time", "og:published_time"],
22
+ itemprops: ["datePublished"]
23
+ }, pageUrl);
24
+ }