@khorsheed/dsh-reader 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +90 -0
- package/README.i18n.yaml +6 -0
- package/README.md +90 -0
- package/cordis.patch.yml +13 -0
- package/lib/client.js +15119 -0
- package/lib/index.js +3660 -0
- package/lib/invariant.js +38 -0
- package/lib/typert.host.d.ts +3 -0
- package/lib/typert.host.js +1738 -0
- package/lib/typert.remote-client.d.ts +82 -0
- package/lib/typert.remote-client.js +1115 -0
- package/lib/types/arxiv.d.ts +47 -0
- package/lib/types/arxiv.js +76 -0
- package/lib/types/client/ReaderPane.d.ts +71 -0
- package/lib/types/client/ReaderPane.js +3135 -0
- package/lib/types/client/contract.d.ts +254 -0
- package/lib/types/client/contract.js +1 -0
- package/lib/types/client/definition.d.ts +27 -0
- package/lib/types/client/definition.js +32 -0
- package/lib/types/client/extract-article.d.ts +102 -0
- package/lib/types/client/extract-article.js +1024 -0
- package/lib/types/client/icons.d.ts +27 -0
- package/lib/types/client/icons.js +11 -0
- package/lib/types/client/index.d.ts +34 -0
- package/lib/types/client/index.js +222 -0
- package/lib/types/client/locales.d.ts +391 -0
- package/lib/types/client/locales.js +382 -0
- package/lib/types/client/parse-rss.d.ts +69 -0
- package/lib/types/client/parse-rss.js +656 -0
- package/lib/types/client/quote.d.ts +89 -0
- package/lib/types/client/quote.js +119 -0
- package/lib/types/client/selectors.d.ts +167 -0
- package/lib/types/client/selectors.js +413 -0
- package/lib/types/client/session.d.ts +171 -0
- package/lib/types/client/session.js +223 -0
- package/lib/types/client/store.d.ts +181 -0
- package/lib/types/client/store.js +226 -0
- package/lib/types/client/translate.d.ts +379 -0
- package/lib/types/client/translate.js +873 -0
- package/lib/types/index.d.ts +48 -0
- package/lib/types/index.js +25 -0
- package/lib/types/invariant.d.ts +29 -0
- package/lib/types/invariant.js +38 -0
- package/lib/types/link-resolvers.d.ts +107 -0
- package/lib/types/link-resolvers.js +299 -0
- package/lib/types/remote.d.ts +224 -0
- package/lib/types/remote.js +275 -0
- package/lib/types/schedule.d.ts +78 -0
- package/lib/types/schedule.js +114 -0
- package/lib/types/service.d.ts +633 -0
- package/lib/types/service.js +1697 -0
- package/lib/types/store.d.ts +224 -0
- package/lib/types/store.js +916 -0
- package/lib/types/types.d.ts +710 -0
- package/lib/types/types.js +182 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +128 -0
|
@@ -0,0 +1,656 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Feed parsing for the browser half: RSS 2.0 / Atom → {@link ReaderEntry}.
|
|
3
|
+
*
|
|
4
|
+
* The host half never parses (its runtime has no XML parser at all), so this
|
|
5
|
+
* module is the single owner of the feed vocabulary. It runs against the raw
|
|
6
|
+
* payload the host already stored, which means re-reading a feed costs no
|
|
7
|
+
* network call.
|
|
8
|
+
*
|
|
9
|
+
* Namespaces are matched by LOCAL name, never by prefix: publishers migrated
|
|
10
|
+
* from `content:encoded` to `media:description` to a bare `content` across RSS
|
|
11
|
+
* and Atom without ever agreeing on a prefix, and matching the local name is
|
|
12
|
+
* what lets one reader handle all of them.
|
|
13
|
+
*
|
|
14
|
+
* @module @khorsheed/dsh-reader/client/parse-rss
|
|
15
|
+
*/
|
|
16
|
+
import { stableEntryId, summarize, } from "../types.js";
|
|
17
|
+
import { normalizeRichText } from "./extract-article.js";
|
|
18
|
+
/** Element local-names that may carry an entry's full body, best first. */
|
|
19
|
+
const CONTENT_TAGS = ['encoded', 'content'];
|
|
20
|
+
/** Element local-names that may carry a short description, best first. */
|
|
21
|
+
const SUMMARY_TAGS = ['description', 'summary'];
|
|
22
|
+
/** Date element local-names, best first. */
|
|
23
|
+
const DATE_TAGS = ['pubDate', 'published', 'updated', 'date'];
|
|
24
|
+
/**
|
|
25
|
+
* Parse one feed payload.
|
|
26
|
+
*
|
|
27
|
+
* A malformed document, a document cut off mid-tag by a size cap, and an HTML
|
|
28
|
+
* error page all arrive here as "not a feed" and are reported as such, never
|
|
29
|
+
* thrown: the caller decides whether to surface or retry.
|
|
30
|
+
*
|
|
31
|
+
* @param raw - the raw payload the host stored for an RSS source.
|
|
32
|
+
* @param sourceId - the source these entries belong to.
|
|
33
|
+
* @returns the parsed feed, or the reason it could not be parsed.
|
|
34
|
+
*/
|
|
35
|
+
export function parseFeed(raw, sourceId) {
|
|
36
|
+
const trimmed = raw.trim();
|
|
37
|
+
if (trimmed.length === 0)
|
|
38
|
+
return { ok: false, error: 'empty payload' };
|
|
39
|
+
let doc;
|
|
40
|
+
try {
|
|
41
|
+
doc = new DOMParser().parseFromString(trimmed, 'application/xml');
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
return { ok: false, error: `not parseable as XML: ${errorMessage(error)}` };
|
|
45
|
+
}
|
|
46
|
+
// A `parsererror` element is how DOMParser reports every malformed document —
|
|
47
|
+
// and the one a reader meets most often is not exotic at all: the host seam
|
|
48
|
+
// caps a payload at ~100,000 characters, and a cap that lands mid-tag makes
|
|
49
|
+
// the WHOLE document malformed. (Measured on the acceptance instance: a
|
|
50
|
+
// 646,905-character feed arrived as exactly 100,000 characters with one
|
|
51
|
+
// complete `<item>` and one cut in half — every entry was lost to one missing
|
|
52
|
+
// `</rss>`.)
|
|
53
|
+
//
|
|
54
|
+
// So a malformed document is not the end: what closed before the cut is
|
|
55
|
+
// still perfectly good XML, and it is recovered below.
|
|
56
|
+
const parseError = doc.querySelector('parsererror');
|
|
57
|
+
if (parseError !== null) {
|
|
58
|
+
const salvage = salvageDocument(trimmed, sourceId, `malformed XML: ${truncate(collapse(parseError.textContent ?? ''), 160)}`);
|
|
59
|
+
if (salvage !== undefined)
|
|
60
|
+
return salvage;
|
|
61
|
+
return { ok: false, error: `malformed XML: ${truncate(collapse(parseError.textContent ?? ''), 160)}` };
|
|
62
|
+
}
|
|
63
|
+
const root = doc.documentElement;
|
|
64
|
+
if (root === null)
|
|
65
|
+
return { ok: false, error: 'no document element' };
|
|
66
|
+
const rootName = root.localName.toLowerCase();
|
|
67
|
+
if (rootName === 'rss' || rootName === 'rdf')
|
|
68
|
+
return parseRssDocument(doc, sourceId);
|
|
69
|
+
if (rootName === 'feed')
|
|
70
|
+
return parseAtomDocument(doc, sourceId);
|
|
71
|
+
return { ok: false, error: `unsupported root element <${rootName}>` };
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Parse an RSS 2.0 (or RSS 1.0 / RDF) document.
|
|
75
|
+
* @param doc - the parsed document.
|
|
76
|
+
* @param sourceId - owning source id.
|
|
77
|
+
* @returns the feed result.
|
|
78
|
+
*/
|
|
79
|
+
function parseRssDocument(doc, sourceId) {
|
|
80
|
+
const channel = doc.querySelector('channel') ?? doc.documentElement;
|
|
81
|
+
const entries = [];
|
|
82
|
+
for (const item of Array.from(doc.querySelectorAll('item'))) {
|
|
83
|
+
const entry = readEntry(item, sourceId);
|
|
84
|
+
if (entry !== undefined)
|
|
85
|
+
entries.push(entry);
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
ok: true,
|
|
89
|
+
feed: {
|
|
90
|
+
...present('title', elementText(child(channel, 'title'))),
|
|
91
|
+
...present('icon', channelIcon(channel)),
|
|
92
|
+
entries,
|
|
93
|
+
},
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Parse an Atom document.
|
|
98
|
+
* @param doc - the parsed document.
|
|
99
|
+
* @param sourceId - owning source id.
|
|
100
|
+
* @returns the feed result.
|
|
101
|
+
*/
|
|
102
|
+
function parseAtomDocument(doc, sourceId) {
|
|
103
|
+
const feed = doc.documentElement;
|
|
104
|
+
const entries = [];
|
|
105
|
+
for (const node of Array.from(feed.getElementsByTagName('entry'))) {
|
|
106
|
+
const entry = readEntry(node, sourceId);
|
|
107
|
+
if (entry !== undefined)
|
|
108
|
+
entries.push(entry);
|
|
109
|
+
}
|
|
110
|
+
return {
|
|
111
|
+
ok: true,
|
|
112
|
+
feed: {
|
|
113
|
+
...present('title', elementText(child(feed, 'title'))),
|
|
114
|
+
...present('icon', elementText(child(feed, 'icon')) ?? elementText(child(feed, 'logo'))),
|
|
115
|
+
entries,
|
|
116
|
+
},
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Read one `<item>` / `<entry>` element.
|
|
121
|
+
*
|
|
122
|
+
* Both dialects differ only in where they put the link and what they call the
|
|
123
|
+
* date, so one reader handles both.
|
|
124
|
+
*
|
|
125
|
+
* @param element - the item or entry element.
|
|
126
|
+
* @param sourceId - owning source id.
|
|
127
|
+
* @returns the entry, or `undefined` when the element carries no title or link.
|
|
128
|
+
*/
|
|
129
|
+
function readEntry(element, sourceId) {
|
|
130
|
+
const title = elementText(child(element, 'title'));
|
|
131
|
+
const link = resolveLink(element);
|
|
132
|
+
const displayTitle = title ?? link;
|
|
133
|
+
if (displayTitle === undefined)
|
|
134
|
+
return undefined;
|
|
135
|
+
const guid = elementText(child(element, 'id')) ?? elementText(child(element, 'guid'));
|
|
136
|
+
const content = bestContent(element);
|
|
137
|
+
const body = content ?? bestBody(element);
|
|
138
|
+
// The body IS only the feed's summary when no content element supplied one:
|
|
139
|
+
// the browser half needs that distinction to know a fetch is still owed.
|
|
140
|
+
const summaryOnly = content === undefined && body !== undefined;
|
|
141
|
+
const summary = bestSummary(element) ?? body;
|
|
142
|
+
const id = stableEntryId({
|
|
143
|
+
title: displayTitle,
|
|
144
|
+
...(link !== undefined ? { link } : {}),
|
|
145
|
+
...(guid !== undefined ? { guid } : {}),
|
|
146
|
+
});
|
|
147
|
+
const tags = readTags(element);
|
|
148
|
+
const author = readAuthor(element);
|
|
149
|
+
return {
|
|
150
|
+
id,
|
|
151
|
+
sourceId,
|
|
152
|
+
title: displayTitle,
|
|
153
|
+
...present('link', link),
|
|
154
|
+
...present('author', author),
|
|
155
|
+
...present('publishedAt', readDate(element)),
|
|
156
|
+
...(tags.length > 0 ? { tags } : {}),
|
|
157
|
+
...present('summary', summarize(summary, 320)),
|
|
158
|
+
...present('contentHtml', body),
|
|
159
|
+
...(summaryOnly ? { summaryOnly: true } : {}),
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* The best available body for the detail view.
|
|
164
|
+
*
|
|
165
|
+
* `content:encoded` / `content` win when present. When a feed ships only a
|
|
166
|
+
* description, that description IS the body the reader has — treating it as
|
|
167
|
+
* body-only would leave the detail view empty for every feed that does not
|
|
168
|
+
* publish full text, which is most of them.
|
|
169
|
+
*/
|
|
170
|
+
/**
|
|
171
|
+
* The feed's own FULL-TEXT element, when it publishes one.
|
|
172
|
+
*
|
|
173
|
+
* Distinct from {@link bestBody} on purpose. `bestBody` falls back to the
|
|
174
|
+
* `<description>`/`<summary>` field so the detail view is never empty for a
|
|
175
|
+
* feed that publishes nothing else — but that fallback is a SUMMARY, and a
|
|
176
|
+
* caller that cannot tell the two apart concludes the entry already has its
|
|
177
|
+
* text. That is exactly how a 167-character summary was rendered as the whole
|
|
178
|
+
* article and why neither the backfill nor opening it ever fetched the page.
|
|
179
|
+
*
|
|
180
|
+
* @param element - the `<item>` / `<entry>`.
|
|
181
|
+
* @returns the rich body, or `undefined` when the feed ships only a summary.
|
|
182
|
+
*/
|
|
183
|
+
function bestContent(element) {
|
|
184
|
+
for (const name of CONTENT_TAGS) {
|
|
185
|
+
const raw = firstDescendantText(element, name);
|
|
186
|
+
if (raw === undefined)
|
|
187
|
+
continue;
|
|
188
|
+
const normalized = normalizeRichText(decodeEntities(raw));
|
|
189
|
+
if (normalized.length > 0)
|
|
190
|
+
return normalized;
|
|
191
|
+
}
|
|
192
|
+
return undefined;
|
|
193
|
+
}
|
|
194
|
+
/** The fallback body (the feed's own summary), used when no content element exists. */
|
|
195
|
+
function bestBody(element) {
|
|
196
|
+
for (const name of [...CONTENT_TAGS, ...SUMMARY_TAGS]) {
|
|
197
|
+
const raw = firstDescendantText(element, name);
|
|
198
|
+
if (raw === undefined)
|
|
199
|
+
continue;
|
|
200
|
+
const normalized = normalizeRichText(decodeEntities(raw));
|
|
201
|
+
if (normalized.length > 0)
|
|
202
|
+
return normalized;
|
|
203
|
+
}
|
|
204
|
+
return undefined;
|
|
205
|
+
}
|
|
206
|
+
/** The best available short description as plain text. */
|
|
207
|
+
function bestSummary(element) {
|
|
208
|
+
for (const name of SUMMARY_TAGS) {
|
|
209
|
+
const raw = firstDescendantText(element, name);
|
|
210
|
+
if (raw === undefined)
|
|
211
|
+
continue;
|
|
212
|
+
const text = stripTags(decodeEntities(raw));
|
|
213
|
+
if (text.length > 0)
|
|
214
|
+
return text;
|
|
215
|
+
}
|
|
216
|
+
return undefined;
|
|
217
|
+
}
|
|
218
|
+
/** Feed-level icon: RSS `<image><url>`, Atom `<icon>`. */
|
|
219
|
+
function channelIcon(channel) {
|
|
220
|
+
if (channel === null)
|
|
221
|
+
return undefined;
|
|
222
|
+
const image = child(channel, 'image');
|
|
223
|
+
const candidates = [
|
|
224
|
+
image === null ? undefined : elementText(child(image, 'url')),
|
|
225
|
+
elementText(child(channel, 'icon')),
|
|
226
|
+
elementText(child(channel, 'logo')),
|
|
227
|
+
];
|
|
228
|
+
for (const candidate of candidates) {
|
|
229
|
+
if (candidate !== undefined && /^https?:\/\//i.test(candidate))
|
|
230
|
+
return candidate;
|
|
231
|
+
}
|
|
232
|
+
return undefined;
|
|
233
|
+
}
|
|
234
|
+
/** The publication date, trying the names both dialects use. */
|
|
235
|
+
function readDate(element) {
|
|
236
|
+
for (const name of DATE_TAGS) {
|
|
237
|
+
const text = elementText(child(element, name)) ?? firstDescendantText(element, name.toLowerCase());
|
|
238
|
+
if (text === undefined)
|
|
239
|
+
continue;
|
|
240
|
+
const parsed = new Date(text);
|
|
241
|
+
if (!Number.isNaN(parsed.getTime()))
|
|
242
|
+
return parsed.toISOString();
|
|
243
|
+
}
|
|
244
|
+
return undefined;
|
|
245
|
+
}
|
|
246
|
+
/** Author name: Atom `<author><name>` or RSS `<author>` / `<dc:creator>`. */
|
|
247
|
+
function readAuthor(element) {
|
|
248
|
+
const author = child(element, 'author');
|
|
249
|
+
const nested = author === null ? undefined : elementText(child(author, 'name'));
|
|
250
|
+
const direct = elementText(author);
|
|
251
|
+
return nested ?? firstDescendantText(element, 'creator') ?? direct;
|
|
252
|
+
}
|
|
253
|
+
/** Category text: RSS `<category>`, Atom `<category term="…">`. */
|
|
254
|
+
function readTags(element) {
|
|
255
|
+
const tags = [];
|
|
256
|
+
for (const node of Array.from(element.children)) {
|
|
257
|
+
if (node.localName !== 'category')
|
|
258
|
+
continue;
|
|
259
|
+
const value = (node.getAttribute('term') ?? node.textContent ?? '').trim();
|
|
260
|
+
if (value.length > 0 && !tags.includes(value))
|
|
261
|
+
tags.push(value);
|
|
262
|
+
}
|
|
263
|
+
return tags;
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Resolve an entry link. Atom keeps it in a `href` attribute (with `rel`
|
|
267
|
+
* selecting the readable one); RSS keeps it in element text; a permalink
|
|
268
|
+
* `guid` is the last resort.
|
|
269
|
+
*/
|
|
270
|
+
function resolveLink(element) {
|
|
271
|
+
const anchors = Array.from(element.children).filter(node => node.localName === 'link');
|
|
272
|
+
const preferred = anchors.find(node => node.getAttribute('rel') === 'alternate')
|
|
273
|
+
?? anchors.find(node => node.getAttribute('rel') === null)
|
|
274
|
+
?? anchors[0];
|
|
275
|
+
const href = preferred?.getAttribute('href')?.trim();
|
|
276
|
+
if (href !== undefined && href.length > 0)
|
|
277
|
+
return href;
|
|
278
|
+
const text = elementText(preferred ?? null);
|
|
279
|
+
if (text !== undefined)
|
|
280
|
+
return text;
|
|
281
|
+
const guid = child(element, 'guid');
|
|
282
|
+
if (guid?.getAttribute('isPermaLink') === 'false')
|
|
283
|
+
return undefined;
|
|
284
|
+
const guidText = elementText(guid);
|
|
285
|
+
return guidText !== undefined && /^https?:\/\//i.test(guidText) ? guidText : undefined;
|
|
286
|
+
}
|
|
287
|
+
/** A direct child by local name (namespace prefixes ignored). */
|
|
288
|
+
function child(element, localName) {
|
|
289
|
+
if (element === null)
|
|
290
|
+
return null;
|
|
291
|
+
for (const node of Array.from(element.children)) {
|
|
292
|
+
if (node.localName === localName)
|
|
293
|
+
return node;
|
|
294
|
+
}
|
|
295
|
+
return null;
|
|
296
|
+
}
|
|
297
|
+
/** The text of the nearest descendant with this local name. */
|
|
298
|
+
function firstDescendantText(element, localName) {
|
|
299
|
+
const direct = child(element, localName);
|
|
300
|
+
if (direct !== null)
|
|
301
|
+
return elementText(direct);
|
|
302
|
+
for (const node of Array.from(element.getElementsByTagName('*'))) {
|
|
303
|
+
if (node.localName === localName)
|
|
304
|
+
return elementText(node);
|
|
305
|
+
}
|
|
306
|
+
return undefined;
|
|
307
|
+
}
|
|
308
|
+
/** Element text with whitespace collapsed, or `undefined` when empty. */
|
|
309
|
+
function elementText(element) {
|
|
310
|
+
const text = collapse(element?.textContent ?? '');
|
|
311
|
+
return text.length > 0 ? text : undefined;
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Decode the entity subset feeds actually use, in ONE left-to-right pass, then
|
|
315
|
+
* apply a second pass only when the result still looks like escaped markup.
|
|
316
|
+
*
|
|
317
|
+
* The single pass matters: `&lt;` means the literal text `<`, so a chain
|
|
318
|
+
* of blanket replaces (or repeated decoding) would turn it into `<` and then
|
|
319
|
+
* into a tag. Scanning left to right and consuming `&` first keeps `<`,
|
|
320
|
+
* which is exactly what a publisher who escaped their markup twice intended.
|
|
321
|
+
*
|
|
322
|
+
* @param input - raw element text.
|
|
323
|
+
* @returns decoded text.
|
|
324
|
+
*/
|
|
325
|
+
export function decodeEntities(input) {
|
|
326
|
+
const once = decodeOnce(input);
|
|
327
|
+
// A second pass is only justified when the once-decoded text contains a
|
|
328
|
+
// COMPLETE escaped element — an opening tag and its matching close. That is
|
|
329
|
+
// what a publisher who escaped their whole body twice produces, and requiring
|
|
330
|
+
// the pair is what keeps two other texts intact: `&lt;` (the literal
|
|
331
|
+
// `<`, which the publisher escaped deliberately) and a lone `<b>`
|
|
332
|
+
// (a single tag meant as text).
|
|
333
|
+
if (/<([a-z][a-z0-9]*)[^<]*>[\s\S]*<\/\1\s*>/i.test(once))
|
|
334
|
+
return decodeOnce(once);
|
|
335
|
+
return once;
|
|
336
|
+
}
|
|
337
|
+
/** The named entities feeds use, mapped to their replacement. */
|
|
338
|
+
const NAMED_ENTITIES = {
|
|
339
|
+
amp: '&',
|
|
340
|
+
lt: '<',
|
|
341
|
+
gt: '>',
|
|
342
|
+
quot: '"',
|
|
343
|
+
apos: "'",
|
|
344
|
+
nbsp: ' ',
|
|
345
|
+
'#39': "'",
|
|
346
|
+
};
|
|
347
|
+
/** One decoding pass: scan left to right, consuming the longest match first. */
|
|
348
|
+
function decodeOnce(text) {
|
|
349
|
+
let out = '';
|
|
350
|
+
let index = 0;
|
|
351
|
+
while (index < text.length) {
|
|
352
|
+
const start = text.indexOf('&', index);
|
|
353
|
+
if (start === -1) {
|
|
354
|
+
out += text.slice(index);
|
|
355
|
+
break;
|
|
356
|
+
}
|
|
357
|
+
out += text.slice(index, start);
|
|
358
|
+
const end = text.indexOf(';', start + 1);
|
|
359
|
+
// A `&` with no terminator, or one absurdly far away, is literal text.
|
|
360
|
+
if (end === -1 || end - start > 12) {
|
|
361
|
+
out += '&';
|
|
362
|
+
index = start + 1;
|
|
363
|
+
continue;
|
|
364
|
+
}
|
|
365
|
+
const body = text.slice(start + 1, end);
|
|
366
|
+
const replacement = replacementFor(body);
|
|
367
|
+
if (replacement === undefined) {
|
|
368
|
+
out += text.slice(start, end + 1);
|
|
369
|
+
}
|
|
370
|
+
else {
|
|
371
|
+
out += replacement;
|
|
372
|
+
}
|
|
373
|
+
index = end + 1;
|
|
374
|
+
}
|
|
375
|
+
return out;
|
|
376
|
+
}
|
|
377
|
+
/** The replacement for one entity body, or `undefined` when it is not an entity. */
|
|
378
|
+
function replacementFor(body) {
|
|
379
|
+
const lower = body.toLowerCase();
|
|
380
|
+
const named = NAMED_ENTITIES[lower];
|
|
381
|
+
if (named !== undefined)
|
|
382
|
+
return named;
|
|
383
|
+
const hex = /^#x([0-9a-f]+)$/i.exec(body);
|
|
384
|
+
if (hex !== null)
|
|
385
|
+
return codePoint(Number.parseInt(hex[1] ?? '', 16));
|
|
386
|
+
const dec = /^#(\d+)$/.exec(body);
|
|
387
|
+
if (dec !== null)
|
|
388
|
+
return codePoint(Number.parseInt(dec[1] ?? '', 10));
|
|
389
|
+
return undefined;
|
|
390
|
+
}
|
|
391
|
+
/** A code point as a character, ignoring invalid and surrogate values. */
|
|
392
|
+
function codePoint(value) {
|
|
393
|
+
if (!Number.isFinite(value) || value < 0 || value > 0x10ffff)
|
|
394
|
+
return '';
|
|
395
|
+
if (value >= 0xd800 && value <= 0xdfff)
|
|
396
|
+
return '';
|
|
397
|
+
return String.fromCodePoint(value);
|
|
398
|
+
}
|
|
399
|
+
/** Strip tags to plain text (used for summaries, where markup is unwanted). */
|
|
400
|
+
function stripTags(html) {
|
|
401
|
+
return collapse(html.replace(/<[^>]*>/g, ' '));
|
|
402
|
+
}
|
|
403
|
+
/** Collapse whitespace and trim. */
|
|
404
|
+
function collapse(text) {
|
|
405
|
+
return text.replace(/\s+/g, ' ').trim();
|
|
406
|
+
}
|
|
407
|
+
/** Truncate for an error message only. */
|
|
408
|
+
function truncate(text, max) {
|
|
409
|
+
return text.length <= max ? text : `${text.slice(0, max)}…`;
|
|
410
|
+
}
|
|
411
|
+
/** Coerce an unknown thrown value into a message. */
|
|
412
|
+
function errorMessage(error) {
|
|
413
|
+
return error instanceof Error ? error.message : String(error);
|
|
414
|
+
}
|
|
415
|
+
/** Build an object with one optional key, or nothing when the value is absent. */
|
|
416
|
+
function present(key, value) {
|
|
417
|
+
return value === undefined ? {} : { [key]: value };
|
|
418
|
+
}
|
|
419
|
+
/* --------------------------------------------------------------- salvage */
|
|
420
|
+
/**
|
|
421
|
+
* Recover what the payload did manage to say, instead of reporting nothing.
|
|
422
|
+
*
|
|
423
|
+
* Complete `<item>` / `<entry>` blocks are re-parsed as fragments (they are
|
|
424
|
+
* valid XML on their own). The block the cut landed inside is repaired on a
|
|
425
|
+
* best-effort basis and returned as a PARTIAL entry, which is what makes the
|
|
426
|
+
* detail view's "content shown in part" note honest rather than decorative: the
|
|
427
|
+
* text exists, it simply stops early, and the note plus the original-page
|
|
428
|
+
* button are how the reader gets the rest.
|
|
429
|
+
*
|
|
430
|
+
* @param raw - the malformed payload.
|
|
431
|
+
* @param sourceId - owning source id.
|
|
432
|
+
* @param reason - the parser's complaint, carried onto the result.
|
|
433
|
+
* @returns the salvaged feed, or `undefined` when not even one entry was found.
|
|
434
|
+
*/
|
|
435
|
+
function salvageDocument(raw, sourceId, reason) {
|
|
436
|
+
const entries = [];
|
|
437
|
+
const feedTitle = firstMatch(raw, /<title[^>]*>([\s\S]{0,300}?)<\/title>/i);
|
|
438
|
+
// The document's own root attributes carry the namespace declarations, and a
|
|
439
|
+
// fragment is only parseable with them: `<content:encoded>` inside the item
|
|
440
|
+
// makes the WHOLE fragment malformed XML when `xmlns:content` is missing —
|
|
441
|
+
// which is exactly how the first salvage attempt lost a complete 42 KB item.
|
|
442
|
+
const rootAttrs = /<rss[^>]*>/i.exec(raw)?.[0]?.replace(/^<rss/i, '').replace(/>$/, '')
|
|
443
|
+
?? /<feed[^>]*>/i.exec(raw)?.[0]?.replace(/^<feed/i, '').replace(/>$/, '')
|
|
444
|
+
?? '';
|
|
445
|
+
for (const tag of ['item', 'entry']) {
|
|
446
|
+
// Complete blocks, in document order.
|
|
447
|
+
const complete = new RegExp(`<${tag}(?:\\s[^>]*)?>[\\s\\S]*?<\\/${tag}>`, 'gi');
|
|
448
|
+
const partial = new RegExp(`<${tag}(?:\\s[^>]*)?>[\\s\\S]*$`, 'i');
|
|
449
|
+
// Everything a complete block covers, so the trailing scan only looks at
|
|
450
|
+
// what is left after the last one.
|
|
451
|
+
let cursor = 0;
|
|
452
|
+
for (const match of raw.matchAll(complete)) {
|
|
453
|
+
const entry = readEntryFragment(match[0], sourceId, false, rootAttrs);
|
|
454
|
+
if (entry !== undefined)
|
|
455
|
+
entries.push(entry);
|
|
456
|
+
cursor = (match.index ?? 0) + match[0].length;
|
|
457
|
+
}
|
|
458
|
+
const rest = raw.slice(cursor);
|
|
459
|
+
const trailing = partial.exec(rest);
|
|
460
|
+
if (trailing !== null) {
|
|
461
|
+
const entry = readEntryFragment(trailing[0], sourceId, true, rootAttrs);
|
|
462
|
+
if (entry !== undefined)
|
|
463
|
+
entries.push(entry);
|
|
464
|
+
}
|
|
465
|
+
if (entries.length > 0)
|
|
466
|
+
break;
|
|
467
|
+
}
|
|
468
|
+
if (entries.length === 0)
|
|
469
|
+
return undefined;
|
|
470
|
+
return {
|
|
471
|
+
ok: true,
|
|
472
|
+
feed: {
|
|
473
|
+
...present('title', feedTitle),
|
|
474
|
+
incomplete: true,
|
|
475
|
+
incompleteReason: reason,
|
|
476
|
+
entries,
|
|
477
|
+
},
|
|
478
|
+
};
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Parse one entry fragment.
|
|
482
|
+
*
|
|
483
|
+
* A complete block is parsed as-is. A partial one is repaired first: every tag
|
|
484
|
+
* still open at the cut is closed, so the text before the cut stays readable
|
|
485
|
+
* instead of vanishing with the malformed markup.
|
|
486
|
+
*
|
|
487
|
+
* @param xml - the block's markup (complete or partial).
|
|
488
|
+
* @param sourceId - owning source id.
|
|
489
|
+
* @param partial - true when the block was cut off.
|
|
490
|
+
* @param rootAttrs - the document root's attributes (its namespace declarations).
|
|
491
|
+
* @returns the entry, or `undefined` when it carries no title and no link.
|
|
492
|
+
*/
|
|
493
|
+
function readEntryFragment(xml, sourceId, partial, rootAttrs) {
|
|
494
|
+
// A complete block needs no repair. A cut-off one gets its still-open tags
|
|
495
|
+
// closed so the half before the cut survives as text.
|
|
496
|
+
const candidates = partial ? [repairMarkup(xml), xml] : [xml];
|
|
497
|
+
for (const candidate of candidates) {
|
|
498
|
+
const wrapped = `<root${rootAttrs}>${candidate}</root>`;
|
|
499
|
+
let doc;
|
|
500
|
+
try {
|
|
501
|
+
doc = new DOMParser().parseFromString(wrapped, 'application/xml');
|
|
502
|
+
}
|
|
503
|
+
catch {
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
if (doc.querySelector('parsererror') !== null)
|
|
507
|
+
continue;
|
|
508
|
+
const element = doc.documentElement.firstElementChild;
|
|
509
|
+
if (element === null)
|
|
510
|
+
continue;
|
|
511
|
+
const entry = readEntry(element, sourceId);
|
|
512
|
+
if (entry === undefined)
|
|
513
|
+
continue;
|
|
514
|
+
if (!partial)
|
|
515
|
+
return entry;
|
|
516
|
+
return { ...entry, partial: true };
|
|
517
|
+
}
|
|
518
|
+
if (!partial)
|
|
519
|
+
return undefined;
|
|
520
|
+
// The repair did not produce parseable XML (closing every open tag is not
|
|
521
|
+
// always enough). The fragment nevertheless states its own fields, and they
|
|
522
|
+
// are readable straight out of the markup — which is what keeps a cut-off
|
|
523
|
+
// entry openable instead of dropping it.
|
|
524
|
+
return readFragmentFields(xml, sourceId);
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Read a cut-off fragment's fields out of its raw markup.
|
|
528
|
+
*
|
|
529
|
+
* Deliberately shallow: each field is the fragment's OWN element, matched
|
|
530
|
+
* without crossing `>` or `<`, and the body is the item-level description field
|
|
531
|
+
* (or the CDATA block it wraps). Nested markup inside the body cannot fake a
|
|
532
|
+
* match, because every pattern here stops at the next angle bracket.
|
|
533
|
+
*
|
|
534
|
+
* @param xml - the cut-off fragment.
|
|
535
|
+
* @param sourceId - owning source id.
|
|
536
|
+
* @returns the partial entry, or `undefined` when it names neither a title nor a link.
|
|
537
|
+
*/
|
|
538
|
+
function readFragmentFields(xml, sourceId) {
|
|
539
|
+
const title = elementTextRaw(xml, 'title');
|
|
540
|
+
const link = elementTextRaw(xml, 'link') ?? /<link[^>]*href=["']([^"']+)["']/i.exec(xml)?.[1];
|
|
541
|
+
const displayTitle = title ?? link;
|
|
542
|
+
if (displayTitle === undefined)
|
|
543
|
+
return undefined;
|
|
544
|
+
const summaryText = stripTags(decodeEntities(elementTextRaw(xml, 'description') ?? ''));
|
|
545
|
+
const body = salvageBody(xml);
|
|
546
|
+
return {
|
|
547
|
+
id: stableEntryId({ title: displayTitle, ...(link !== undefined ? { link } : {}) }),
|
|
548
|
+
sourceId,
|
|
549
|
+
title: displayTitle,
|
|
550
|
+
...present('link', link),
|
|
551
|
+
...present('author', elementTextRaw(xml, 'creator') ?? elementTextRaw(xml, 'author')),
|
|
552
|
+
...present('publishedAt', readDateRaw(xml)),
|
|
553
|
+
// A short item description is a fine card excerpt; a long one is body text
|
|
554
|
+
// that happens to sit in the description field, and belongs in the detail
|
|
555
|
+
// view only.
|
|
556
|
+
...(summaryText.length > 0 && summaryText.length <= 320 ? { summary: summaryText } : {}),
|
|
557
|
+
...(body !== undefined ? { contentHtml: body } : {}),
|
|
558
|
+
partial: true,
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
/** One element's own text, read without crossing an angle bracket. */
|
|
562
|
+
function elementTextRaw(xml, localName) {
|
|
563
|
+
const match = new RegExp(`<${localName}[^>]*>([^<]*)<`, 'i').exec(xml);
|
|
564
|
+
const value = decodeEntities(match?.[1] ?? '').trim();
|
|
565
|
+
return value.length > 0 ? value : undefined;
|
|
566
|
+
}
|
|
567
|
+
/** A fragment's date field, in whichever dialect it declares one. */
|
|
568
|
+
function readDateRaw(xml) {
|
|
569
|
+
for (const name of DATE_TAGS) {
|
|
570
|
+
const raw = elementTextRaw(xml, name);
|
|
571
|
+
if (raw === undefined)
|
|
572
|
+
continue;
|
|
573
|
+
const parsed = new Date(raw);
|
|
574
|
+
if (!Number.isNaN(parsed.getTime()))
|
|
575
|
+
return parsed.toISOString();
|
|
576
|
+
}
|
|
577
|
+
return undefined;
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* The readable body a cut-off fragment still carries.
|
|
581
|
+
*
|
|
582
|
+
* Two shapes cover what publishers actually ship: markup that made it through
|
|
583
|
+
* the cut (normalized as usual), and a CDATA/CDATA-like block whose closing
|
|
584
|
+
* `]]>` survived even though the element that wraps it did not — the common
|
|
585
|
+
* shape for a feed whose item body is one long `<description><![CDATA[…]`.
|
|
586
|
+
*
|
|
587
|
+
* @param xml - the fragment.
|
|
588
|
+
* @returns normalized markup, or `undefined` when nothing readable is left.
|
|
589
|
+
*/
|
|
590
|
+
function salvageBody(xml) {
|
|
591
|
+
// Every `<description` in the fragment, as prefixes of it. The item-level
|
|
592
|
+
// field is the EARLIEST one and therefore the LONGEST prefix — the cue that
|
|
593
|
+
// separates it from markup nested inside the body, which is full of its own
|
|
594
|
+
// `<figure><figcaption><description>`-shaped content.
|
|
595
|
+
const starts = [...xml.matchAll(/<description[^>]*>/gi)].map(match => match.index ?? 0);
|
|
596
|
+
const source = starts.length === 0 ? xml : xml.slice(Math.min(...starts));
|
|
597
|
+
const cdata = /<!\[CDATA\[([\s\S]*?)(?:\]\]>|$)/.exec(source)?.[1];
|
|
598
|
+
if (cdata !== undefined && cdata.trim().length > 0) {
|
|
599
|
+
const normalized = normalizeRichText(cdata);
|
|
600
|
+
if (normalized.length > 0)
|
|
601
|
+
return normalized;
|
|
602
|
+
}
|
|
603
|
+
const markup = /<(?:p|div|article|section)\b[\s\S]*$/i.exec(source)?.[0];
|
|
604
|
+
if (markup === undefined)
|
|
605
|
+
return undefined;
|
|
606
|
+
const repaired = repairMarkup(markup);
|
|
607
|
+
const normalized = normalizeRichText(repaired);
|
|
608
|
+
return normalized.length > 0 ? normalized : undefined;
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* Close the tags a truncated fragment left open, so its readable half survives.
|
|
612
|
+
*
|
|
613
|
+
* Deliberately conservative: it only appends closers, never rewrites or drops
|
|
614
|
+
* text. If the result still does not parse, the caller falls back to reading
|
|
615
|
+
* the fragment's title and link out of the raw markup.
|
|
616
|
+
*
|
|
617
|
+
* @param xml - the fragment.
|
|
618
|
+
* @returns the fragment with closers appended.
|
|
619
|
+
*/
|
|
620
|
+
function repairMarkup(xml) {
|
|
621
|
+
// 1. Drop a dangling `<` or a half-written tag at the very end.
|
|
622
|
+
let repaired = xml.replace(/<[^>]*$/, '');
|
|
623
|
+
// 2. Close an unterminated CDATA section — the common case for a feed whose
|
|
624
|
+
// item body is one long CDATA block.
|
|
625
|
+
const cdataOpens = (repaired.match(/<!\[CDATA\[/g) ?? []).length;
|
|
626
|
+
const cdataCloses = (repaired.match(/\]\]>/g) ?? []).length;
|
|
627
|
+
for (let index = 0; index < cdataOpens - cdataCloses; index++)
|
|
628
|
+
repaired += ']]>';
|
|
629
|
+
// 3. Trim back to the last `>`, so the scan below sees only whole tags.
|
|
630
|
+
const lastGt = repaired.lastIndexOf('>');
|
|
631
|
+
if (lastGt < 0)
|
|
632
|
+
return repaired;
|
|
633
|
+
repaired = repaired.slice(0, lastGt + 1);
|
|
634
|
+
// 4. Close every still-open element, innermost first.
|
|
635
|
+
const open = [];
|
|
636
|
+
for (const match of repaired.matchAll(/<(\/?)([A-Za-z_][\w.:-]*)(?:\s[^>]*?)?(\/?)>/g)) {
|
|
637
|
+
const [, closing, name, selfClosing] = match;
|
|
638
|
+
if (selfClosing === '/' || name === undefined || name.startsWith('!') || name.startsWith('?'))
|
|
639
|
+
continue;
|
|
640
|
+
if (closing === '/') {
|
|
641
|
+
const at = open.lastIndexOf(name);
|
|
642
|
+
if (at >= 0)
|
|
643
|
+
open.splice(at, 1);
|
|
644
|
+
}
|
|
645
|
+
else {
|
|
646
|
+
open.push(name);
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
return repaired + open.reverse().map(name => `</${name}>`).join('');
|
|
650
|
+
}
|
|
651
|
+
/** The first capture of a pattern, or `undefined`. */
|
|
652
|
+
function firstMatch(text, pattern) {
|
|
653
|
+
const match = pattern.exec(text);
|
|
654
|
+
const value = match?.[1]?.trim();
|
|
655
|
+
return value !== undefined && value.length > 0 ? value : undefined;
|
|
656
|
+
}
|