feedkeeper 0.26.6 → 0.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/server/dist/api/items.js +3 -3
- package/server/dist/api/v1/index.js +1 -0
- package/server/dist/api/v1/serialize.js +2 -1
- package/server/dist/db/migrations/0037_item_author.sql +2 -0
- package/server/dist/db/migrations/0038_author_revision.sql +7 -0
- package/server/dist/feeds/articleContent.js +3 -3
- package/server/dist/feeds/author.js +52 -0
- package/server/dist/feeds/contentNoise.js +199 -5
- package/server/dist/feeds/poller.js +2 -0
- package/server/dist/feeds/repository.js +6 -4
- package/server/dist/mcp/items.js +1 -1
- package/server/package.json +1 -1
- package/web/dist/assets/index-Ci8_fWis.js +32 -0
- package/web/dist/index.html +1 -1
- package/web/dist/sw.js +1 -1
- package/web/dist/assets/index-CqwqZaF0.js +0 -32
package/package.json
CHANGED
package/server/dist/api/items.js
CHANGED
|
@@ -49,8 +49,8 @@ itemsRouter.get("/", (req, res) => {
|
|
|
49
49
|
res.json(items.map(({ content_lead: _lead, lead_version: _leadVersion, ...item }) => ({
|
|
50
50
|
...item,
|
|
51
51
|
content_snippet: cleanSnippet(item.content_snippet, item.title),
|
|
52
|
-
content_html: readerHtml(item.content_html, item.title),
|
|
53
|
-
full_content_html: readerHtml(item.full_content_html, item.title),
|
|
52
|
+
content_html: readerHtml(item.content_html, item.title, item.author),
|
|
53
|
+
full_content_html: readerHtml(item.full_content_html, item.title, item.author),
|
|
54
54
|
read: Boolean(item.read),
|
|
55
55
|
bookmarked: Boolean(item.bookmarked),
|
|
56
56
|
})));
|
|
@@ -126,7 +126,7 @@ itemsRouter.post("/:itemId/extract-content", async (req, res) => {
|
|
|
126
126
|
const [served] = withArchivedImages([{ id: result.id, link: findItemById(result.id)?.link ?? null, full_content_html: result.fullContentHtml }], requestOrigin(req));
|
|
127
127
|
res.json({
|
|
128
128
|
id: result.id,
|
|
129
|
-
full_content_html: readerHtml(served.full_content_html, result.title ?? findItemById(result.id)?.title),
|
|
129
|
+
full_content_html: readerHtml(served.full_content_html, result.title ?? findItemById(result.id)?.title, findItemById(result.id)?.author),
|
|
130
130
|
...(result.cached ? {} : { title: result.title, byline: result.byline }),
|
|
131
131
|
cached: result.cached,
|
|
132
132
|
});
|
|
@@ -29,6 +29,7 @@ export function serializeItem(row, progress, withContent, hasNote = false) {
|
|
|
29
29
|
subscriptionId: row.feed_id,
|
|
30
30
|
subscriptionTitle: row.feed_title,
|
|
31
31
|
title: row.title,
|
|
32
|
+
author: row.author ?? null,
|
|
32
33
|
url: row.link,
|
|
33
34
|
publishedAt: row.published_at,
|
|
34
35
|
addedAt: row.created_at,
|
|
@@ -37,7 +38,7 @@ export function serializeItem(row, progress, withContent, hasNote = false) {
|
|
|
37
38
|
imageUrl: row.image_url,
|
|
38
39
|
hasFullText: Boolean(row.full_content_html),
|
|
39
40
|
content: contentMetadata(row),
|
|
40
|
-
...(withContent ? { contentHtml: readerHtml(row.content_html, row.title), fullTextHtml: readerHtml(row.full_content_html, row.title) } : {}),
|
|
41
|
+
...(withContent ? { contentHtml: readerHtml(row.content_html, row.title, row.author), fullTextHtml: readerHtml(row.full_content_html, row.title, row.author) } : {}),
|
|
41
42
|
state: {
|
|
42
43
|
read: Boolean(row.read),
|
|
43
44
|
saved: Boolean(row.bookmarked),
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
-- A client that already holds an item learns of a later author through the content revision. Only the revision
|
|
2
|
+
-- moves: the extraction status stays, so no full text is fetched again for it.
|
|
3
|
+
CREATE TRIGGER trg_author_revision AFTER UPDATE OF author ON items
|
|
4
|
+
WHEN OLD.author IS NOT NEW.author
|
|
5
|
+
BEGIN
|
|
6
|
+
UPDATE items SET content_revision = OLD.content_revision + 1 WHERE id = NEW.id;
|
|
7
|
+
END;
|
|
@@ -20,14 +20,14 @@ const readerCache = new Map();
|
|
|
20
20
|
* Article HTML as a reader shows it, without breadcrumbs, repeated headline and similar page chrome. Computed when
|
|
21
21
|
* served and never stored, so the publisher's original stays intact and rule changes apply to old articles too.
|
|
22
22
|
*/
|
|
23
|
-
export function readerHtml(html, title) {
|
|
23
|
+
export function readerHtml(html, title, author) {
|
|
24
24
|
if (!html)
|
|
25
25
|
return html ?? null;
|
|
26
|
-
const key = createHash("sha1").update(title ?? "").update("\0").update(html).digest("base64");
|
|
26
|
+
const key = createHash("sha1").update(title ?? "").update("\0").update(author ?? "").update("\0").update(html).digest("base64");
|
|
27
27
|
const cached = readerCache.get(key);
|
|
28
28
|
if (cached !== undefined)
|
|
29
29
|
return cached;
|
|
30
|
-
const cleaned = cleanReaderHtml(html, title);
|
|
30
|
+
const cleaned = cleanReaderHtml(html, title, {}, author);
|
|
31
31
|
if (readerCache.size >= READER_CACHE_LIMIT)
|
|
32
32
|
readerCache.delete(readerCache.keys().next().value);
|
|
33
33
|
readerCache.set(key, cleaned);
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { decodeEntities } from "./text.js";
|
|
2
|
+
const GENERIC_NAMES = /^(admin(istrator)?|redaktion(steam)?|redaktionsteam|editors?|editorial( team| staff)?|staff|team|webmaster|redakteur(in)?|autor|author|unknown|unbekannt|n\/?a|null|undefined|anonymous|anonym|gast|guest|user|news|newsdesk|desk)$/i;
|
|
3
|
+
const MAX_AUTHORS = 6;
|
|
4
|
+
const MAX_LENGTH = 160;
|
|
5
|
+
function key(value) {
|
|
6
|
+
return value.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "");
|
|
7
|
+
}
|
|
8
|
+
/** One name of a feed's author field. `staff@site.test (Anna Muster)` is the name in brackets; a bare address is no name. */
|
|
9
|
+
function nameOf(raw) {
|
|
10
|
+
let text = decodeEntities(raw).replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
|
|
11
|
+
const bracketed = text.match(/^\S+@\S+\s*\(([^)]+)\)$/);
|
|
12
|
+
if (bracketed)
|
|
13
|
+
text = bracketed[1].trim();
|
|
14
|
+
else if (/^<?\S+@\S+>?$/.test(text))
|
|
15
|
+
return null;
|
|
16
|
+
const withAddress = text.match(/^(.+?)\s*<\S+@\S+>$/);
|
|
17
|
+
if (withAddress)
|
|
18
|
+
text = withAddress[1].trim();
|
|
19
|
+
text = text.replace(/^(von|by|text|autor(in)?|author)\s*:?\s+/i, "").trim();
|
|
20
|
+
if (!text || /^https?:\/\//i.test(text) || GENERIC_NAMES.test(text))
|
|
21
|
+
return null;
|
|
22
|
+
return text;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* The author line of an item: every distinct name the feed gives, in order, joined by commas. Nothing is made up:
|
|
26
|
+
* addresses without a name, generic names ("admin", "Redaktion") and the publication's own name are no authors.
|
|
27
|
+
*/
|
|
28
|
+
export function authorLine(raw, publication = {}) {
|
|
29
|
+
const values = (Array.isArray(raw) ? raw : [raw]).filter((value) => typeof value === "string");
|
|
30
|
+
const own = new Set();
|
|
31
|
+
if (publication.title)
|
|
32
|
+
own.add(key(publication.title));
|
|
33
|
+
try {
|
|
34
|
+
if (publication.url)
|
|
35
|
+
own.add(key(new URL(publication.url).hostname.replace(/^www\./, "")));
|
|
36
|
+
}
|
|
37
|
+
catch { /* an unparsable feed address is no reason to lose the author */ }
|
|
38
|
+
// "Engadget" is the publication of "Engadget - Technology News" and of engadget.com alike.
|
|
39
|
+
const isOwn = (name) => name.length >= 4 && [...own].some(known => known === name || known.startsWith(name));
|
|
40
|
+
const names = [];
|
|
41
|
+
for (const value of values) {
|
|
42
|
+
for (const part of (decodeEntities(value) ?? "").split(/\s*(?:;|\s\/\s|\s&\s|\s+und\s+|\s+and\s)\s*/i)) {
|
|
43
|
+
const name = nameOf(part);
|
|
44
|
+
if (!name || isOwn(key(name)) || names.some(known => key(known) === key(name)))
|
|
45
|
+
continue;
|
|
46
|
+
names.push(name);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
const line = names.slice(0, MAX_AUTHORS).join(", ");
|
|
50
|
+
return line && line.length <= MAX_LENGTH ? line : line ? line.slice(0, MAX_LENGTH).replace(/[,\s]+\S*$/, "") || null : null;
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=author.js.map
|
|
@@ -45,10 +45,28 @@ const CONSENT_COMPANIONS = [
|
|
|
45
45
|
/\bInhalte? (von \S+ )?erlauben\b/i, /-Inhalte erlauben\b/i, /^Ich bin damit einverstanden, dass mir .+ angezeigt werden/i,
|
|
46
46
|
/Personenbezogene Daten können an Drittplattformen/i, /^Link zum .+-Inhalt$/i, /^Mehr dazu in unserer Datenschutzerklärung/i,
|
|
47
47
|
];
|
|
48
|
+
const PRODUCT_CARD_CHARACTERS = 220;
|
|
49
|
+
/** An amount of money: `88,89 EUR`, `$199`, `19.99 €`. */
|
|
50
|
+
const PRICE = /(?:\d[\d.,]*\s?(?:€|EUR|\$|USD|£|GBP|CHF)|(?:€|\$|£)\s?\d)/;
|
|
48
51
|
const MONTHS = "Januar|Februar|März|Maerz|April|Mai|Juni|Juli|August|September|Oktober|November|Dezember|January|February|March|June|July|October|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec|Jan\\.|Feb\\.";
|
|
49
52
|
const TIME_OF_DAY = "(?:[,\\s•·|-]+\\d{1,2}[:.]\\d{2}(?::\\d{2})?(?:\\s?(?:Uhr|[ap]\\.?m\\.?))?)?";
|
|
50
53
|
/** A whole line that is a date, with or without a time: `15. Oktober 2026, 10:51 Uhr`, `Oct 15, 2026`, `15.10.2026`. */
|
|
51
|
-
const
|
|
54
|
+
const DATE_PART = `(?:\\d{1,2}\\.\\s?\\d{1,2}\\.\\s?\\d{2,4}|\\d{1,2}\\.?\\s(?:${MONTHS})\\.?\\s\\d{4}|(?:${MONTHS})\\.?\\s\\d{1,2},?\\s\\d{4})`;
|
|
55
|
+
const DATE_ONLY = new RegExp(`^${DATE_PART}${TIME_OF_DAY}$`, "iu");
|
|
56
|
+
/** The date at the end of a byline (`von Michael Bartl I 15.10.2026 - 11:41`): the author stays, the date is in the reader's head. */
|
|
57
|
+
const BYLINE = /^(von|by|text|autor(in)?|author|written by|words by|story by|report(ed)? by|posted by|photo(s)? by)\b/i;
|
|
58
|
+
const BYLINE_DATE_TAIL = new RegExp(`(?:^|\\s)[|•·–—I-]\\s*${DATE_PART}${TIME_OF_DAY}\\s*$`, "iu");
|
|
59
|
+
function stripBylineDate(block) {
|
|
60
|
+
const walker = block.ownerDocument.createTreeWalker(block, 4);
|
|
61
|
+
let last = null;
|
|
62
|
+
for (let node = walker.nextNode(); node; node = walker.nextNode())
|
|
63
|
+
if (flat(node.textContent))
|
|
64
|
+
last = node;
|
|
65
|
+
if (!last || !BYLINE_DATE_TAIL.test(last.data))
|
|
66
|
+
return false;
|
|
67
|
+
last.data = last.data.replace(BYLINE_DATE_TAIL, "");
|
|
68
|
+
return true;
|
|
69
|
+
}
|
|
52
70
|
/** A whole line that is a reading time: `2 min`, `5 Min. Lesezeit`, `Lesezeit: 3 Minuten`, `4 min read`. */
|
|
53
71
|
const READING_TIME_ONLY = /^(?:Lesezeit:?\s*)?\d{1,3}\s*(?:min\.?|minuten?|minutes?)(?:\s*(?:Lesezeit|read|lesen))?$/i;
|
|
54
72
|
/** JSON-LD and similar: an object with a schema context or type, standing as text. */
|
|
@@ -62,7 +80,7 @@ function isStructuredData(text) {
|
|
|
62
80
|
return false;
|
|
63
81
|
}
|
|
64
82
|
}
|
|
65
|
-
function removeReaderNoise(root, title, stats) {
|
|
83
|
+
function removeReaderNoise(root, title, stats, author) {
|
|
66
84
|
let changed = false;
|
|
67
85
|
const drop = (element, rule) => {
|
|
68
86
|
if (!element.parentNode)
|
|
@@ -90,6 +108,22 @@ function removeReaderNoise(root, title, stats) {
|
|
|
90
108
|
if (onlyLinks)
|
|
91
109
|
drop(list, "breadcrumb");
|
|
92
110
|
}
|
|
111
|
+
// A product card of a shop link: its picture is a CSS background, which no reader draws, and its price is a snapshot
|
|
112
|
+
// that is out of date when the article is read later. What is left would be a label, a link and a number.
|
|
113
|
+
for (const shell of [...root.querySelectorAll("[style*='url(' i]")]) {
|
|
114
|
+
if (!shell.parentNode || !/background(-image)?\s*:[^;]*url\(/i.test(shell.getAttribute("style") ?? "") || flat(shell.textContent) !== "")
|
|
115
|
+
continue;
|
|
116
|
+
let card = null;
|
|
117
|
+
for (let ancestor = shell.parentElement; ancestor; ancestor = ancestor.parentElement) {
|
|
118
|
+
const text = flat(ancestor.textContent);
|
|
119
|
+
if (text.length > PRODUCT_CARD_CHARACTERS)
|
|
120
|
+
break;
|
|
121
|
+
if (ancestor.querySelector("a[href]") && PRICE.test(text))
|
|
122
|
+
card = ancestor;
|
|
123
|
+
}
|
|
124
|
+
if (card)
|
|
125
|
+
drop(card, "product-card");
|
|
126
|
+
}
|
|
93
127
|
// Structured data is for machines. Frameworks render it as an element of any name (`<component type="application/ld+json">`),
|
|
94
128
|
// and a stray block may hold nothing else: neither is text for a reader.
|
|
95
129
|
for (const element of [...root.querySelectorAll("[type]")]) {
|
|
@@ -102,6 +136,7 @@ function removeReaderNoise(root, title, stats) {
|
|
|
102
136
|
}
|
|
103
137
|
const headline = normalized(title ?? "");
|
|
104
138
|
const seen = new Set();
|
|
139
|
+
const authorKeys = (author ?? "").split(",").map(name => normalized(name)).filter(Boolean);
|
|
105
140
|
let leading = 0;
|
|
106
141
|
let reachedText = false;
|
|
107
142
|
for (const block of leafBlocks(root)) {
|
|
@@ -135,6 +170,15 @@ function removeReaderNoise(root, title, stats) {
|
|
|
135
170
|
drop(block, "reading-time");
|
|
136
171
|
continue;
|
|
137
172
|
}
|
|
173
|
+
// When the feed names the author, the reader shows the name in its head: a byline that only says that goes.
|
|
174
|
+
if (BYLINE.test(text) && authorKeys.length > 0 && authorKeys.every(name => normalized(text).includes(name))) {
|
|
175
|
+
drop(block, "byline");
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
if (BYLINE.test(text) && stripBylineDate(block)) {
|
|
179
|
+
stats["byline-date"] = (stats["byline-date"] ?? 0) + 1;
|
|
180
|
+
changed = true;
|
|
181
|
+
}
|
|
138
182
|
}
|
|
139
183
|
if (headline && normalized(text) === headline && (position < 4 || (heading && position < 14))) {
|
|
140
184
|
drop(block, "title-repeat");
|
|
@@ -181,6 +225,34 @@ function removeReaderNoise(root, title, stats) {
|
|
|
181
225
|
seen.add(key);
|
|
182
226
|
}
|
|
183
227
|
}
|
|
228
|
+
// A comment section's house rules at the end of the text: no reader has comments, so heading and what follows go.
|
|
229
|
+
const total = (root.textContent ?? "").length;
|
|
230
|
+
for (const block of leafBlocks(root)) {
|
|
231
|
+
if (!block.parentNode || !COMMENT_RULES_HEADING.test(flat(block.textContent)))
|
|
232
|
+
continue;
|
|
233
|
+
if (total - charsBefore(root, block) > TAIL_CHARACTERS)
|
|
234
|
+
continue;
|
|
235
|
+
let node = block;
|
|
236
|
+
while (node.parentNode && node !== root) {
|
|
237
|
+
while (node.nextSibling)
|
|
238
|
+
node.nextSibling.remove();
|
|
239
|
+
node = node.parentNode;
|
|
240
|
+
}
|
|
241
|
+
drop(block, "comment-rules");
|
|
242
|
+
break;
|
|
243
|
+
}
|
|
244
|
+
// An ad label whose ad is not there (a script fills it): sponsored content itself stays, the label above nothing goes.
|
|
245
|
+
const labelled = leafBlocks(root);
|
|
246
|
+
labelled.forEach((block, index) => {
|
|
247
|
+
if (!block.parentNode || !AD_LABEL.test(flat(block.textContent)) || block.querySelector("img, iframe, a"))
|
|
248
|
+
return;
|
|
249
|
+
const sibling = block.nextElementSibling;
|
|
250
|
+
if (sibling && (sibling.matches("img, iframe, video, picture, figure, a") || sibling.querySelector("img, iframe, video, picture, a")))
|
|
251
|
+
return;
|
|
252
|
+
const next = labelled.slice(index + 1).find(other => other.parentNode && flat(other.textContent));
|
|
253
|
+
if (!next || /^H[1-6]$/.test(next.tagName) || AD_LABEL.test(flat(next.textContent)))
|
|
254
|
+
drop(block, "empty-ad-label");
|
|
255
|
+
});
|
|
184
256
|
// Consent placeholders of embedded third-party content: only the neighbours of a known anchor phrase go.
|
|
185
257
|
const blocks = leafBlocks(root);
|
|
186
258
|
blocks.forEach((block, index) => {
|
|
@@ -278,15 +350,135 @@ function serialize(root) {
|
|
|
278
350
|
return holder.innerHTML;
|
|
279
351
|
}
|
|
280
352
|
/** The article HTML as a reader should show it. Unchanged input is returned as is. */
|
|
281
|
-
export function cleanReaderHtml(html, title, stats = {}) {
|
|
353
|
+
export function cleanReaderHtml(html, title, stats = {}, author) {
|
|
282
354
|
if (!html)
|
|
283
355
|
return html;
|
|
284
356
|
const root = JSDOM.fragment(html);
|
|
285
357
|
const images = resolveCustomImages(root, stats);
|
|
286
358
|
const variants = collapseImageVariants(root, stats);
|
|
287
|
-
const
|
|
359
|
+
const videos = resolveVideoPlayers(root, stats);
|
|
360
|
+
const noise = removeReaderNoise(root, title, stats, author);
|
|
288
361
|
const typography = removeTypography(root, stats);
|
|
289
|
-
|
|
362
|
+
videos.links.forEach(link => { if (root.contains(link))
|
|
363
|
+
link.setAttribute("data-media", "video"); });
|
|
364
|
+
return images || variants || videos.changed || noise || typography ? serialize(root) : html;
|
|
365
|
+
}
|
|
366
|
+
const PLAY_LABEL = /^(video starten|video abspielen|abspielen|wiedergabe( starten)?|play( video)?|watch( video)?|start video)$/i;
|
|
367
|
+
const PLAY_SOURCE = /(?:^|[/_.-])play[-_]?(?:icon|button|btn)?\.(?:svg|png|webp|gif)(?:[?#]|$)/i;
|
|
368
|
+
const DURATION = /^\d{1,2}:\d{2}(:\d{2})?$/;
|
|
369
|
+
const PLAYER_BLOCK_LIMIT = 120;
|
|
370
|
+
function isPlayOverlay(image) {
|
|
371
|
+
return PLAY_LABEL.test(flat(image.getAttribute("alt"))) || PLAY_SOURCE.test((image.getAttribute("src") ?? "").trim());
|
|
372
|
+
}
|
|
373
|
+
/** Takes away what is left empty once a player is gone, up to (not including) the fragment itself. */
|
|
374
|
+
function pruneEmpty(node) {
|
|
375
|
+
while (node && node.parentNode && node.nodeType === 1) {
|
|
376
|
+
const element = node;
|
|
377
|
+
if (flat(element.textContent) !== "" || element.querySelector("img, picture, video, iframe, a[href]"))
|
|
378
|
+
return;
|
|
379
|
+
const parent = element.parentNode;
|
|
380
|
+
element.remove();
|
|
381
|
+
node = parent;
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
/** Direct video URL of the embeds that are common enough to be worth a link. Anything else is not turned into a link. */
|
|
385
|
+
function embedTarget(source) {
|
|
386
|
+
let address;
|
|
387
|
+
try {
|
|
388
|
+
address = new URL(source.startsWith("//") ? `https:${source}` : source);
|
|
389
|
+
}
|
|
390
|
+
catch {
|
|
391
|
+
return null;
|
|
392
|
+
}
|
|
393
|
+
const host = address.hostname.replace(/^www\./, "");
|
|
394
|
+
const segments = address.pathname.split("/").filter(Boolean);
|
|
395
|
+
if (/^youtube(-nocookie)?\.com$/.test(host) && segments[0] === "embed" && /^[\w-]{6,}$/.test(segments[1] ?? "")) {
|
|
396
|
+
return { url: `https://www.youtube.com/watch?v=${segments[1]}`, name: "YouTube" };
|
|
397
|
+
}
|
|
398
|
+
if (host === "player.vimeo.com" && segments[0] === "video" && /^\d+$/.test(segments[1] ?? "")) {
|
|
399
|
+
return { url: `https://vimeo.com/${segments[1]}`, name: "Vimeo" };
|
|
400
|
+
}
|
|
401
|
+
if (host === "dailymotion.com" && segments[0] === "embed" && segments[1] === "video" && /^\w+$/.test(segments[2] ?? "")) {
|
|
402
|
+
return { url: `https://www.dailymotion.com/video/${segments[2]}`, name: "Dailymotion" };
|
|
403
|
+
}
|
|
404
|
+
return null;
|
|
405
|
+
}
|
|
406
|
+
function videoLink(document, url, text) {
|
|
407
|
+
const paragraph = document.createElement("p");
|
|
408
|
+
const link = document.createElement("a");
|
|
409
|
+
link.setAttribute("href", url);
|
|
410
|
+
link.textContent = text;
|
|
411
|
+
paragraph.append(link);
|
|
412
|
+
return paragraph;
|
|
413
|
+
}
|
|
414
|
+
/**
|
|
415
|
+
* Players need the publisher's script: a reader without it shows their play-button picture as a broken image and
|
|
416
|
+
* leaves a dead block. What can be reached stays as one plain link to the video (`data-media="video"` tells a client
|
|
417
|
+
* that the target is a video page); what cannot, because the player is a script-driven button without an address,
|
|
418
|
+
* is left out. The video itself is never embedded here: that is for the client to decide.
|
|
419
|
+
*/
|
|
420
|
+
function resolveVideoPlayers(root, stats) {
|
|
421
|
+
const links = [];
|
|
422
|
+
let changed = false;
|
|
423
|
+
const count = () => { stats["video"] = (stats["video"] ?? 0) + 1; changed = true; };
|
|
424
|
+
for (const image of [...root.querySelectorAll("img")].filter(isPlayOverlay)) {
|
|
425
|
+
if (!root.contains(image))
|
|
426
|
+
continue;
|
|
427
|
+
const anchor = image.closest("a[href]");
|
|
428
|
+
if (anchor) {
|
|
429
|
+
const parent = anchor.parentNode;
|
|
430
|
+
image.remove();
|
|
431
|
+
const href = anchor.getAttribute("href");
|
|
432
|
+
const scope = anchor.closest("p, div, figure, li") ?? anchor.parentElement;
|
|
433
|
+
const twin = scope ? [...scope.querySelectorAll("a[href]")].find(other => other !== anchor && other.getAttribute("href") === href) : undefined;
|
|
434
|
+
const label = flat(anchor.textContent);
|
|
435
|
+
if (twin) {
|
|
436
|
+
anchor.remove();
|
|
437
|
+
pruneEmpty(parent);
|
|
438
|
+
links.push(twin);
|
|
439
|
+
}
|
|
440
|
+
else {
|
|
441
|
+
if (label === "" || DURATION.test(label))
|
|
442
|
+
anchor.textContent = flat(anchor.getAttribute("title")) || "Video";
|
|
443
|
+
links.push(anchor);
|
|
444
|
+
}
|
|
445
|
+
count();
|
|
446
|
+
continue;
|
|
447
|
+
}
|
|
448
|
+
// The player block is the widest wrapper that holds nothing but the label, the picture and no link to follow.
|
|
449
|
+
let block = image;
|
|
450
|
+
while (block.parentElement) {
|
|
451
|
+
const parent = block.parentElement;
|
|
452
|
+
if (flat(parent.textContent).length > PLAYER_BLOCK_LIMIT || parent.querySelector("a[href]") || parent.querySelectorAll("img").length > 1)
|
|
453
|
+
break;
|
|
454
|
+
block = parent;
|
|
455
|
+
}
|
|
456
|
+
const parent = block.parentNode;
|
|
457
|
+
block.remove();
|
|
458
|
+
pruneEmpty(parent);
|
|
459
|
+
count();
|
|
460
|
+
}
|
|
461
|
+
for (const frame of [...root.querySelectorAll("iframe")]) {
|
|
462
|
+
const target = embedTarget(frame.getAttribute("src") ?? frame.getAttribute("data-src") ?? "");
|
|
463
|
+
if (!target)
|
|
464
|
+
continue;
|
|
465
|
+
const parent = frame.parentNode;
|
|
466
|
+
const replacement = videoLink(frame.ownerDocument, target.url, flat(frame.getAttribute("title")) || target.name);
|
|
467
|
+
frame.replaceWith(replacement);
|
|
468
|
+
pruneEmpty(parent);
|
|
469
|
+
links.push(replacement.querySelector("a"));
|
|
470
|
+
count();
|
|
471
|
+
}
|
|
472
|
+
for (const video of [...root.querySelectorAll("video")]) {
|
|
473
|
+
const source = (video.getAttribute("src") ?? video.querySelector("source[src]")?.getAttribute("src") ?? "").trim();
|
|
474
|
+
if (!/^https?:\/\//i.test(source))
|
|
475
|
+
continue;
|
|
476
|
+
const replacement = videoLink(video.ownerDocument, source, flat(video.getAttribute("title")) || "Video");
|
|
477
|
+
video.replaceWith(replacement);
|
|
478
|
+
links.push(replacement.querySelector("a"));
|
|
479
|
+
count();
|
|
480
|
+
}
|
|
481
|
+
return { changed, links };
|
|
290
482
|
}
|
|
291
483
|
/**
|
|
292
484
|
* Some publishers wrap a picture in their own element and let a script fill the image: `<a-img src="/imgs/…">`
|
|
@@ -390,6 +582,8 @@ const COMMENT_PATTERNS = [
|
|
|
390
582
|
/\b(let us know|tell us) .{0,40}in the comments\b/i,
|
|
391
583
|
/\bNur angemeldete Benutzer können kommentieren\b/i,
|
|
392
584
|
];
|
|
585
|
+
const AD_LABEL = /^(Anzeige|Werbung|Advertisement|Advertising|Advertorial|Ads?|Sponsored( content| post)?|Paid (content|post)|Gesponsert|Promoted)\s*:?$/i;
|
|
586
|
+
const COMMENT_RULES_HEADING = /^(Regeln für Kommentare|Kommentarregeln|Kommentar-Richtlinien|Community[- ]Richtlinien|(Comment(ing)?|Community|Discussion|House) (rules|guidelines|policy|policies)|Rules for comments|Commenting rules)\s*:?$/i;
|
|
393
587
|
const TAG_LABEL = /^(Schlagw(ö|oe)rter|Schlagworte|Tags?|Keywords|Themen|Topics)\s*:?$/i;
|
|
394
588
|
const VIDEO_CARD = /^\d{1,2}:\d{2}(:\d{2})?(?=\D)\s*\S/;
|
|
395
589
|
const SECTION_LABEL = /^(Auch interessant|Das könnte (dich|Sie|euch) (auch )?interessieren|Mehr zum Thema|Lesen Sie auch|Lies auch|Lest auch|Aktuelle News|Weitere (Artikel|Meldungen|News)|Popular Stories|Related (Stories|Articles|Posts|Reading)|More (Stories|from)|Top Stories|Meistgelesen|Topaktuell|Regeln für Kommentare|Kommentare|Comments?)\b\s*:?/i;
|
|
@@ -10,6 +10,7 @@ import { db } from "../db/index.js";
|
|
|
10
10
|
import { configurePollHostCooldowns, PollQueue } from "./pollQueue.js";
|
|
11
11
|
import { feedRequests, PollDeferredError, sourceHost } from "./requestGate.js";
|
|
12
12
|
import { normalizePublishedAt } from "./dates.js";
|
|
13
|
+
import { authorLine } from "./author.js";
|
|
13
14
|
configurePollHostCooldowns();
|
|
14
15
|
function mediaAttribute(value, key) {
|
|
15
16
|
if (typeof value !== "object" || value === null)
|
|
@@ -111,6 +112,7 @@ async function pollAttempt(feed, signal) {
|
|
|
111
112
|
contentSnippet: decodeEntities(item.contentSnippet ?? item.content) ?? undefined,
|
|
112
113
|
contentHtml: item.contentEncoded ?? item["content:encoded"] ?? item.content ?? null,
|
|
113
114
|
publishedAt: normalizePublishedAt(item.isoDate ?? item.pubDate) ?? undefined, imageUrl: extractImageUrl(item),
|
|
115
|
+
author: authorLine([item.creator, item.author], { title: current.title, url: current.site_url ?? current.url }),
|
|
114
116
|
}));
|
|
115
117
|
// No await in this transaction. Items, health/validators and job completion become visible together.
|
|
116
118
|
const newItems = db.transaction(() => {
|
|
@@ -136,8 +136,8 @@ export function updateFeedAfterPoll(feedId, data) {
|
|
|
136
136
|
}
|
|
137
137
|
}
|
|
138
138
|
export function upsertItems(feedId, items) {
|
|
139
|
-
const insert = db.prepare(`INSERT INTO items (feed_id, guid, title, link, content_snippet, content_html, published_at, image_url)
|
|
140
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
139
|
+
const insert = db.prepare(`INSERT INTO items (feed_id, guid, title, link, content_snippet, content_html, published_at, image_url, author)
|
|
140
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
141
141
|
ON CONFLICT (feed_id, guid) DO NOTHING`);
|
|
142
142
|
const update = db.prepare(`UPDATE items SET
|
|
143
143
|
title = COALESCE(?, title),
|
|
@@ -145,7 +145,8 @@ export function upsertItems(feedId, items) {
|
|
|
145
145
|
content_snippet = COALESCE(?, content_snippet),
|
|
146
146
|
content_html = COALESCE(?, content_html),
|
|
147
147
|
published_at = COALESCE(?, published_at),
|
|
148
|
-
image_url = COALESCE(?, image_url)
|
|
148
|
+
image_url = COALESCE(?, image_url),
|
|
149
|
+
author = COALESCE(?, author)
|
|
149
150
|
WHERE feed_id = ? AND guid = ?`);
|
|
150
151
|
const insertMany = db.transaction((rows) => {
|
|
151
152
|
let inserted = 0;
|
|
@@ -159,6 +160,7 @@ export function upsertItems(feedId, items) {
|
|
|
159
160
|
row.contentHtml ?? null,
|
|
160
161
|
row.publishedAt ?? null,
|
|
161
162
|
largerHeroImage(row.imageUrl, row.contentHtml, row.link ?? null) ?? row.imageUrl ?? null,
|
|
163
|
+
row.author ?? null,
|
|
162
164
|
];
|
|
163
165
|
const result = insert.run(...values);
|
|
164
166
|
if (result.changes > 0) {
|
|
@@ -174,7 +176,7 @@ export function upsertItems(feedId, items) {
|
|
|
174
176
|
// The summary belongs to the body it was sent with: keeping the longer body keeps its summary too, or
|
|
175
177
|
// a feed that alternates between two variants would change the content on every poll.
|
|
176
178
|
const keptPrevious = retained !== null && next !== retained;
|
|
177
|
-
update.run(row.title ?? null, row.link ?? null, keptPrevious ? null : row.contentSnippet ?? null, retained, row.publishedAt ?? null, row.imageUrl ?? null, feedId, row.guid);
|
|
179
|
+
update.run(row.title ?? null, row.link ?? null, keptPrevious ? null : row.contentSnippet ?? null, retained, row.publishedAt ?? null, row.imageUrl ?? null, row.author ?? null, feedId, row.guid);
|
|
178
180
|
// Polling rewrites every item; only a change of what the lead is made from needs it derived again.
|
|
179
181
|
if (previous) {
|
|
180
182
|
const changed = (retained !== null && retained !== previous.content_html) ||
|
package/server/dist/mcp/items.js
CHANGED
|
@@ -51,7 +51,7 @@ export function getItemForMcp(userId, itemId, maxChars = MAX_CONTENT_LENGTH, off
|
|
|
51
51
|
return null;
|
|
52
52
|
if (revision !== undefined && revision !== item.content_revision)
|
|
53
53
|
throw new Error("content_revision_conflict");
|
|
54
|
-
const content = readerHtml(item.full_content_html ?? item.content_html, item.title);
|
|
54
|
+
const content = readerHtml(item.full_content_html ?? item.content_html, item.title, item.author);
|
|
55
55
|
const body = format === "text" ? sourceText(item) : content ?? "";
|
|
56
56
|
let end = Math.min(offset + Math.min(maxChars, MAX_CONTENT_LENGTH), body.length);
|
|
57
57
|
if (end < body.length && /[\uD800-\uDBFF]/.test(body[end - 1]))
|