feedkeeper 0.32.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/server/dist/feeds/content/embeds/headings.js +38 -0
- package/server/dist/feeds/content/embeds/index.js +3 -2
- package/server/dist/feeds/content/embeds/media.js +45 -0
- package/server/dist/feeds/content/index.js +5 -1
- package/server/dist/feeds/content/noise/boilerplate.js +106 -0
- package/server/dist/feeds/content/noise/loading.js +29 -0
- package/server/dist/feeds/content/noise/pixels.js +27 -0
- package/server/dist/feeds/content/patterns/boilerplate/de.js +22 -0
- package/server/dist/feeds/content/patterns/boilerplate/en.js +21 -0
- package/server/dist/feeds/content/patterns/boilerplate/index.js +6 -0
- package/server/dist/feeds/content/patterns/loading/de.js +5 -0
- package/server/dist/feeds/content/patterns/loading/en.js +8 -0
- package/server/dist/feeds/content/patterns/loading/index.js +5 -0
- package/server/dist/feeds/content/patterns/sidebars/de.js +2 -0
- package/server/dist/feeds/content/patterns/sidebars/en.js +2 -0
- package/server/dist/feeds/feedIcon.js +12 -6
- package/server/dist/feeds/poller.js +12 -10
- package/server/package.json +1 -1
package/package.json
CHANGED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { count, flat, pruneEmpty } from "../shared.js";
|
|
2
|
+
const HEADINGS = "h2, h3, h4, h5";
|
|
3
|
+
const SCOPE_LEVELS = 4;
|
|
4
|
+
/** The wrapper of a card and its heading is small; a larger block holds the story, and its headings stay. */
|
|
5
|
+
const SCOPE_TEXT_LIMIT = 400;
|
|
6
|
+
const FOLLOWING = 4;
|
|
7
|
+
/**
|
|
8
|
+
* Some sites put the title of a video or post above its player as a heading that links to a page of their own. Once the player is a
|
|
9
|
+
* card with the same title, the title stands twice. The heading goes when it holds nothing but that title, is at most one link, and sits
|
|
10
|
+
* in the small wrapper the card sits in.
|
|
11
|
+
*/
|
|
12
|
+
export const embedHeadings = {
|
|
13
|
+
name: "embed-headings",
|
|
14
|
+
description: "A heading that only repeats the title of the video or post card right after it is removed.",
|
|
15
|
+
apply(root, stats) {
|
|
16
|
+
let changed = false;
|
|
17
|
+
for (const card of [...root.querySelectorAll("a[data-media]")]) {
|
|
18
|
+
const title = flat(card.textContent).toLowerCase();
|
|
19
|
+
if (title === "")
|
|
20
|
+
continue;
|
|
21
|
+
let scope = card.parentElement;
|
|
22
|
+
for (let level = 0; scope && level < SCOPE_LEVELS && flat(scope.textContent).length <= SCOPE_TEXT_LIMIT; level++, scope = scope.parentElement) {
|
|
23
|
+
const heading = [...scope.querySelectorAll(HEADINGS)].find(candidate => root.contains(candidate) && !candidate.contains(card) && flat(candidate.textContent).toLowerCase() === title &&
|
|
24
|
+
candidate.querySelectorAll("a").length <= 1 && (candidate.compareDocumentPosition(card) & FOLLOWING) !== 0);
|
|
25
|
+
if (!heading)
|
|
26
|
+
continue;
|
|
27
|
+
const parent = heading.parentNode;
|
|
28
|
+
heading.remove();
|
|
29
|
+
pruneEmpty(parent);
|
|
30
|
+
count(stats, "embed-headings");
|
|
31
|
+
changed = true;
|
|
32
|
+
break;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return changed;
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
//# sourceMappingURL=headings.js.map
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { consentEmbeds } from "./consent.js";
|
|
2
|
+
import { mediaElements } from "./media.js";
|
|
2
3
|
import { socialEmbeds } from "./social.js";
|
|
3
4
|
import { videoPlayers } from "./video.js";
|
|
4
5
|
/**
|
|
5
6
|
* Embeds, in the order they must run: consent plugins first, because the address of the embed sits inside their
|
|
6
|
-
* notice, then the video players and iframes, then the social posts.
|
|
7
|
+
* notice, then the video players and iframes, then native audio and video elements, then the social posts.
|
|
7
8
|
*/
|
|
8
|
-
export const EMBED_RULES = [consentEmbeds, videoPlayers, socialEmbeds];
|
|
9
|
+
export const EMBED_RULES = [consentEmbeds, videoPlayers, mediaElements, socialEmbeds];
|
|
9
10
|
/**
|
|
10
11
|
* Turns every embed of a page into the marked link a reader shows as a card. A fetched page must go through this
|
|
11
12
|
* before its scripts, frames and `noscript` blocks are removed: that is where consent plugins and players keep the address.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { count, flat, linkParagraph, pruneEmpty } from "../shared.js";
|
|
2
|
+
/** The address a native `<audio>` or `<video>` plays: its own `src`, or the first `<source>` inside it. Anything but http(s) is no address. */
|
|
3
|
+
function mediaAddress(element) {
|
|
4
|
+
const source = element.getAttribute("src") ?? element.querySelector("source[src]")?.getAttribute("src") ?? "";
|
|
5
|
+
const address = source.trim();
|
|
6
|
+
if (address === "")
|
|
7
|
+
return null;
|
|
8
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(address) && !/^https?:/i.test(address))
|
|
9
|
+
return null;
|
|
10
|
+
return address;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* A native `<audio>` or `<video>` element needs the player of the reader's engine and the publisher's script to do anything useful: without
|
|
14
|
+
* them it shows a bar with an error. What names an address becomes one marked link to the file (`data-media="audio" | "video"`), what does
|
|
15
|
+
* not, because the file is set by a script, is left out.
|
|
16
|
+
*/
|
|
17
|
+
export const mediaElements = {
|
|
18
|
+
name: "media-elements",
|
|
19
|
+
description: "Native audio and video elements become one marked link to their file, or are removed when they name no address.",
|
|
20
|
+
apply(root, stats) {
|
|
21
|
+
const document = root.ownerDocument ?? root;
|
|
22
|
+
let changed = false;
|
|
23
|
+
for (const element of [...root.querySelectorAll("audio, video")]) {
|
|
24
|
+
if (!root.contains(element))
|
|
25
|
+
continue;
|
|
26
|
+
const kind = element.tagName.toLowerCase() === "audio" ? "audio" : "video";
|
|
27
|
+
const address = mediaAddress(element);
|
|
28
|
+
const parent = element.parentNode;
|
|
29
|
+
if (address) {
|
|
30
|
+
const label = flat(element.getAttribute("title")) || flat(element.getAttribute("aria-label")) || (kind === "audio" ? "Audio" : "Video");
|
|
31
|
+
const replacement = linkParagraph(document, address, label);
|
|
32
|
+
replacement.querySelector("a")?.setAttribute("data-media", kind);
|
|
33
|
+
element.replaceWith(replacement);
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
element.remove();
|
|
37
|
+
pruneEmpty(parent);
|
|
38
|
+
}
|
|
39
|
+
count(stats, "media-elements");
|
|
40
|
+
changed = true;
|
|
41
|
+
}
|
|
42
|
+
return changed;
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
//# sourceMappingURL=media.js.map
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import { EMBED_RULES, resolveEmbeds } from "./embeds/index.js";
|
|
2
|
+
import { embedHeadings } from "./embeds/headings.js";
|
|
3
|
+
import { boilerplate } from "./noise/boilerplate.js";
|
|
4
|
+
import { loadingPlaceholders } from "./noise/loading.js";
|
|
5
|
+
import { trackingPixels } from "./noise/pixels.js";
|
|
2
6
|
import { sidebars } from "./noise/sidebars.js";
|
|
3
7
|
/**
|
|
4
8
|
* Every rule of the content pipeline, in the order they run on a fetched page: embeds first (their address sits in
|
|
5
9
|
* tags that are removed next), then the clutter around the story. The tests check each rule against its fixtures in
|
|
6
10
|
* `test/fixtures/content/<rule name>` and that running it twice changes nothing. See `docs/content-rules.md`.
|
|
7
11
|
*/
|
|
8
|
-
export const PAGE_RULES = [...EMBED_RULES, sidebars];
|
|
12
|
+
export const PAGE_RULES = [...EMBED_RULES, embedHeadings, sidebars, loadingPlaceholders, trackingPixels, boilerplate];
|
|
9
13
|
export { resolveEmbeds };
|
|
10
14
|
/** What a fetched page goes through before its scripts, frames and `noscript` blocks are removed. */
|
|
11
15
|
export function cleanPage(root, stats = {}) {
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { BOILERPLATE_ANYWHERE, BOILERPLATE_TRAILING } from "../patterns/boilerplate/index.js";
|
|
2
|
+
import { count, flat, pruneEmpty } from "../shared.js";
|
|
3
|
+
const BLOCKS = "p, div, li, h2, h3, h4, h5, h6, span";
|
|
4
|
+
const NESTED = "p, div, li, ul, ol, h2, h3, h4, h5, h6";
|
|
5
|
+
/** Footers can only sit in the second half of the text: before it, the same words are part of the story. */
|
|
6
|
+
const AFTER_SHARE = 0.5;
|
|
7
|
+
/** A list after a removed box ("Follow topics and authors …") that holds names, or nothing at all, belongs to the box. */
|
|
8
|
+
const LIST_ITEM_LIMIT = 60;
|
|
9
|
+
function textBefore(root, node) {
|
|
10
|
+
const range = root.ownerDocument?.createRange() ?? root.createRange();
|
|
11
|
+
range.setStart(root, 0);
|
|
12
|
+
range.setEndBefore(node);
|
|
13
|
+
return flat(range.toString()).length;
|
|
14
|
+
}
|
|
15
|
+
/** The list that directly follows a block, when it only holds short names or empty items. */
|
|
16
|
+
function followingNameList(block) {
|
|
17
|
+
const next = block.nextElementSibling;
|
|
18
|
+
if (!next || !/^(UL|OL)$/.test(next.tagName))
|
|
19
|
+
return null;
|
|
20
|
+
const items = [...next.querySelectorAll("li")];
|
|
21
|
+
return items.length > 0 && items.every(item => flat(item.textContent).length <= LIST_ITEM_LIMIT) ? next : null;
|
|
22
|
+
}
|
|
23
|
+
/** Elements that start a line of their own; a run of text between two `<br>` that holds one of them is not a plain line. */
|
|
24
|
+
const LINE_BREAKING = `${NESTED}, img, picture, video, audio, iframe, table, blockquote, figure`;
|
|
25
|
+
/** The text nodes and inline elements of a container, cut at every `<br>`. */
|
|
26
|
+
function linesOf(container) {
|
|
27
|
+
const lines = [[]];
|
|
28
|
+
container.childNodes.forEach(node => {
|
|
29
|
+
if (node.nodeName === "BR")
|
|
30
|
+
lines.push([]);
|
|
31
|
+
else
|
|
32
|
+
lines[lines.length - 1].push(node);
|
|
33
|
+
});
|
|
34
|
+
return lines;
|
|
35
|
+
}
|
|
36
|
+
function isPlainLine(nodes) {
|
|
37
|
+
return nodes.every(node => node.nodeType === 3 || (node.nodeType === 1 && !node.matches(LINE_BREAKING) && !node.querySelector(LINE_BREAKING)));
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Lines that sites add to their own text: "Follow us on Google", the words of a download button, the file name of a picture shown as text,
|
|
41
|
+
* "This article first appeared on …", the related links under a MacRumors story. A block goes when its whole text is one of these lines
|
|
42
|
+
* (footers only in the second half), together with the list of names that follows a "follow topics" box.
|
|
43
|
+
*/
|
|
44
|
+
export const boilerplate = {
|
|
45
|
+
name: "boilerplate",
|
|
46
|
+
description: "Blocks that only hold a follow-us call, a download button text, an image file name or an \"appeared first on\" footer are removed.",
|
|
47
|
+
apply(root, stats) {
|
|
48
|
+
let changed = false;
|
|
49
|
+
const total = flat(root.textContent).length;
|
|
50
|
+
for (const block of [...root.querySelectorAll(BLOCKS)]) {
|
|
51
|
+
if (!root.contains(block) || block.querySelector("img, picture, video, audio, iframe") || block.querySelector(NESTED))
|
|
52
|
+
continue;
|
|
53
|
+
const text = flat(block.textContent);
|
|
54
|
+
if (text === "" || text.length > 400)
|
|
55
|
+
continue;
|
|
56
|
+
const anywhere = BOILERPLATE_ANYWHERE.some(pattern => pattern.test(text));
|
|
57
|
+
const trailing = !anywhere && total > 0 && textBefore(root, block) / total >= AFTER_SHARE && BOILERPLATE_TRAILING.some(pattern => pattern.test(text));
|
|
58
|
+
if (!anywhere && !trailing)
|
|
59
|
+
continue;
|
|
60
|
+
const parent = block.parentNode;
|
|
61
|
+
followingNameList(block)?.remove();
|
|
62
|
+
block.remove();
|
|
63
|
+
pruneEmpty(parent);
|
|
64
|
+
count(stats, "boilerplate");
|
|
65
|
+
changed = true;
|
|
66
|
+
}
|
|
67
|
+
return lineFooters(root, stats, total) || changed;
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
/**
|
|
71
|
+
* Footers that stand as bare text between `<br>` ("This article … first appeared on …", "Discuss this article in our forums"), not in a block of
|
|
72
|
+
* their own. Each such line is checked like a block; the breaks left at the end of the container go with them.
|
|
73
|
+
*/
|
|
74
|
+
function lineFooters(root, stats, total) {
|
|
75
|
+
let changed = false;
|
|
76
|
+
const containers = [root, ...root.querySelectorAll("div, p, li, td, section, article, main, blockquote")];
|
|
77
|
+
for (const container of containers) {
|
|
78
|
+
if (![...container.childNodes].some(node => node.nodeName === "BR"))
|
|
79
|
+
continue;
|
|
80
|
+
let removed = false;
|
|
81
|
+
for (const line of linesOf(container)) {
|
|
82
|
+
if (line.length === 0 || !isPlainLine(line))
|
|
83
|
+
continue;
|
|
84
|
+
const text = flat(line.map(node => node.textContent ?? "").join(""));
|
|
85
|
+
if (text === "" || text.length > 400)
|
|
86
|
+
continue;
|
|
87
|
+
const anywhere = BOILERPLATE_ANYWHERE.some(pattern => pattern.test(text));
|
|
88
|
+
const trailing = !anywhere && total > 0 && textBefore(root, line[0]) / total >= AFTER_SHARE && BOILERPLATE_TRAILING.some(pattern => pattern.test(text));
|
|
89
|
+
if (!anywhere && !trailing)
|
|
90
|
+
continue;
|
|
91
|
+
const after = line[line.length - 1].nextSibling;
|
|
92
|
+
line.forEach(node => node.parentNode?.removeChild(node));
|
|
93
|
+
if (after?.nodeName === "BR")
|
|
94
|
+
after.parentNode?.removeChild(after);
|
|
95
|
+
count(stats, "boilerplate");
|
|
96
|
+
removed = true;
|
|
97
|
+
}
|
|
98
|
+
if (!removed)
|
|
99
|
+
continue;
|
|
100
|
+
while (container.lastChild && (container.lastChild.nodeName === "BR" || (container.lastChild.nodeType === 3 && flat(container.lastChild.textContent) === "")))
|
|
101
|
+
container.removeChild(container.lastChild);
|
|
102
|
+
changed = true;
|
|
103
|
+
}
|
|
104
|
+
return changed;
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=boilerplate.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { LOADING_LABELS } from "../patterns/loading/index.js";
|
|
2
|
+
import { count, flat, pruneEmpty } from "../shared.js";
|
|
3
|
+
/**
|
|
4
|
+
* A script fills these blocks with content ("wird geladen …", "Loading…"). A reader does not run it, so the placeholder would stay in the
|
|
5
|
+
* story as a line of text. Only a block that holds nothing but the label goes: no link, no picture, no other words.
|
|
6
|
+
*/
|
|
7
|
+
export const loadingPlaceholders = {
|
|
8
|
+
name: "loading-placeholders",
|
|
9
|
+
description: "Blocks that hold only a loading label such as \"wird geladen …\" or \"Loading…\" are removed.",
|
|
10
|
+
apply(root, stats) {
|
|
11
|
+
let changed = false;
|
|
12
|
+
for (const block of [...root.querySelectorAll("p, div, span, li")]) {
|
|
13
|
+
if (!root.contains(block) || block.querySelector("a[href], img, picture, video, audio, iframe"))
|
|
14
|
+
continue;
|
|
15
|
+
if (!LOADING_LABELS.some(pattern => pattern.test(flat(block.textContent))))
|
|
16
|
+
continue;
|
|
17
|
+
// The innermost block that holds the label: a wrapper with more blocks inside is looked at on its own.
|
|
18
|
+
if (block.querySelector("p, div, li"))
|
|
19
|
+
continue;
|
|
20
|
+
const parent = block.parentNode;
|
|
21
|
+
block.remove();
|
|
22
|
+
pruneEmpty(parent);
|
|
23
|
+
count(stats, "loading-placeholders");
|
|
24
|
+
changed = true;
|
|
25
|
+
}
|
|
26
|
+
return changed;
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
//# sourceMappingURL=loading.js.map
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { count, pruneEmpty } from "../shared.js";
|
|
2
|
+
const LIMIT = 2;
|
|
3
|
+
function isPixel(image) {
|
|
4
|
+
const width = Number(image.getAttribute("width"));
|
|
5
|
+
const height = Number(image.getAttribute("height"));
|
|
6
|
+
return width > 0 && height > 0 && width <= LIMIT && height <= LIMIT;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Counting pixels (VG Wort, Chartbeat, publisher analytics) arrive as pictures of one by one pixel inside the text. They show nothing, only
|
|
10
|
+
* a gap in some readers, and every client would load them: they go.
|
|
11
|
+
*/
|
|
12
|
+
export const trackingPixels = {
|
|
13
|
+
name: "tracking-pixels",
|
|
14
|
+
description: "Pictures of one or two pixels in each direction, which only count a view, are removed.",
|
|
15
|
+
apply(root, stats) {
|
|
16
|
+
let changed = false;
|
|
17
|
+
for (const image of [...root.querySelectorAll("img")].filter(isPixel)) {
|
|
18
|
+
const parent = image.parentNode;
|
|
19
|
+
image.remove();
|
|
20
|
+
pruneEmpty(parent);
|
|
21
|
+
count(stats, "tracking-pixels");
|
|
22
|
+
changed = true;
|
|
23
|
+
}
|
|
24
|
+
return changed;
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
//# sourceMappingURL=pixels.js.map
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Blocks, in German, that publishers put into their text without being part of the story. A block counts only when its whole text is one
|
|
3
|
+
* of these lines (see `noise/boilerplate.ts`), so a sentence of the story that contains the words stays.
|
|
4
|
+
*/
|
|
5
|
+
export const ANYWHERE = [
|
|
6
|
+
// Aufrufe, der Seite bei Google oder in anderen Diensten zu folgen.
|
|
7
|
+
/^(?:Uns|Folge uns|Folgen Sie uns) (?:auf|bei) Google(?: News| Discover)? folgen$/i,
|
|
8
|
+
/^(?:Wähle|Wählen Sie|Füge|Fügen Sie) [^.!?]{2,50} (?:als )?(?:bevorzugte|bevorzugten|bevorzugter)\b[^.!?]{0,30}$/i,
|
|
9
|
+
// „★ ifun.de bei Google bevorzugen ★“ und ähnliche Bitten, die Seite als bevorzugte Quelle zu wählen.
|
|
10
|
+
/^★?\s*[^★!?]{2,40} bei Google bevorzugen\s*★?$/i,
|
|
11
|
+
// Texte von Download-Schaltflächen, deren Grafik nicht mehr da ist.
|
|
12
|
+
/^(?:Laden im (?:App|Mac App|Google Play) Store|Im (?:App|Mac App) Store laden|Jetzt bei Google Play|Bei Google Play herunterladen|Laden bei Google Play)$/i,
|
|
13
|
+
// Der Dateiname eines Bildes, den ein Link oder eine Beschriftung als Text zeigt.
|
|
14
|
+
/^[\w\-. ]{3,120}\.(?:jpe?g|png|webp|gif|avif)$/i,
|
|
15
|
+
];
|
|
16
|
+
/** Fußzeilen am Ende des Textes; sie zählen nur in der zweiten Hälfte. */
|
|
17
|
+
export const TRAILING = [
|
|
18
|
+
/^(?:Der Beitrag|Dieser Beitrag|Der Artikel|Dieser Artikel) .{3,200} (?:erschien|erschien zuerst|erschien erstmals) (?:zuerst )?(?:auf|bei) .{2,60}$/i,
|
|
19
|
+
/^Der Beitrag .{3,200} wurde (?:zuerst|erstmals) (?:auf|bei) .{2,60} veröffentlicht\.?$/i,
|
|
20
|
+
/^Diskutiere (?:diesen|den) (?:Artikel|Beitrag) .{0,40}$/i,
|
|
21
|
+
];
|
|
22
|
+
//# sourceMappingURL=de.js.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Blocks, in English, that publishers put into their text without being part of the story. A block counts only when its whole text is one
|
|
3
|
+
* of these lines (see `noise/boilerplate.ts`), so a sentence of the story that contains the words stays.
|
|
4
|
+
*/
|
|
5
|
+
export const ANYWHERE = [
|
|
6
|
+
// Calls to follow the site on Google or elsewhere.
|
|
7
|
+
/^(?:Follow us on Google(?: News| Discover)?|Add [^.!?]{2,50} as a preferred (?:source|publisher)(?: on Google)?)$/i,
|
|
8
|
+
// The text of download buttons whose picture is gone.
|
|
9
|
+
/^(?:Download on the (?:App|Mac App) Store|Get it on Google Play|Available on Google Play)$/i,
|
|
10
|
+
];
|
|
11
|
+
/** Footers at the end of the text; they count only in the second half. */
|
|
12
|
+
export const TRAILING = [
|
|
13
|
+
/^(?:This (?:article|post)|The post),? .{3,200} (?:first )?appeared (?:first )?on .{2,60}$/i,
|
|
14
|
+
/^Discuss this (?:article|story|post)\b.{0,40}$/i,
|
|
15
|
+
// MacRumors: the related links under every article.
|
|
16
|
+
/^Related Roundups?:.{0,300}$/i,
|
|
17
|
+
/^(?:Tag|Tags|Buyer's Guides?|Related Forums?):.{0,200}$/i,
|
|
18
|
+
// The Verge: the box that offers to follow the story's topics and authors.
|
|
19
|
+
/^Follow topics and authors from this story\b.{0,200}$/i,
|
|
20
|
+
];
|
|
21
|
+
//# sourceMappingURL=en.js.map
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import * as de from "./de.js";
|
|
2
|
+
import * as en from "./en.js";
|
|
3
|
+
/** One file per language, searched all at once. To add a language, add a file next to these and list it here. */
|
|
4
|
+
export const BOILERPLATE_ANYWHERE = [de.ANYWHERE, en.ANYWHERE].flat();
|
|
5
|
+
export const BOILERPLATE_TRAILING = [de.TRAILING, en.TRAILING].flat();
|
|
6
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whole-line placeholders that a script replaces with content (a comment box, a gallery, a player). A reader that does not run the
|
|
3
|
+
* script keeps them as text. Only a block that holds nothing else counts, so a sentence that contains the word stays.
|
|
4
|
+
*/
|
|
5
|
+
export default [
|
|
6
|
+
/^(?:loading|please wait)(?:\s*(?:\.{3}|…))?$/i,
|
|
7
|
+
];
|
|
8
|
+
//# sourceMappingURL=en.js.map
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** Headings, in German, that start a list of other articles. See `en.ts` for how they are used. */
|
|
2
2
|
export default [
|
|
3
3
|
/^(?:Mehr zum Thema|Weitere Artikel|Weitere Meldungen|Ähnliche Artikel|Verwandte Artikel|Auch interessant|Das könnte (?:dich|Sie|euch) (?:auch )?interessieren|Meistgelesen(?:e Artikel)?|Beliebte Artikel|Lesen Sie auch|Lies auch|Empfehlungen)$/i,
|
|
4
|
+
// Das Willkommen einer Seite („Willkommen bei GamePro!“) mit Logo und Links am Ende des Textes.
|
|
5
|
+
/^Willkommen bei [^.!?]{2,40}!?$/i,
|
|
4
6
|
];
|
|
5
7
|
//# sourceMappingURL=de.js.map
|
|
@@ -4,5 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
export default [
|
|
6
6
|
/^(?:Popular (?:Stories|Articles|Posts)|Most (?:Read|Popular)|Trending(?: Now| Stories)?|Related (?:Articles|Stories|Posts|Reading|Content)|You (?:might|may) also like|More (?:from|stories|articles)\b.*|Read (?:more|next)|Further reading|Top Stories|Recommended(?: for you)?|Latest (?:Stories|News))$/i,
|
|
7
|
+
// A site's welcome ("Welcome to GamePro!") with logo and links at the end of the text.
|
|
8
|
+
/^Welcome to [^.!?]{2,40}!?$/i,
|
|
7
9
|
];
|
|
8
10
|
//# sourceMappingURL=en.js.map
|
|
@@ -31,9 +31,13 @@ export function canAccessFeed(userId, feedId) {
|
|
|
31
31
|
OR EXISTS (SELECT 1 FROM items i JOIN item_bookmarks b ON b.item_id = i.id WHERE i.feed_id = f.id AND b.user_id = ?)
|
|
32
32
|
OR EXISTS (SELECT 1 FROM items i JOIN item_notes n ON n.item_id = i.id WHERE i.feed_id = f.id AND n.user_id = ?))`).get(feedId, userId, userId, userId));
|
|
33
33
|
}
|
|
34
|
-
|
|
34
|
+
/**
|
|
35
|
+
* Declared icons first, then the stored one, then the well-known files of the site, and last the feed's own logo (`<image>` of an RSS channel):
|
|
36
|
+
* a channel logo is often a wide or transparent banner and no icon, so it only serves when the site offers nothing better.
|
|
37
|
+
*/
|
|
38
|
+
function candidates(feed, declaredUrls, logoUrls) {
|
|
35
39
|
const urls = new Set(declaredUrls);
|
|
36
|
-
if (feed.icon_url)
|
|
40
|
+
if (feed.icon_url && !logoUrls.includes(feed.icon_url))
|
|
37
41
|
urls.add(feed.icon_url);
|
|
38
42
|
for (const source of [feed.site_url, feed.url]) {
|
|
39
43
|
try {
|
|
@@ -47,6 +51,8 @@ function candidates(feed, declaredUrls) {
|
|
|
47
51
|
// Try the next source.
|
|
48
52
|
}
|
|
49
53
|
}
|
|
54
|
+
for (const url of logoUrls)
|
|
55
|
+
urls.add(url);
|
|
50
56
|
return [...urls];
|
|
51
57
|
}
|
|
52
58
|
/** Rejects 8-bit grayscale+alpha or RGBA PNGs whose alpha channel is 0 across every pixel. */
|
|
@@ -160,7 +166,7 @@ export function storeFeedIcon(feedId, buffer, mime) {
|
|
|
160
166
|
}
|
|
161
167
|
/** Re-check remote candidates during the weekly feed poll, even if a stored icon exists. */
|
|
162
168
|
const refreshing = new Map();
|
|
163
|
-
export function refreshFeedIcon(feedId, siteUrl, declaredUrls = [], signal) {
|
|
169
|
+
export function refreshFeedIcon(feedId, siteUrl, declaredUrls = [], signal, logoUrls = []) {
|
|
164
170
|
const current = refreshing.get(feedId);
|
|
165
171
|
if (current)
|
|
166
172
|
return current;
|
|
@@ -169,16 +175,16 @@ export function refreshFeedIcon(feedId, siteUrl, declaredUrls = [], signal) {
|
|
|
169
175
|
const controller = new AbortController();
|
|
170
176
|
const timeout = setTimeout(() => controller.abort(new DOMException("Icon deadline exceeded", "TimeoutError")), readPollSettings().attemptTimeoutMs);
|
|
171
177
|
const deadline = signal ? AbortSignal.any([controller.signal, signal]) : controller.signal;
|
|
172
|
-
const promise = refreshIcon(feedId, siteUrl, declaredUrls, deadline).finally(() => {
|
|
178
|
+
const promise = refreshIcon(feedId, siteUrl, declaredUrls, deadline, logoUrls).finally(() => {
|
|
173
179
|
clearTimeout(timeout);
|
|
174
180
|
refreshing.delete(feedId);
|
|
175
181
|
});
|
|
176
182
|
refreshing.set(feedId, promise);
|
|
177
183
|
return promise;
|
|
178
184
|
}
|
|
179
|
-
async function refreshIcon(feedId, siteUrl, declaredUrls, signal) {
|
|
185
|
+
async function refreshIcon(feedId, siteUrl, declaredUrls, signal, logoUrls) {
|
|
180
186
|
const feed = db.prepare("SELECT icon_url, site_url, url FROM feeds WHERE id = ?").get(feedId);
|
|
181
|
-
for (const url of feed ? candidates({ ...feed, site_url: siteUrl ?? feed.site_url }, declaredUrls) : []) {
|
|
187
|
+
for (const url of feed ? candidates({ ...feed, site_url: siteUrl ?? feed.site_url }, declaredUrls, logoUrls) : []) {
|
|
182
188
|
if (signal.aborted)
|
|
183
189
|
break;
|
|
184
190
|
try {
|
|
@@ -34,24 +34,26 @@ const parser = new Parser({
|
|
|
34
34
|
],
|
|
35
35
|
},
|
|
36
36
|
});
|
|
37
|
+
/** The feed's own addresses: `icon` is meant as an icon, the `image` of an RSS channel is its logo and ranks below the site's icon files. */
|
|
37
38
|
function extractFeedIconUrls(parsed, baseUrl) {
|
|
39
|
+
const result = { icons: [], logos: [] };
|
|
38
40
|
if (!parsed)
|
|
39
|
-
return
|
|
40
|
-
const
|
|
41
|
-
for (const raw of [parsed.icon, parsed.image?.url]) {
|
|
41
|
+
return result;
|
|
42
|
+
const read = (raw, into) => {
|
|
42
43
|
if (typeof raw !== "string" || !raw.trim())
|
|
43
|
-
|
|
44
|
+
return;
|
|
44
45
|
try {
|
|
45
46
|
const resolved = new URL(raw.trim(), baseUrl);
|
|
46
|
-
if (resolved.protocol === "https:" || resolved.protocol === "http:")
|
|
47
|
-
|
|
48
|
-
}
|
|
47
|
+
if (resolved.protocol === "https:" || resolved.protocol === "http:")
|
|
48
|
+
into.push(resolved.toString());
|
|
49
49
|
}
|
|
50
50
|
catch {
|
|
51
51
|
// Ignore malformed feed icon URLs.
|
|
52
52
|
}
|
|
53
|
-
}
|
|
54
|
-
|
|
53
|
+
};
|
|
54
|
+
read(parsed.icon, result.icons);
|
|
55
|
+
read(parsed.image?.url, result.logos);
|
|
56
|
+
return result;
|
|
55
57
|
}
|
|
56
58
|
function cleanImageUrl(url) {
|
|
57
59
|
return url.trim().replace(/&/g, "&").replace(/&/g, "&");
|
|
@@ -157,7 +159,7 @@ async function pollAttempt(feed, signal) {
|
|
|
157
159
|
const htmlUrls = siteUrl ? await discoverIconUrls(siteUrl, signal) : [];
|
|
158
160
|
const feedUrls = extractFeedIconUrls(parsed, siteUrl || feed.url);
|
|
159
161
|
signal.throwIfAborted();
|
|
160
|
-
await refreshFeedIcon(feed.id, siteUrl, [...htmlUrls, ...feedUrls], signal);
|
|
162
|
+
await refreshFeedIcon(feed.id, siteUrl, [...htmlUrls, ...feedUrls.icons], signal, feedUrls.logos);
|
|
161
163
|
}
|
|
162
164
|
}
|
|
163
165
|
catch { /* Keep the successful poll; icon failures are best effort. */ }
|