feedkeeper 0.27.1 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/server/dist/api/items.js +3 -2
- package/server/dist/api/v1/index.js +1 -0
- package/server/dist/api/v1/resources.js +13 -11
- package/server/dist/auth/tokens.js +10 -1
- package/server/dist/db/index.js +4 -0
- package/server/dist/db/migrations/0039_performance_indexes.sql +96 -0
- package/server/dist/db/migrations/0040_empty_gallery.sql +10 -0
- package/server/dist/feeds/archive.js +26 -14
- package/server/dist/feeds/articleContent.js +1 -1
- package/server/dist/feeds/cleanup.js +11 -8
- package/server/dist/feeds/contentNoise.js +24 -17
- package/server/dist/feeds/extractor.js +2 -4
- package/server/dist/feeds/fullTextQueue.js +9 -3
- package/server/dist/feeds/galleryContent.js +33 -0
- package/server/dist/feeds/imageCache.js +10 -0
- package/server/dist/feeds/inbox.js +5 -4
- package/server/dist/feeds/lead.js +13 -5
- package/server/dist/feeds/pollQueue.js +11 -2
- package/server/dist/feeds/poller.js +1 -1
- package/server/dist/feeds/repository.js +112 -19
- package/server/dist/feeds/service.js +4 -4
- package/server/dist/native/editions.js +20 -15
- package/server/dist/oauth/store.js +9 -1
- package/server/dist/push.js +2 -0
- package/server/dist/pushOutbox.js +1 -1
- package/server/dist/sync/changeLog.js +114 -33
- package/server/package.json +1 -1
- package/web/dist/assets/index-CTTNHCfi.js +32 -0
- package/web/dist/index.html +1 -1
- package/web/dist/sw.js +62 -13
- package/web/dist/assets/index-Ci8_fWis.js +0 -32
|
@@ -5,8 +5,8 @@ import { hasUserCapability } from "../auth/capabilities.js";
|
|
|
5
5
|
import { configureFullTextPool, deferFullTextHost } from "./fullTextQueue.js";
|
|
6
6
|
import { extractArticle } from "./extractor.js";
|
|
7
7
|
import { assertPublicHttpUrl, normalizeUrlCandidate } from "./ssrfGuard.js";
|
|
8
|
-
import { articleImageUrls, scheduleArchive } from "./archive.js";
|
|
9
|
-
import { findFeedById, findFeedByUrl, isUserSubscribed,
|
|
8
|
+
import { articleImageUrls, pruneArchive, scheduleArchive } from "./archive.js";
|
|
9
|
+
import { findFeedById, findFeedByUrl, isUserSubscribed, findSubscriptionForUser, subscribe, bookmarkItem, findItemById, findItemForUser, setItemNote, } from "./repository.js";
|
|
10
10
|
function escapeHtml(str) {
|
|
11
11
|
return str
|
|
12
12
|
.replace(/&/g, "&")
|
|
@@ -31,7 +31,7 @@ export function ensureUserInbox(userId) {
|
|
|
31
31
|
if (!isUserSubscribed(userId, feed.id)) {
|
|
32
32
|
subscribe(userId, feed.id, "Inbox");
|
|
33
33
|
}
|
|
34
|
-
const sub =
|
|
34
|
+
const sub = findSubscriptionForUser(userId, feed.id);
|
|
35
35
|
if (!sub) {
|
|
36
36
|
throw new Error("Failed to ensure user inbox subscription");
|
|
37
37
|
}
|
|
@@ -53,7 +53,7 @@ export async function saveToInbox(userId, input) {
|
|
|
53
53
|
try {
|
|
54
54
|
configureFullTextPool();
|
|
55
55
|
const extracted = await extractArticle(targetUrl, undefined, 5, () => hasUserCapability(userId, "fulltext"));
|
|
56
|
-
if (extracted.status === "throttled")
|
|
56
|
+
if (extracted.status === "throttled" && !extracted.deferred)
|
|
57
57
|
deferFullTextHost(extracted.host, extracted.retryAt);
|
|
58
58
|
if (extracted.status === "ok" || extracted.status === "partial") {
|
|
59
59
|
extractionStatus = extracted.status === "ok" ? "ready" : "partial";
|
|
@@ -130,6 +130,7 @@ export function deleteInboxItem(userId, itemId) {
|
|
|
130
130
|
db.prepare("DELETE FROM item_notes WHERE item_id = ?").run(itemId);
|
|
131
131
|
db.prepare("DELETE FROM items WHERE id = ?").run(itemId);
|
|
132
132
|
})();
|
|
133
|
+
pruneArchive();
|
|
133
134
|
return true;
|
|
134
135
|
}
|
|
135
136
|
//# sourceMappingURL=inbox.js.map
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { db } from "../db/index.js";
|
|
3
3
|
import { LEAD_VERSION, leadText } from "./contentNoise.js";
|
|
4
|
+
import { isEmptyGallery } from "./galleryContent.js";
|
|
4
5
|
const BACKFILL_BATCH = 50;
|
|
5
6
|
const BACKFILL_PAUSE_MS = 100;
|
|
6
7
|
const COMPUTED_LIMIT = 300;
|
|
@@ -10,10 +11,17 @@ export function computeLead(item) {
|
|
|
10
11
|
}
|
|
11
12
|
/** Derive and store the lead of one item; call after its title, summary or content was written. */
|
|
12
13
|
export function refreshLead(itemId) {
|
|
13
|
-
const item = db.prepare("SELECT title, content_snippet, content_html, full_content_html FROM items WHERE id = ?").get(itemId);
|
|
14
|
+
const item = db.prepare("SELECT title, link, content_snippet, content_html, full_content_html FROM items WHERE id = ?").get(itemId);
|
|
14
15
|
if (!item)
|
|
15
16
|
return;
|
|
16
|
-
db.prepare("UPDATE items SET content_lead = ?, lead_version = ? WHERE id = ?")
|
|
17
|
+
db.prepare("UPDATE items SET content_lead = ?, lead_version = ?, is_empty_gallery = ? WHERE id = ?")
|
|
18
|
+
.run(computeLead(item), LEAD_VERSION, Number(isEmptyGallery(item)), itemId);
|
|
19
|
+
}
|
|
20
|
+
/** A changed feed summary/link can affect visibility while the lead still belongs to the retained full text. */
|
|
21
|
+
export function refreshGalleryClassification(itemId) {
|
|
22
|
+
const item = db.prepare("SELECT link, content_snippet, content_html, full_content_html FROM items WHERE id = ?").get(itemId);
|
|
23
|
+
if (item)
|
|
24
|
+
db.prepare("UPDATE items SET is_empty_gallery = ? WHERE id = ?").run(Number(isEmptyGallery(item)), itemId);
|
|
17
25
|
}
|
|
18
26
|
/**
|
|
19
27
|
* The lead to serve: the stored one while its rules are current, otherwise derived on the spot, so an article is
|
|
@@ -35,10 +43,10 @@ export function leadOf(item) {
|
|
|
35
43
|
}
|
|
36
44
|
/** Derive leads for items that have none or were derived by older rules; returns how many were handled. */
|
|
37
45
|
export function backfillLeads(limit = BACKFILL_BATCH) {
|
|
38
|
-
const rows = db.prepare("SELECT id, title, content_snippet, content_html, full_content_html FROM items WHERE lead_version IS NULL OR lead_version <> ? LIMIT ?").all(LEAD_VERSION, limit);
|
|
39
|
-
const update = db.prepare("UPDATE items SET content_lead = ?, lead_version = ? WHERE id = ?");
|
|
46
|
+
const rows = db.prepare("SELECT id, title, link, content_snippet, content_html, full_content_html FROM items WHERE lead_version IS NULL OR lead_version <> ? LIMIT ?").all(LEAD_VERSION, limit);
|
|
47
|
+
const update = db.prepare("UPDATE items SET content_lead = ?, lead_version = ?, is_empty_gallery = ? WHERE id = ?");
|
|
40
48
|
db.transaction(() => { for (const row of rows)
|
|
41
|
-
update.run(computeLead(row), LEAD_VERSION, row.id); })();
|
|
49
|
+
update.run(computeLead(row), LEAD_VERSION, Number(isEmptyGallery(row)), row.id); })();
|
|
42
50
|
return rows.length;
|
|
43
51
|
}
|
|
44
52
|
/** Works through the backlog in small steps so polling and requests are never held up for long. */
|
|
@@ -190,8 +190,17 @@ export class PollQueue {
|
|
|
190
190
|
continue;
|
|
191
191
|
}
|
|
192
192
|
// A URL may have been changed since enqueue. Do not dispatch under its old host budget.
|
|
193
|
-
|
|
194
|
-
|
|
193
|
+
let currentHost;
|
|
194
|
+
try {
|
|
195
|
+
currentHost = sourceHost(feed.url);
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
db.prepare("DELETE FROM feed_poll_jobs WHERE feed_id = ?").run(job.feed_id);
|
|
199
|
+
this.resolve(job.feed_id, { newItems: 0, error: "invalid_feed_url" });
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
if (currentHost !== job.host) {
|
|
203
|
+
db.prepare("UPDATE feed_poll_jobs SET host = ? WHERE feed_id = ?").run(currentHost, feed.id);
|
|
195
204
|
continue;
|
|
196
205
|
}
|
|
197
206
|
const waiter = this.waiters.get(feed.id);
|
|
@@ -132,10 +132,10 @@ async function pollAttempt(feed, signal) {
|
|
|
132
132
|
try {
|
|
133
133
|
signal.throwIfAborted();
|
|
134
134
|
if (iconCheckDue(feed.icon_checked_at)) {
|
|
135
|
+
updateFeedIcon(feed.id, feed.icon_url);
|
|
135
136
|
const siteUrl = parsed?.link || feed.site_url;
|
|
136
137
|
const urls = siteUrl ? await discoverIconUrls(siteUrl, signal) : [];
|
|
137
138
|
signal.throwIfAborted();
|
|
138
|
-
updateFeedIcon(feed.id, feed.icon_url);
|
|
139
139
|
await refreshFeedIcon(feed.id, siteUrl, urls, signal);
|
|
140
140
|
}
|
|
141
141
|
}
|
|
@@ -2,7 +2,7 @@ import { db } from "../db/index.js";
|
|
|
2
2
|
import { articleText } from "./articleContent.js";
|
|
3
3
|
import { LEAD_VERSION } from "./contentNoise.js";
|
|
4
4
|
import { largerHeroImage } from "./heroImage.js";
|
|
5
|
-
import { refreshLead } from "./lead.js";
|
|
5
|
+
import { refreshGalleryClassification, refreshLead } from "./lead.js";
|
|
6
6
|
import { decodeEntities } from "./text.js";
|
|
7
7
|
export function findFeedByUrl(url) {
|
|
8
8
|
return db.prepare("SELECT * FROM feeds WHERE url = ?").get(url);
|
|
@@ -43,8 +43,8 @@ export function removeFeedIfUnused(feedId) {
|
|
|
43
43
|
if (countFeedSubscribers(feedId) > 0)
|
|
44
44
|
return;
|
|
45
45
|
db.prepare(`DELETE FROM items WHERE feed_id = ? AND id NOT IN (SELECT item_id FROM item_bookmarks) AND id NOT IN (SELECT item_id FROM item_notes)
|
|
46
|
-
AND NOT
|
|
47
|
-
WHERE
|
|
46
|
+
AND id NOT IN (SELECT CAST(selected.value AS INTEGER) FROM editions e, json_each(e.item_ids) selected
|
|
47
|
+
WHERE e.source = 'curated' AND e.status <> 'dismissed')`).run(feedId);
|
|
48
48
|
db.prepare("DELETE FROM feeds WHERE id = ? AND is_system_inbox = 0 AND NOT EXISTS (SELECT 1 FROM items WHERE feed_id = ?)").run(feedId, feedId);
|
|
49
49
|
}
|
|
50
50
|
/** Feeds kept only for saved articles or annotated articles whose last bookmark and note are gone. */
|
|
@@ -54,8 +54,10 @@ export function removeUnusedFeeds() {
|
|
|
54
54
|
AND NOT EXISTS (SELECT 1 FROM subscriptions s WHERE s.feed_id = feeds.id)
|
|
55
55
|
AND NOT EXISTS (SELECT 1 FROM items i JOIN item_bookmarks b ON b.item_id = i.id WHERE i.feed_id = feeds.id)
|
|
56
56
|
AND NOT EXISTS (SELECT 1 FROM items i JOIN item_notes n ON n.item_id = i.id WHERE i.feed_id = feeds.id)
|
|
57
|
-
AND NOT EXISTS (SELECT 1 FROM items i
|
|
58
|
-
|
|
57
|
+
AND NOT EXISTS (SELECT 1 FROM items i WHERE i.feed_id = feeds.id AND i.id IN (
|
|
58
|
+
SELECT CAST(selected.value AS INTEGER) FROM editions e, json_each(e.item_ids) selected
|
|
59
|
+
WHERE e.source = 'curated' AND e.status <> 'dismissed'
|
|
60
|
+
))`).run().changes;
|
|
59
61
|
}
|
|
60
62
|
/** Subscribers can open every item of their feeds; anyone can open what they saved or added a note to. */
|
|
61
63
|
export function canAccessItem(userId, itemId) {
|
|
@@ -168,7 +170,7 @@ export function upsertItems(feedId, items) {
|
|
|
168
170
|
refreshLead(Number(result.lastInsertRowid));
|
|
169
171
|
}
|
|
170
172
|
else {
|
|
171
|
-
const previous = db.prepare("SELECT id, title, content_snippet, content_html, full_content_html, lead_version FROM items WHERE feed_id = ? AND guid = ?").get(feedId, row.guid);
|
|
173
|
+
const previous = db.prepare("SELECT id, title, link, image_url, content_snippet, content_html, full_content_html, lead_version FROM items WHERE feed_id = ? AND guid = ?").get(feedId, row.guid);
|
|
172
174
|
const next = row.contentHtml;
|
|
173
175
|
// Some publishers replace complete feed bodies with short teasers on a later poll.
|
|
174
176
|
const retained = previous?.content_html && next && next.length < previous.content_html.length * 0.9 &&
|
|
@@ -176,14 +178,30 @@ export function upsertItems(feedId, items) {
|
|
|
176
178
|
// The summary belongs to the body it was sent with: keeping the longer body keeps its summary too, or
|
|
177
179
|
// a feed that alternates between two variants would change the content on every poll.
|
|
178
180
|
const keptPrevious = retained !== null && next !== retained;
|
|
179
|
-
|
|
180
|
-
|
|
181
|
+
const nextSnippet = keptPrevious ? null : row.contentSnippet ?? null;
|
|
182
|
+
const nextImage = largerHeroImage(row.imageUrl, retained, row.link ?? previous?.link ?? null) ?? previous?.image_url ?? row.imageUrl ?? null;
|
|
181
183
|
if (previous) {
|
|
182
|
-
const changed = (
|
|
183
|
-
(row.
|
|
184
|
-
(!keptPrevious && row.contentSnippet !== undefined && row.contentSnippet !== previous.content_snippet)
|
|
185
|
-
|
|
184
|
+
const changed = (row.title !== undefined && row.title !== previous.title) ||
|
|
185
|
+
(row.link !== undefined && row.link !== previous.link) ||
|
|
186
|
+
(!keptPrevious && row.contentSnippet !== undefined && row.contentSnippet !== previous.content_snippet) ||
|
|
187
|
+
(retained !== null && retained !== previous.content_html) ||
|
|
188
|
+
(row.publishedAt !== undefined && row.publishedAt !== null) ||
|
|
189
|
+
(nextImage !== previous.image_url) ||
|
|
190
|
+
(row.author !== undefined);
|
|
191
|
+
if (changed) {
|
|
192
|
+
update.run(row.title ?? null, row.link ?? null, nextSnippet, retained, row.publishedAt ?? null, nextImage, row.author ?? null, feedId, row.guid);
|
|
193
|
+
const leadChanged = (retained !== null && retained !== previous.content_html) ||
|
|
194
|
+
(row.link !== undefined && row.link !== previous.link) ||
|
|
195
|
+
(row.title !== undefined && row.title !== previous.title) ||
|
|
196
|
+
(!keptPrevious && row.contentSnippet !== undefined && row.contentSnippet !== previous.content_snippet);
|
|
197
|
+
if (previous.lead_version !== LEAD_VERSION || (leadChanged && !previous.full_content_html))
|
|
198
|
+
refreshLead(previous.id);
|
|
199
|
+
else if (leadChanged)
|
|
200
|
+
refreshGalleryClassification(previous.id);
|
|
201
|
+
}
|
|
202
|
+
else if (previous.lead_version !== LEAD_VERSION && !previous.full_content_html) {
|
|
186
203
|
refreshLead(previous.id);
|
|
204
|
+
}
|
|
187
205
|
}
|
|
188
206
|
}
|
|
189
207
|
}
|
|
@@ -193,8 +211,18 @@ export function upsertItems(feedId, items) {
|
|
|
193
211
|
}
|
|
194
212
|
export function listItemsForUser(userId, opts = {}) {
|
|
195
213
|
// Saved articles stay visible without a subscription (e.g. after unsubscribing).
|
|
196
|
-
const conditions = [
|
|
214
|
+
const conditions = [];
|
|
215
|
+
if (opts.bookmarkedOnly) {
|
|
216
|
+
conditions.push("b.item_id IS NOT NULL");
|
|
217
|
+
}
|
|
218
|
+
else {
|
|
219
|
+
conditions.push("(s.id IS NOT NULL OR b.item_id IS NOT NULL)");
|
|
220
|
+
}
|
|
197
221
|
const params = [];
|
|
222
|
+
// Keep the stored item available to direct reads and sync; deliberate saves/notes override stream filtering.
|
|
223
|
+
conditions.push(`(i.is_empty_gallery = 0 OR b.item_id IS NOT NULL OR
|
|
224
|
+
EXISTS (SELECT 1 FROM item_notes gallery_note WHERE gallery_note.item_id = i.id AND gallery_note.user_id = ?))`);
|
|
225
|
+
params.push(userId);
|
|
198
226
|
if (opts.feedId) {
|
|
199
227
|
conditions.push("i.feed_id = ?");
|
|
200
228
|
params.push(opts.feedId);
|
|
@@ -297,6 +325,51 @@ export function findItemForUser(userId, itemId) {
|
|
|
297
325
|
EXISTS (SELECT 1 FROM editions e, json_each(e.item_ids) selected WHERE e.user_id = ?
|
|
298
326
|
AND selected.value = i.id AND e.source = 'curated' AND e.status <> 'dismissed'))`).get(userId, userId, userId, userId, itemId, userId);
|
|
299
327
|
}
|
|
328
|
+
export function findItemsForUser(userId, itemIds) {
|
|
329
|
+
if (!itemIds.length)
|
|
330
|
+
return [];
|
|
331
|
+
const unique = [...new Set(itemIds)];
|
|
332
|
+
return db.prepare(`SELECT i.*, COALESCE(NULLIF(TRIM(s.label), ''), NULLIF(TRIM(f.title), ''), f.url) AS feed_title,
|
|
333
|
+
f.site_url AS feed_site_url, f.url AS feed_url, s.full_text_mode AS feed_full_text_mode, f.icon_url AS feed_icon_url,
|
|
334
|
+
(r.item_id IS NOT NULL) AS read, (b.item_id IS NOT NULL) AS bookmarked, b.created_at AS bookmarked_at
|
|
335
|
+
FROM items i
|
|
336
|
+
JOIN feeds f ON f.id = i.feed_id
|
|
337
|
+
LEFT JOIN subscriptions s ON s.feed_id = i.feed_id AND s.user_id = ?
|
|
338
|
+
LEFT JOIN item_reads r ON r.item_id = i.id AND r.user_id = ?
|
|
339
|
+
LEFT JOIN item_bookmarks b ON b.item_id = i.id AND b.user_id = ?
|
|
340
|
+
LEFT JOIN item_notes n ON n.item_id = i.id AND n.user_id = ?
|
|
341
|
+
WHERE i.id IN (SELECT CAST(value AS INTEGER) FROM json_each(?))
|
|
342
|
+
AND (s.id IS NOT NULL OR b.item_id IS NOT NULL OR n.item_id IS NOT NULL OR
|
|
343
|
+
EXISTS (SELECT 1 FROM editions e, json_each(e.item_ids) selected WHERE e.user_id = ?
|
|
344
|
+
AND selected.value = i.id AND e.source = 'curated' AND e.status <> 'dismissed'))`).all(userId, userId, userId, userId, JSON.stringify(unique), userId);
|
|
345
|
+
}
|
|
346
|
+
export function findSubscriptionForUser(userId, feedId) {
|
|
347
|
+
return db
|
|
348
|
+
.prepare(`SELECT
|
|
349
|
+
f.*,
|
|
350
|
+
fi.hash AS icon_hash,
|
|
351
|
+
s.id AS subscription_id,
|
|
352
|
+
NULLIF(TRIM(s.label), '') AS label,
|
|
353
|
+
s.folder_id AS folder_id,
|
|
354
|
+
fo.name AS folder_name,
|
|
355
|
+
s.position AS position,
|
|
356
|
+
s.full_text_mode AS full_text_mode,
|
|
357
|
+
s.notify AS notify,
|
|
358
|
+
s.badge AS badge,
|
|
359
|
+
(
|
|
360
|
+
SELECT COUNT(*) FROM items i
|
|
361
|
+
WHERE i.feed_id = f.id
|
|
362
|
+
AND NOT EXISTS (
|
|
363
|
+
SELECT 1 FROM item_reads r WHERE r.item_id = i.id AND r.user_id = ?
|
|
364
|
+
)
|
|
365
|
+
) AS unread_count
|
|
366
|
+
FROM subscriptions s
|
|
367
|
+
JOIN feeds f ON f.id = s.feed_id
|
|
368
|
+
LEFT JOIN feed_icons fi ON fi.feed_id = f.id
|
|
369
|
+
LEFT JOIN folders fo ON fo.id = s.folder_id
|
|
370
|
+
WHERE s.user_id = ? AND s.feed_id = ?`)
|
|
371
|
+
.get(userId, userId, feedId) ?? null;
|
|
372
|
+
}
|
|
300
373
|
export function updateSubscriptionLabel(userId, feedId, label) {
|
|
301
374
|
db.prepare("UPDATE subscriptions SET label = ? WHERE user_id = ? AND feed_id = ?").run(label?.trim() || null, userId, feedId);
|
|
302
375
|
}
|
|
@@ -498,6 +571,7 @@ export function repairEncodedText() {
|
|
|
498
571
|
const snippet = decodeEntities(row.content_snippet);
|
|
499
572
|
if (title !== row.title || snippet !== row.content_snippet) {
|
|
500
573
|
update.run(title ?? null, snippet ?? null, row.id);
|
|
574
|
+
refreshLead(row.id);
|
|
501
575
|
fixed++;
|
|
502
576
|
}
|
|
503
577
|
}
|
|
@@ -620,7 +694,8 @@ export function needsSubstringSearch(raw) {
|
|
|
620
694
|
}
|
|
621
695
|
export function searchItemsForUser(userId, opts) {
|
|
622
696
|
const fts = needsSubstringSearch(opts.query) ? "" : formatFtsQuery(opts.query);
|
|
623
|
-
const conditions = ["(s.id IS NOT NULL OR b.item_id IS NOT NULL OR n.user_id IS NOT NULL)"
|
|
697
|
+
const conditions = ["(s.id IS NOT NULL OR b.item_id IS NOT NULL OR n.user_id IS NOT NULL)",
|
|
698
|
+
"(i.is_empty_gallery = 0 OR b.item_id IS NOT NULL OR n.user_id IS NOT NULL)"];
|
|
624
699
|
const params = [];
|
|
625
700
|
if (opts.feedId) {
|
|
626
701
|
conditions.push("i.feed_id = ?");
|
|
@@ -678,9 +753,24 @@ export function searchItemsForUser(userId, opts) {
|
|
|
678
753
|
}
|
|
679
754
|
const ranked = opts.sort !== "date" && fts;
|
|
680
755
|
// Materialize once: a virtual-table MATCH in the LEFT JOIN repeats the full ranking scan for each article.
|
|
756
|
+
// Page IDs before loading HTML to avoid copying large content blobs into temporary B-trees during sorting.
|
|
681
757
|
const querySql = `
|
|
682
|
-
${ranked ? "WITH ranked_matches AS MATERIALIZED (SELECT rowid, rank FROM items_fts WHERE items_fts MATCH ?)" : ""}
|
|
683
|
-
|
|
758
|
+
${ranked ? "WITH ranked_matches AS MATERIALIZED (SELECT rowid, rank FROM items_fts WHERE items_fts MATCH ?)," : "WITH"}
|
|
759
|
+
page AS (
|
|
760
|
+
SELECT i.id
|
|
761
|
+
FROM items i
|
|
762
|
+
JOIN feeds f ON f.id = i.feed_id
|
|
763
|
+
LEFT JOIN subscriptions s ON s.feed_id = i.feed_id AND s.user_id = ?
|
|
764
|
+
LEFT JOIN item_reads r ON r.item_id = i.id AND r.user_id = ?
|
|
765
|
+
LEFT JOIN item_bookmarks b ON b.item_id = i.id AND b.user_id = ?
|
|
766
|
+
LEFT JOIN item_notes n ON n.item_id = i.id AND n.user_id = ?
|
|
767
|
+
${ranked ? "LEFT JOIN ranked_matches fts ON fts.rowid = i.id" : ""}
|
|
768
|
+
WHERE ${conditions.join(" AND ")}
|
|
769
|
+
GROUP BY i.id
|
|
770
|
+
ORDER BY ${orderBy}
|
|
771
|
+
LIMIT ? OFFSET ?
|
|
772
|
+
)
|
|
773
|
+
SELECT i.*,
|
|
684
774
|
COALESCE(NULLIF(TRIM(s.label), ''), NULLIF(TRIM(f.title), ''), f.url) AS feed_title,
|
|
685
775
|
f.site_url AS feed_site_url,
|
|
686
776
|
f.url AS feed_url,
|
|
@@ -689,16 +779,15 @@ export function searchItemsForUser(userId, opts) {
|
|
|
689
779
|
(r.item_id IS NOT NULL) AS read,
|
|
690
780
|
(b.item_id IS NOT NULL) AS bookmarked,
|
|
691
781
|
b.created_at AS bookmarked_at
|
|
692
|
-
FROM
|
|
782
|
+
FROM page
|
|
783
|
+
JOIN items i ON i.id = page.id
|
|
693
784
|
JOIN feeds f ON f.id = i.feed_id
|
|
694
785
|
LEFT JOIN subscriptions s ON s.feed_id = i.feed_id AND s.user_id = ?
|
|
695
786
|
LEFT JOIN item_reads r ON r.item_id = i.id AND r.user_id = ?
|
|
696
787
|
LEFT JOIN item_bookmarks b ON b.item_id = i.id AND b.user_id = ?
|
|
697
788
|
LEFT JOIN item_notes n ON n.item_id = i.id AND n.user_id = ?
|
|
698
789
|
${ranked ? "LEFT JOIN ranked_matches fts ON fts.rowid = i.id" : ""}
|
|
699
|
-
WHERE ${conditions.join(" AND ")}
|
|
700
790
|
ORDER BY ${orderBy}
|
|
701
|
-
LIMIT ? OFFSET ?
|
|
702
791
|
`;
|
|
703
792
|
const queryParams = [
|
|
704
793
|
...(ranked ? [fts] : []),
|
|
@@ -709,6 +798,10 @@ export function searchItemsForUser(userId, opts) {
|
|
|
709
798
|
...params,
|
|
710
799
|
limit,
|
|
711
800
|
offset,
|
|
801
|
+
userId,
|
|
802
|
+
userId,
|
|
803
|
+
userId,
|
|
804
|
+
userId,
|
|
712
805
|
];
|
|
713
806
|
const items = db.prepare(querySql).all(...queryParams);
|
|
714
807
|
return { items, total };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { config } from "../config.js";
|
|
2
|
-
import { createFeed, findFeedByUrl, isUserSubscribed, subscribe, unsubscribe as unsubscribeRepo,
|
|
2
|
+
import { createFeed, findFeedByUrl, isUserSubscribed, subscribe, unsubscribe as unsubscribeRepo, findSubscriptionForUser, updateSubscriptionLabel, updateSubscriptionFolder, updateFeedPollInterval, updateSubscriptionFullTextMode, updateSubscriptionNotify, updateSubscriptionBadge, findFolderById, findFeedById, countFeedSubscribers, replaceFeedUrl, moveSubscription, } from "./repository.js";
|
|
3
3
|
import { pollFeed, probeFeed } from "./poller.js";
|
|
4
4
|
import { assertPublicHttpUrl } from "./ssrfGuard.js";
|
|
5
5
|
import { discoverFeeds } from "./discovery.js";
|
|
@@ -66,7 +66,7 @@ export async function subscribeToFeed(userId, inputUrl, label = null, folderId =
|
|
|
66
66
|
unsubscribeRepo(userId, feed.id);
|
|
67
67
|
throw new NoFeedsFoundError();
|
|
68
68
|
}
|
|
69
|
-
const subscription =
|
|
69
|
+
const subscription = findSubscriptionForUser(userId, feed.id);
|
|
70
70
|
if (!subscription)
|
|
71
71
|
throw new FeedError("subscription_lookup_failed");
|
|
72
72
|
return subscription;
|
|
@@ -109,7 +109,7 @@ export function updateFeedSettings(userId, feedId, settings) {
|
|
|
109
109
|
const minutes = Math.max(settings.pollIntervalMinutes, config.minPollIntervalMinutes);
|
|
110
110
|
updateFeedPollInterval(feedId, minutes);
|
|
111
111
|
}
|
|
112
|
-
const subscription =
|
|
112
|
+
const subscription = findSubscriptionForUser(userId, feedId);
|
|
113
113
|
if (!subscription)
|
|
114
114
|
throw new FeedError("subscription_lookup_failed");
|
|
115
115
|
return subscription;
|
|
@@ -127,7 +127,7 @@ export async function changeFeedUrl(userId, feedId, inputUrl) {
|
|
|
127
127
|
const { url } = await resolveFeedUrl(inputUrl);
|
|
128
128
|
let target = feedId;
|
|
129
129
|
const lookup = () => {
|
|
130
|
-
const subscription =
|
|
130
|
+
const subscription = findSubscriptionForUser(userId, target);
|
|
131
131
|
if (!subscription)
|
|
132
132
|
throw new FeedError("subscription_lookup_failed");
|
|
133
133
|
return subscription;
|
|
@@ -108,11 +108,11 @@ export function publishCuratedEdition(userId, input, now = new Date()) {
|
|
|
108
108
|
for (const id of data.itemIds) {
|
|
109
109
|
if (!canAccessItem(userId, id))
|
|
110
110
|
throw new NativeError("item_not_found", 404);
|
|
111
|
-
const item = db.prepare(`SELECT s.folder_id, s.id IS NOT NULL AS subscribed,
|
|
111
|
+
const item = db.prepare(`SELECT s.folder_id, s.id IS NOT NULL AS subscribed, i.is_empty_gallery,
|
|
112
112
|
EXISTS (SELECT 1 FROM user_muted_keywords m WHERE m.user_id = ? AND
|
|
113
113
|
(instr(lower(coalesce(i.title, '')), lower(m.keyword)) > 0 OR instr(lower(coalesce(i.content_snippet, '')), lower(m.keyword)) > 0)) AS muted
|
|
114
114
|
FROM items i LEFT JOIN subscriptions s ON s.feed_id = i.feed_id AND s.user_id = ? WHERE i.id = ?`).get(userId, userId, id);
|
|
115
|
-
if (item.muted || (item.subscribed && (item.folder_id === null ? !prefs.showUnfiled : prefs.hiddenFolderIds.includes(item.folder_id)))) {
|
|
115
|
+
if (item.is_empty_gallery || item.muted || (item.subscribed && (item.folder_id === null ? !prefs.showUnfiled : prefs.hiddenFolderIds.includes(item.folder_id)))) {
|
|
116
116
|
throw new NativeError("item_excluded");
|
|
117
117
|
}
|
|
118
118
|
}
|
|
@@ -214,19 +214,24 @@ export function editionCandidates(userId, now = new Date(), limit = 200) {
|
|
|
214
214
|
const previousItemIds = db.prepare("SELECT item_ids FROM edition_history WHERE user_id = ? AND created_at >= ?").all(userId, new Date(now.getTime() - 72 * 3_600_000).toISOString()).flatMap((row) => JSON.parse(row.item_ids));
|
|
215
215
|
const previous = new Set(previousItemIds);
|
|
216
216
|
// Fetch bounded candidates per source, so a busy feed cannot hide the others.
|
|
217
|
-
const rows = db.prepare(`SELECT
|
|
218
|
-
|
|
219
|
-
coalesce(
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
217
|
+
const rows = db.prepare(`SELECT r.id, r.feed_id, r.folder_id, r.title, r.link, r.content_snippet, r.image_url,
|
|
218
|
+
r.source_name, r.folder_name, r.published, r.source_rank,
|
|
219
|
+
length(coalesce(full_item.full_content_html, full_item.content_html, r.content_snippet, '')) AS content_chars
|
|
220
|
+
FROM (
|
|
221
|
+
SELECT i.id, i.feed_id, s.folder_id, i.title, i.link, i.content_snippet, i.image_url,
|
|
222
|
+
coalesce(s.label, f.title) AS source_name, folders.name AS folder_name,
|
|
223
|
+
strftime('%Y-%m-%dT%H:%M:%fZ', coalesce(julianday(i.published_at), julianday(i.created_at))) AS published,
|
|
224
|
+
row_number() OVER (PARTITION BY i.feed_id ORDER BY coalesce(julianday(i.published_at), julianday(i.created_at)) DESC, i.id DESC) AS source_rank
|
|
225
|
+
FROM items i JOIN subscriptions s ON s.feed_id = i.feed_id AND s.user_id = ?
|
|
226
|
+
JOIN feeds f ON f.id = i.feed_id LEFT JOIN folders ON folders.id = s.folder_id
|
|
227
|
+
WHERE i.is_empty_gallery = 0
|
|
228
|
+
AND NOT EXISTS (SELECT 1 FROM item_reads r WHERE r.item_id = i.id AND r.user_id = ?)
|
|
229
|
+
AND NOT EXISTS (SELECT 1 FROM user_muted_keywords m WHERE m.user_id = ? AND
|
|
230
|
+
(instr(lower(coalesce(i.title, '')), lower(m.keyword)) > 0 OR instr(lower(coalesce(i.content_snippet, '')), lower(m.keyword)) > 0))
|
|
231
|
+
AND coalesce(julianday(i.published_at), julianday(i.created_at)) BETWEEN julianday(?) AND julianday(?)
|
|
232
|
+
) r
|
|
233
|
+
JOIN items full_item ON full_item.id = r.id
|
|
234
|
+
WHERE r.source_rank <= 40 ORDER BY r.source_rank, r.published DESC, r.id DESC LIMIT 4000`)
|
|
230
235
|
.all(userId, userId, userId, new Date(now.getTime() - 7 * 86_400_000).toISOString(), now.toISOString());
|
|
231
236
|
const scored = rows.filter((row) => row.folder_id === null ? prefs.showUnfiled : !prefs.hiddenFolderIds.includes(row.folder_id))
|
|
232
237
|
.map((row) => ({
|
|
@@ -72,13 +72,20 @@ export function rotateGrant(refreshToken, clientId) {
|
|
|
72
72
|
return { ok: true, scope: grant.scope, tokens: { accessToken, refreshToken: newRefresh, expiresIn: ACCESS_TTL_MS / 1000 } };
|
|
73
73
|
})();
|
|
74
74
|
}
|
|
75
|
+
const grantLastTouch = new Map();
|
|
76
|
+
const GRANT_TOUCH_INTERVAL_MS = 5 * 60 * 1000;
|
|
75
77
|
export function resolveAccessToken(token) {
|
|
76
78
|
const row = db
|
|
77
79
|
.prepare("SELECT * FROM oauth_grants WHERE access_hash = ?")
|
|
78
80
|
.get(hash(token));
|
|
79
81
|
if (!row || row.access_expires_at <= now())
|
|
80
82
|
return null;
|
|
81
|
-
|
|
83
|
+
const nowMs = Date.now();
|
|
84
|
+
const lastTouch = grantLastTouch.get(row.id) ?? 0;
|
|
85
|
+
if (!row.last_used_at || nowMs - lastTouch >= GRANT_TOUCH_INTERVAL_MS) {
|
|
86
|
+
grantLastTouch.set(row.id, nowMs);
|
|
87
|
+
db.prepare("UPDATE oauth_grants SET last_used_at = ? WHERE id = ?").run(now(), row.id);
|
|
88
|
+
}
|
|
82
89
|
return { userId: row.user_id, scope: row.scope, grantId: row.id };
|
|
83
90
|
}
|
|
84
91
|
/** Revokes by either token. Returns whether a grant was removed. */
|
|
@@ -89,6 +96,7 @@ export function revokeByToken(token, clientId) {
|
|
|
89
96
|
.get(tokenHash, tokenHash);
|
|
90
97
|
if (!grant || (clientId && grant.client_id !== clientId))
|
|
91
98
|
return false;
|
|
99
|
+
grantLastTouch.delete(grant.id);
|
|
92
100
|
return db.prepare("DELETE FROM oauth_grants WHERE id = ?").run(grant.id).changes > 0;
|
|
93
101
|
}
|
|
94
102
|
export function listGrantsForUser(userId) {
|
package/server/dist/push.js
CHANGED
|
@@ -112,6 +112,8 @@ export function newItemsPushPayload(feed, userId, label, latest) {
|
|
|
112
112
|
const muted = db.prepare("SELECT keyword FROM user_muted_keywords WHERE user_id = ?")
|
|
113
113
|
.all(userId).map(row => row.keyword.toLowerCase());
|
|
114
114
|
const fresh = latest.filter(item => {
|
|
115
|
+
if (item.is_empty_gallery)
|
|
116
|
+
return false;
|
|
115
117
|
const text = `${item.title ?? ""} ${item.content_snippet ?? ""}`.toLowerCase();
|
|
116
118
|
return !muted.some(keyword => text.includes(keyword));
|
|
117
119
|
});
|
|
@@ -111,7 +111,7 @@ export class PushOutbox {
|
|
|
111
111
|
if (!feed)
|
|
112
112
|
return; // Feed deletion cascades the event.
|
|
113
113
|
const latest = db.prepare(`
|
|
114
|
-
SELECT id,title,content_snippet FROM items WHERE feed_id = ? AND id <= ? ORDER BY id DESC LIMIT ?`)
|
|
114
|
+
SELECT id,title,content_snippet,is_empty_gallery FROM items WHERE feed_id = ? AND id <= ? ORDER BY id DESC LIMIT ?`)
|
|
115
115
|
.all(event.feed_id, event.max_item_id, event.item_count);
|
|
116
116
|
const payload = newItemsPushPayload(feed, subscription.user_id, subscription.label, latest);
|
|
117
117
|
if (!payload) {
|