feedkeeper 0.23.1 → 0.24.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,9 @@
1
1
  import { isCachedConsentSnippet } from "./extractor.js";
2
2
  import { db } from "../db/index.js";
3
3
  import { hasUserCapability } from "../auth/capabilities.js";
4
- import { loadFullText } from "./fullText.js";
4
+ import { loadFullText, executeFullText } from "./fullText.js";
5
+ import { articleWorkPool, restartArticleWorkPool } from "./articleWorkPool.js";
6
+ import { sourceHost } from "./requestGate.js";
5
7
  import { canAccessItem, findSubscriptionFullTextMode, findItemById } from "./repository.js";
6
8
  /** Refillable durable queue, bounded independently of library size. */
7
9
  export function scheduleFullTexts() {
@@ -18,6 +20,7 @@ export function scheduleFullTexts() {
18
20
  FROM editions e, json_each(e.item_ids) ids
19
21
  JOIN items i ON i.id = ids.value
20
22
  WHERE e.user_id IN (SELECT value FROM json_each(?))
23
+ AND NOT EXISTS (SELECT 1 FROM subscriptions disabled WHERE disabled.user_id=e.user_id AND disabled.feed_id=i.feed_id AND disabled.full_text_mode='never')
21
24
  AND e.status = 'active' AND e.expires_at > strftime('%Y-%m-%dT%H:%M:%fZ','now')
22
25
  AND i.link IS NOT NULL AND i.full_content_html IS NULL
23
26
  AND i.extraction_status != 'feed_only'
@@ -30,6 +33,7 @@ export function scheduleFullTexts() {
30
33
  FROM item_bookmarks b
31
34
  JOIN items i ON i.id = b.item_id
32
35
  WHERE b.user_id IN (SELECT value FROM json_each(?))
36
+ AND NOT EXISTS (SELECT 1 FROM subscriptions disabled WHERE disabled.user_id=b.user_id AND disabled.feed_id=i.feed_id AND disabled.full_text_mode='never')
33
37
  AND i.link IS NOT NULL AND i.full_content_html IS NULL
34
38
  AND i.extraction_status != 'feed_only'
35
39
  AND (i.extraction_retry_at IS NULL OR i.extraction_retry_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now'))
@@ -41,6 +45,7 @@ export function scheduleFullTexts() {
41
45
  FROM item_notes n
42
46
  JOIN items i ON i.id = n.item_id
43
47
  WHERE n.user_id IN (SELECT value FROM json_each(?))
48
+ AND NOT EXISTS (SELECT 1 FROM subscriptions disabled WHERE disabled.user_id=n.user_id AND disabled.feed_id=i.feed_id AND disabled.full_text_mode='never')
44
49
  AND i.link IS NOT NULL AND i.full_content_html IS NULL
45
50
  AND i.extraction_status != 'feed_only'
46
51
  AND (i.extraction_retry_at IS NULL OR i.extraction_retry_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now'))
@@ -61,8 +66,8 @@ export function scheduleFullTexts() {
61
66
  SELECT item_id AS id, user_id, MAX(priority) AS priority
62
67
  FROM candidate_articles
63
68
  GROUP BY item_id
64
- ORDER BY priority DESC, created_at DESC LIMIT ?`).all(JSON.stringify(eligible), JSON.stringify(eligible), JSON.stringify(eligible), JSON.stringify(eligible), room);
65
- const add = db.prepare("INSERT OR IGNORE INTO full_text_jobs(item_id, user_id, priority) VALUES (?, ?, ?)");
69
+ ORDER BY (priority + CAST((julianday('now')-julianday(created_at))*1440 / 60 AS INTEGER)) DESC, created_at ASC LIMIT ?`).all(JSON.stringify(eligible), JSON.stringify(eligible), JSON.stringify(eligible), JSON.stringify(eligible), room);
70
+ const add = db.prepare("INSERT OR IGNORE INTO full_text_jobs(item_id, user_id, priority, created_at) VALUES (?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ','now'))");
66
71
  let count = 0;
67
72
  db.transaction(() => {
68
73
  for (const row of candidates)
@@ -81,10 +86,7 @@ export function enqueueArticleContents(userId, ids) {
81
86
  if (unique.some(id => !canAccessItem(userId, id)))
82
87
  throw new Error("item_not_found");
83
88
  return db.transaction(() => {
84
- const count = db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get();
85
- if (count.n + unique.length > 1000)
86
- throw new Error("queue_full");
87
- const add = db.prepare("INSERT INTO full_text_jobs(item_id,user_id,priority) VALUES (?,?,4) ON CONFLICT(item_id) DO UPDATE SET priority = MAX(priority,4), user_id = excluded.user_id");
89
+ const add = db.prepare("INSERT INTO full_text_jobs(item_id,user_id,priority,created_at) VALUES (?,?,4,strftime('%Y-%m-%dT%H:%M:%fZ','now')) ON CONFLICT(item_id) DO UPDATE SET priority = MAX(priority,4), user_id = excluded.user_id");
88
90
  const queued = [];
89
91
  for (const id of unique) {
90
92
  const item = findItemById(id);
@@ -92,13 +94,15 @@ export function enqueueArticleContents(userId, ids) {
92
94
  continue;
93
95
  if (item.extraction_retry_at && Date.parse(item.extraction_retry_at) > Date.now())
94
96
  continue;
97
+ ensureRoom(id);
95
98
  add.run(id, userId);
96
99
  queued.push(id);
97
100
  }
101
+ wake();
98
102
  return queued;
99
103
  })();
100
104
  }
101
- /** Synchronous consumers can prepare selected articles without waiting for the worker. */
105
+ /** Synchronous consumers join durable article jobs and wait for their results. */
102
106
  export async function prepareArticleContents(userId, ids) {
103
107
  if (ids.length > 50 || ids.some(id => !Number.isSafeInteger(id) || id <= 0))
104
108
  throw new Error("invalid_source_selection");
@@ -119,41 +123,240 @@ export async function prepareArticleContents(userId, ids) {
119
123
  }
120
124
  return results;
121
125
  }
122
- let running = false;
123
- export async function runFullTextQueue() {
124
- if (running)
125
- return;
126
- running = true;
126
+ const active = new Map();
127
+ const listeners = new Map();
128
+ let listenerCount = 0;
129
+ let timer;
130
+ let seedTimer;
131
+ let stopping = false;
132
+ let scheduled = false;
133
+ const startedAt = Date.now();
134
+ let completed = 0;
135
+ let failed = 0;
136
+ let consumerTimeouts = 0;
137
+ let attemptErrors = 0;
138
+ const errorCounts = {};
139
+ let retries = 0;
140
+ let throttled = 0;
141
+ let rejected = 0;
142
+ function ensureRoom(id) {
143
+ if (!db.prepare("SELECT 1 FROM full_text_jobs WHERE item_id = ?").get(id) &&
144
+ db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get().n >= 1000)
145
+ throw new Error("queue_full");
146
+ }
147
+ /** Waiting HTTP consumers are bounded separately from the durable queue. Their deadline never deletes work. */
148
+ export function requestFullText(user, id, force) {
149
+ if (stopping || listenerCount >= 64) {
150
+ rejected++;
151
+ return Promise.resolve({ ok: false, error: "queue_full" });
152
+ }
127
153
  try {
128
- const queued = db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get();
129
- if (queued.n < 200)
130
- scheduleFullTexts();
131
- const jobs = db.prepare("SELECT * FROM full_text_jobs WHERE available_at <= ? ORDER BY priority DESC, available_at LIMIT 1").all(new Date().toISOString());
132
- await Promise.all(jobs.map(async (job) => {
133
- const item = findItemById(job.item_id);
134
- if (item && hasUserCapability(job.user_id, "fulltext") && canAccessItem(job.user_id, job.item_id) && findSubscriptionFullTextMode(job.user_id, item.feed_id) !== "never") {
135
- await loadFullText(job.user_id, job.item_id);
154
+ ensureRoom(id);
155
+ db.prepare(`INSERT INTO full_text_jobs(item_id,user_id,priority,force,created_at) VALUES (?,?,5,?,?)
156
+ ON CONFLICT(item_id) DO UPDATE SET user_id=excluded.user_id, priority=MAX(priority,5), force=MAX(force,excluded.force)`)
157
+ .run(id, user, Number(force), new Date().toISOString());
158
+ }
159
+ catch (error) {
160
+ rejected++;
161
+ if (error instanceof Error && error.message === "queue_full")
162
+ return Promise.resolve({ ok: false, error: "queue_full" });
163
+ console.error(JSON.stringify({ event: "fulltext_queue_error" }));
164
+ return Promise.resolve({ ok: false, error: "extraction_failed" });
165
+ }
166
+ return new Promise(resolve => {
167
+ const entry = { user, resolve, timer: setTimeout(() => {
168
+ listeners.get(id)?.delete(entry);
169
+ listenerCount--;
170
+ consumerTimeouts++;
171
+ if (!listeners.get(id)?.size)
172
+ listeners.delete(id);
173
+ resolve({ ok: false, error: "timeout" });
174
+ }, 30_000) };
175
+ const pending = listeners.get(id) ?? new Set();
176
+ pending.add(entry);
177
+ listeners.set(id, pending);
178
+ listenerCount++;
179
+ wake();
180
+ });
181
+ }
182
+ export function authorizedFullTextRequester(id, preferred) {
183
+ const owner = db.prepare("SELECT user_id FROM full_text_jobs WHERE item_id=?").get(id)?.user_id;
184
+ const candidates = new Set([owner, preferred, ...[...(listeners.get(id) ?? [])].map(entry => entry.user)]);
185
+ return [...candidates].find(user => user !== undefined && canAccessItem(user, id) && hasUserCapability(user, "fulltext") && findSubscriptionFullTextMode(user, findItemById(id).feed_id) !== "never");
186
+ }
187
+ function finishListeners(id, result) {
188
+ const pending = listeners.get(id);
189
+ listeners.delete(id);
190
+ for (const entry of pending ?? []) {
191
+ clearTimeout(entry.timer);
192
+ listenerCount--;
193
+ const allowed = canAccessItem(entry.user, id) && hasUserCapability(entry.user, "fulltext");
194
+ entry.resolve(allowed ? result : { ok: false, error: "capability_not_available" });
195
+ }
196
+ }
197
+ function wake() {
198
+ if (stopping || scheduled)
199
+ return;
200
+ scheduled = true;
201
+ queueMicrotask(() => {
202
+ scheduled = false;
203
+ try {
204
+ pump();
205
+ }
206
+ catch {
207
+ console.error(JSON.stringify({ event: "fulltext_queue_error" }));
208
+ if (!stopping) {
209
+ timer = setTimeout(wake, 1000);
210
+ timer.unref();
136
211
  }
137
- db.prepare("DELETE FROM full_text_jobs WHERE item_id = ?").run(job.item_id);
138
- }));
212
+ }
213
+ });
214
+ }
215
+ export function configureFullTextPool() {
216
+ const pool = articleWorkPool();
217
+ pool.cooldown = host => Date.parse(db.prepare("SELECT retry_at FROM full_text_hosts WHERE host=?").get(host)?.retry_at ?? "") || 0;
218
+ return pool;
219
+ }
220
+ export function deferFullTextHost(host, retryAt) {
221
+ const previous = db.prepare("SELECT retry_at FROM full_text_hosts WHERE host=?").get(host);
222
+ const until = Math.max(Date.now() + 1000, retryAt ?? 0, Date.parse(previous?.retry_at ?? "") || 0);
223
+ db.prepare("INSERT INTO full_text_hosts(host,retry_at) VALUES (?,?) ON CONFLICT(host) DO UPDATE SET retry_at=excluded.retry_at")
224
+ .run(host, new Date(until).toISOString());
225
+ }
226
+ function pump() {
227
+ if (stopping)
228
+ return;
229
+ if (timer)
230
+ clearTimeout(timer);
231
+ const pool = configureFullTextPool();
232
+ if (seedTimer && db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get().n < pool.limits.concurrency * 2)
233
+ scheduleFullTexts();
234
+ const jobs = db.prepare(`SELECT j.*, i.link FROM full_text_jobs j JOIN items i ON i.id=j.item_id
235
+ WHERE julianday(j.available_at) <= julianday(?) ORDER BY (j.priority + CAST((julianday('now')-julianday(j.created_at))*86400000 / ? AS INTEGER)) DESC,
236
+ j.created_at, j.item_id LIMIT 1000`).all(new Date().toISOString(), pool.limits.agingMs);
237
+ for (const job of jobs) {
238
+ if (active.size >= pool.limits.concurrency)
239
+ break;
240
+ if (active.has(job.item_id))
241
+ continue;
242
+ let host;
243
+ try {
244
+ host = sourceHost(job.link ?? "");
245
+ }
246
+ catch {
247
+ db.prepare("DELETE FROM full_text_jobs WHERE item_id=?").run(job.item_id);
248
+ finishListeners(job.item_id, { ok: false, error: "item_has_no_link" });
249
+ continue;
250
+ }
251
+ const cooldown = db.prepare("SELECT retry_at FROM full_text_hosts WHERE host=?").get(host);
252
+ if (cooldown && Date.parse(cooldown.retry_at) > Date.now()) {
253
+ db.prepare("UPDATE full_text_jobs SET available_at=? WHERE item_id=?").run(cooldown.retry_at, job.item_id);
254
+ continue;
255
+ }
256
+ if (!pool.canStart(host))
257
+ continue;
258
+ const work = execute(job, host).finally(() => { active.delete(job.item_id); wake(); });
259
+ active.set(job.item_id, work);
260
+ }
261
+ if (db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get().n) {
262
+ timer = setTimeout(wake, 250);
263
+ timer.unref();
264
+ }
265
+ }
266
+ async function execute(job, host) {
267
+ try {
268
+ const result = await executeFullText(job.user_id, job.item_id, !!job.force, job.priority);
269
+ if (stopping)
270
+ return; // The SQLite row is the restart record, including its original age.
271
+ if (!result.ok) {
272
+ attemptErrors++;
273
+ errorCounts[result.error] = (errorCounts[result.error] ?? 0) + 1;
274
+ }
275
+ if (!result.ok && result.error === "source_busy") {
276
+ throttled++;
277
+ deferFullTextHost(result.host ?? host, result.retryAt);
278
+ }
279
+ if (!result.ok && result.retryable && (result.deferred || job.attempts < 2)) {
280
+ const delay = Math.min(60_000, 1000 * 2 ** job.attempts) * (0.5 + Math.random() * 0.5);
281
+ const retryAt = Math.max(Date.now() + delay, result.retryAt ?? 0);
282
+ db.transaction(() => {
283
+ db.prepare("UPDATE full_text_jobs SET attempts=attempts+?,available_at=? WHERE item_id=?").run(result.deferred ? 0 : 1, new Date(retryAt).toISOString(), job.item_id);
284
+ db.prepare("UPDATE items SET extraction_status='pending',extraction_retry_at=NULL WHERE id=?").run(job.item_id);
285
+ })();
286
+ retries++;
287
+ return;
288
+ }
289
+ if (!result.ok && result.retryable) {
290
+ const retryAt = Math.max(Date.now() + 3600_000, result.retryAt ?? 0);
291
+ db.prepare("UPDATE items SET extraction_status=?,extraction_retry_at=? WHERE id=?")
292
+ .run(result.error === "source_busy" ? "bot_blocked" : "failed", new Date(retryAt).toISOString(), job.item_id);
293
+ }
294
+ db.prepare("DELETE FROM full_text_jobs WHERE item_id=?").run(job.item_id);
295
+ completed++;
296
+ if (!result.ok)
297
+ failed++;
298
+ finishListeners(job.item_id, result);
139
299
  }
140
- finally {
141
- running = false;
300
+ catch {
301
+ // A database failure retains the restart record and fails the current waiters explicitly.
302
+ finishListeners(job.item_id, { ok: false, error: "extraction_failed" });
303
+ console.error(JSON.stringify({ event: "fulltext_queue_error" }));
304
+ await new Promise(resolve => setTimeout(resolve, 1000));
142
305
  }
143
306
  }
307
+ export function fullTextMetrics() {
308
+ const queued = db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get().n;
309
+ const oldest = db.prepare("SELECT MIN(created_at) oldest FROM full_text_jobs").get().oldest;
310
+ const work = articleWorkPool().metrics();
311
+ return { queued, waiters: listenerCount, ...work, completed, failed, attemptErrors, errorCounts: { ...errorCounts }, consumerTimeouts, retries, throttled, rejected,
312
+ meanDurationMs: work.completed ? work.durationMs / work.completed : 0,
313
+ throughputPerSecond: completed / Math.max(1, (Date.now() - startedAt) / 1000),
314
+ oldestWaitMs: oldest ? Math.max(0, Date.now() - Date.parse(oldest)) : 0 };
315
+ }
316
+ export async function runFullTextQueue() {
317
+ if (stopping)
318
+ return;
319
+ if (db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get().n < 200)
320
+ scheduleFullTexts();
321
+ pump();
322
+ await Promise.all(active.values());
323
+ }
144
324
  export function startFullTextScheduler() {
145
- const tick = () => { void runFullTextQueue().catch(() => console.error("[fulltext] queue failed")); };
146
- // One-time startup repair of old consent-only caches, without contacting publishers.
147
- const cached = db.prepare("SELECT id, full_content_html FROM items WHERE full_content_html IS NOT NULL AND (lower(full_content_html) LIKE '%cookie%' OR lower(full_content_html) LIKE '%consent%') LIMIT 100").all();
325
+ if (seedTimer)
326
+ return seedTimer;
327
+ stopping = false;
328
+ restartArticleWorkPool();
329
+ // Older versions can insert rows without the added age column after a rollback.
330
+ db.prepare("UPDATE full_text_jobs SET created_at=available_at WHERE created_at=''").run();
331
+ const tick = () => {
332
+ db.prepare("DELETE FROM full_text_hosts WHERE julianday(retry_at) <= julianday(?)").run(new Date().toISOString());
333
+ scheduleFullTexts();
334
+ wake();
335
+ console.log(JSON.stringify({ event: "fulltext_queue", ...fullTextMetrics() }));
336
+ };
337
+ // Repair old consent-only caches once; the normal bounded planner performs network work.
338
+ const cached = db.prepare("SELECT id,full_content_html FROM items WHERE full_content_html IS NOT NULL AND (lower(full_content_html) LIKE '%cookie%' OR lower(full_content_html) LIKE '%consent%') LIMIT 100").all();
148
339
  db.transaction(() => {
149
340
  for (const item of cached)
150
341
  if (isCachedConsentSnippet(item.full_content_html)) {
151
- db.prepare("UPDATE items SET full_content_html = NULL, extraction_status = 'pending', extraction_retry_at = NULL WHERE id = ?").run(item.id);
342
+ db.prepare("UPDATE items SET full_content_html=NULL,extraction_status='pending',extraction_retry_at=NULL WHERE id=?").run(item.id);
152
343
  }
153
344
  })();
154
345
  tick();
155
- const timer = setInterval(tick, 15000);
156
- timer.unref();
157
- return timer;
346
+ seedTimer = setInterval(tick, 15_000);
347
+ seedTimer.unref();
348
+ return seedTimer;
349
+ }
350
+ export async function stopFullTextScheduler() {
351
+ stopping = true;
352
+ if (timer)
353
+ clearTimeout(timer);
354
+ if (seedTimer)
355
+ clearInterval(seedTimer);
356
+ seedTimer = undefined;
357
+ for (const id of listeners.keys())
358
+ finishListeners(id, { ok: false, error: "queue_full" });
359
+ await articleWorkPool().close();
360
+ await Promise.all(active.values());
158
361
  }
159
362
  //# sourceMappingURL=fullTextQueue.js.map
@@ -163,7 +163,7 @@ export class ImageCache {
163
163
  if (buffer.length === entry.size && type) {
164
164
  entry.used = this.now();
165
165
  this.counters.hits++;
166
- return { buffer, mime: type.mime, path: this.file(key) };
166
+ return { buffer, mime: type.mime, path: this.file(key), cacheTtlMs: Math.max(0, entry.expires - this.now()) };
167
167
  }
168
168
  }
169
169
  catch { /* Re-fetch evicted or damaged files through the guarded downloader. */ }
@@ -182,6 +182,7 @@ export class ImageCache {
182
182
  }
183
183
  const result = { buffer: image.buffer, mime: type.mime, path: image.path };
184
184
  const ttl = Math.min(this.settings.ttlMs, image.cacheTtlMs ?? this.settings.ttlMs);
185
+ result.cacheTtlMs = ttl;
185
186
  if (this.diskAvailable && ttl > 0 && image.buffer.length <= this.settings.maxBytes) {
186
187
  try {
187
188
  await this.serialize(async () => {
@@ -1,5 +1,7 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { db } from "../db/index.js";
3
+ import { hasUserCapability } from "../auth/capabilities.js";
4
+ import { configureFullTextPool, deferFullTextHost } from "./fullTextQueue.js";
3
5
  import { extractArticle } from "./extractor.js";
4
6
  import { assertPublicHttpUrl, normalizeUrlCandidate } from "./ssrfGuard.js";
5
7
  import { articleImageUrls, scheduleArchive } from "./archive.js";
@@ -46,9 +48,12 @@ export async function saveToInbox(userId, input) {
46
48
  const normalized = normalizeUrlCandidate(input.url.trim());
47
49
  await assertPublicHttpUrl(normalized);
48
50
  targetUrl = normalized;
49
- if (!contentHtml) {
51
+ if (!contentHtml && hasUserCapability(userId, "fulltext")) {
50
52
  try {
51
- const extracted = await extractArticle(targetUrl);
53
+ configureFullTextPool();
54
+ const extracted = await extractArticle(targetUrl, undefined, 5, () => hasUserCapability(userId, "fulltext"));
55
+ if (extracted.status === "throttled")
56
+ deferFullTextHost(extracted.host, extracted.retryAt);
52
57
  if (extracted.status === "ok" || extracted.status === "partial") {
53
58
  extractionStatus = extracted.status === "ok" ? "ready" : "partial";
54
59
  extractedPartial = extracted.status === "partial";
@@ -62,7 +67,7 @@ export async function saveToInbox(userId, input) {
62
67
  }
63
68
  }
64
69
  catch (err) {
65
- console.warn(`[inbox] Article extraction failed for ${targetUrl}:`, err);
70
+ console.warn(JSON.stringify({ event: "inbox_extraction_failed", error: err instanceof Error ? err.name : "unknown" }));
66
71
  }
67
72
  }
68
73
  }
@@ -197,6 +197,8 @@ export async function stopPollingScheduler() {
197
197
  clearInterval(scheduler);
198
198
  scheduler = undefined;
199
199
  const { stopImageCache } = await import("./archive.js");
200
+ const { stopFullTextScheduler } = await import("./fullTextQueue.js");
201
+ await stopFullTextScheduler();
200
202
  await stopImageCache();
201
203
  await Promise.all([queue?.stop(), stopPushOutbox()]);
202
204
  feedRequests.stop();
@@ -650,7 +650,10 @@ export function searchItemsForUser(userId, opts) {
650
650
  if (opts.sort !== "date" && fts) {
651
651
  orderBy = "COALESCE(fts.rank, 0) ASC, COALESCE(julianday(i.published_at), julianday(i.created_at)) DESC";
652
652
  }
653
+ const ranked = opts.sort !== "date" && fts;
654
+ // Materialize once: a virtual-table MATCH in the LEFT JOIN repeats the full ranking scan for each article.
653
655
  const querySql = `
656
+ ${ranked ? "WITH ranked_matches AS MATERIALIZED (SELECT rowid, rank FROM items_fts WHERE items_fts MATCH ?)" : ""}
654
657
  SELECT DISTINCT i.*,
655
658
  COALESCE(NULLIF(TRIM(s.label), ''), NULLIF(TRIM(f.title), ''), f.url) AS feed_title,
656
659
  f.site_url AS feed_site_url,
@@ -666,17 +669,17 @@ export function searchItemsForUser(userId, opts) {
666
669
  LEFT JOIN item_reads r ON r.item_id = i.id AND r.user_id = ?
667
670
  LEFT JOIN item_bookmarks b ON b.item_id = i.id AND b.user_id = ?
668
671
  LEFT JOIN item_notes n ON n.item_id = i.id AND n.user_id = ?
669
- ${opts.sort !== "date" && fts ? "LEFT JOIN items_fts fts ON fts.rowid = i.id AND items_fts MATCH ?" : ""}
672
+ ${ranked ? "LEFT JOIN ranked_matches fts ON fts.rowid = i.id" : ""}
670
673
  WHERE ${conditions.join(" AND ")}
671
674
  ORDER BY ${orderBy}
672
675
  LIMIT ? OFFSET ?
673
676
  `;
674
677
  const queryParams = [
678
+ ...(ranked ? [fts] : []),
675
679
  userId,
676
680
  userId,
677
681
  userId,
678
682
  userId,
679
- ...(opts.sort !== "date" && fts ? [fts] : []),
680
683
  ...params,
681
684
  limit,
682
685
  offset,
@@ -0,0 +1,20 @@
1
+ import helmet from "helmet";
2
+ /**
3
+ * Upgrading every request to https is only right when the instance is served over https. On a plain http address
4
+ * (localhost, a home network) Safari would upgrade scripts and images to an https origin that does not exist.
5
+ */
6
+ export function securityHeaderOptions(publicUrl) {
7
+ const servedOverHttps = new URL(publicUrl).protocol === "https:";
8
+ return {
9
+ contentSecurityPolicy: {
10
+ directives: {
11
+ ...helmet.contentSecurityPolicy.getDefaultDirectives(),
12
+ "script-src": ["'self'"],
13
+ "img-src": ["'self'", "data:", "https:", "http:"],
14
+ "upgrade-insecure-requests": servedOverHttps ? [] : null,
15
+ },
16
+ },
17
+ strictTransportSecurity: servedOverHttps,
18
+ };
19
+ }
20
+ //# sourceMappingURL=securityHeaders.js.map
@@ -3,6 +3,7 @@ import { fileURLToPath } from "node:url";
3
3
  import { existsSync } from "node:fs";
4
4
  import express from "express";
5
5
  import helmet from "helmet";
6
+ import { securityHeaderOptions } from "./securityHeaders.js";
6
7
  import cors from "cors";
7
8
  import cookieSession from "cookie-session";
8
9
  import rateLimit from "express-rate-limit";
@@ -42,15 +43,7 @@ if (config.clientIpHeader && !Array.isArray(config.trustProxy)) {
42
43
  }
43
44
  // Before every rate limiter, so limits count the CDN's client rather than its edge.
44
45
  app.use(clientAddressFromHeader(config.clientIpHeader, config.trustProxy));
45
- app.use(helmet({
46
- contentSecurityPolicy: {
47
- directives: {
48
- ...helmet.contentSecurityPolicy.getDefaultDirectives(),
49
- "script-src": ["'self'"],
50
- "img-src": ["'self'", "data:", "https:", "http:"],
51
- },
52
- },
53
- }));
46
+ app.use(helmet(securityHeaderOptions(config.publicUrl)));
54
47
  // Send X-Robots-Tag to ensure all HTTP endpoints disallow indexing even outside HTML
55
48
  app.use((_req, res, next) => {
56
49
  res.setHeader("X-Robots-Tag", "noindex, nofollow");
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@feedkeeper/server",
3
- "version": "0.23.1",
3
+ "version": "0.24.1",
4
4
  "private": true,
5
5
  "type": "module"
6
6
  }