feedkeeper 0.23.0 → 0.24.1

package/server/dist/feeds/fullTextQueue.js

@@ -1,7 +1,9 @@
 import { isCachedConsentSnippet } from "./extractor.js";
 import { db } from "../db/index.js";
 import { hasUserCapability } from "../auth/capabilities.js";
-import { loadFullText } from "./fullText.js";
+import { loadFullText, executeFullText } from "./fullText.js";
+import { articleWorkPool, restartArticleWorkPool } from "./articleWorkPool.js";
+import { sourceHost } from "./requestGate.js";
 import { canAccessItem, findSubscriptionFullTextMode, findItemById } from "./repository.js";
 /** Refillable durable queue, bounded independently of library size. */
 export function scheduleFullTexts() {
@@ -18,6 +20,7 @@ export function scheduleFullTexts() {
       FROM editions e, json_each(e.item_ids) ids
       JOIN items i ON i.id = ids.value
       WHERE e.user_id IN (SELECT value FROM json_each(?))
+        AND NOT EXISTS (SELECT 1 FROM subscriptions disabled WHERE disabled.user_id=e.user_id AND disabled.feed_id=i.feed_id AND disabled.full_text_mode='never')
         AND e.status = 'active' AND e.expires_at > strftime('%Y-%m-%dT%H:%M:%fZ','now')
         AND i.link IS NOT NULL AND i.full_content_html IS NULL
         AND i.extraction_status != 'feed_only'
@@ -30,6 +33,7 @@ export function scheduleFullTexts() {
       FROM item_bookmarks b
       JOIN items i ON i.id = b.item_id
       WHERE b.user_id IN (SELECT value FROM json_each(?))
+        AND NOT EXISTS (SELECT 1 FROM subscriptions disabled WHERE disabled.user_id=b.user_id AND disabled.feed_id=i.feed_id AND disabled.full_text_mode='never')
         AND i.link IS NOT NULL AND i.full_content_html IS NULL
         AND i.extraction_status != 'feed_only'
         AND (i.extraction_retry_at IS NULL OR i.extraction_retry_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now'))
@@ -41,6 +45,7 @@ export function scheduleFullTexts() {
       FROM item_notes n
       JOIN items i ON i.id = n.item_id
       WHERE n.user_id IN (SELECT value FROM json_each(?))
+        AND NOT EXISTS (SELECT 1 FROM subscriptions disabled WHERE disabled.user_id=n.user_id AND disabled.feed_id=i.feed_id AND disabled.full_text_mode='never')
         AND i.link IS NOT NULL AND i.full_content_html IS NULL
         AND i.extraction_status != 'feed_only'
         AND (i.extraction_retry_at IS NULL OR i.extraction_retry_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now'))
@@ -61,8 +66,8 @@ export function scheduleFullTexts() {
     SELECT item_id AS id, user_id, MAX(priority) AS priority
     FROM candidate_articles
     GROUP BY item_id
-    ORDER BY priority DESC, created_at DESC LIMIT ?`).all(JSON.stringify(eligible), JSON.stringify(eligible), JSON.stringify(eligible), JSON.stringify(eligible), room);
-    const add = db.prepare("INSERT OR IGNORE INTO full_text_jobs(item_id, user_id, priority) VALUES (?, ?, ?)");
+    ORDER BY (priority + CAST((julianday('now')-julianday(created_at))*1440 / 60 AS INTEGER)) DESC, created_at ASC LIMIT ?`).all(JSON.stringify(eligible), JSON.stringify(eligible), JSON.stringify(eligible), JSON.stringify(eligible), room);
+    const add = db.prepare("INSERT OR IGNORE INTO full_text_jobs(item_id, user_id, priority, created_at) VALUES (?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ','now'))");
     let count = 0;
     db.transaction(() => {
         for (const row of candidates)
@@ -81,10 +86,7 @@ export function enqueueArticleContents(userId, ids) {
     if (unique.some(id => !canAccessItem(userId, id)))
         throw new Error("item_not_found");
     return db.transaction(() => {
-        const count = db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get();
-        if (count.n + unique.length > 1000)
-            throw new Error("queue_full");
-        const add = db.prepare("INSERT INTO full_text_jobs(item_id,user_id,priority) VALUES (?,?,4) ON CONFLICT(item_id) DO UPDATE SET priority = MAX(priority,4), user_id = excluded.user_id");
+        const add = db.prepare("INSERT INTO full_text_jobs(item_id,user_id,priority,created_at) VALUES (?,?,4,strftime('%Y-%m-%dT%H:%M:%fZ','now')) ON CONFLICT(item_id) DO UPDATE SET priority = MAX(priority,4), user_id = excluded.user_id");
         const queued = [];
         for (const id of unique) {
             const item = findItemById(id);
@@ -92,13 +94,15 @@ export function enqueueArticleContents(userId, ids) {
                 continue;
             if (item.extraction_retry_at && Date.parse(item.extraction_retry_at) > Date.now())
                 continue;
+            ensureRoom(id);
             add.run(id, userId);
             queued.push(id);
         }
+        wake();
         return queued;
     })();
 }
-/** Synchronous consumers can prepare selected articles without waiting for the worker. */
+/** Synchronous consumers join durable article jobs and wait for their results. */
 export async function prepareArticleContents(userId, ids) {
     if (ids.length > 50 || ids.some(id => !Number.isSafeInteger(id) || id <= 0))
         throw new Error("invalid_source_selection");
@@ -119,41 +123,240 @@ export async function prepareArticleContents(userId, ids) {
     }
     return results;
 }
-let running = false;
-export async function runFullTextQueue() {
-    if (running)
-        return;
-    running = true;
+const active = new Map();
+const listeners = new Map();
+let listenerCount = 0;
+let timer;
+let seedTimer;
+let stopping = false;
+let scheduled = false;
+const startedAt = Date.now();
+let completed = 0;
+let failed = 0;
+let consumerTimeouts = 0;
+let attemptErrors = 0;
+const errorCounts = {};
+let retries = 0;
+let throttled = 0;
+let rejected = 0;
+function ensureRoom(id) {
+    if (!db.prepare("SELECT 1 FROM full_text_jobs WHERE item_id = ?").get(id) &&
+        db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get().n >= 1000)
+        throw new Error("queue_full");
+}
+/** Waiting HTTP consumers are bounded separately from the durable queue. Their deadline never deletes work. */
+export function requestFullText(user, id, force) {
+    if (stopping || listenerCount >= 64) {
+        rejected++;
+        return Promise.resolve({ ok: false, error: "queue_full" });
+    }
     try {
-        const queued = db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get();
-        if (queued.n < 200)
-            scheduleFullTexts();
-        const jobs = db.prepare("SELECT * FROM full_text_jobs WHERE available_at <= ? ORDER BY priority DESC, available_at LIMIT 1").all(new Date().toISOString());
-        await Promise.all(jobs.map(async (job) => {
-            const item = findItemById(job.item_id);
-            if (item && hasUserCapability(job.user_id, "fulltext") && canAccessItem(job.user_id, job.item_id) && findSubscriptionFullTextMode(job.user_id, item.feed_id) !== "never") {
-                await loadFullText(job.user_id, job.item_id);
+        ensureRoom(id);
+        db.prepare(`INSERT INTO full_text_jobs(item_id,user_id,priority,force,created_at) VALUES (?,?,5,?,?)
+      ON CONFLICT(item_id) DO UPDATE SET user_id=excluded.user_id, priority=MAX(priority,5), force=MAX(force,excluded.force)`)
+            .run(id, user, Number(force), new Date().toISOString());
+    }
+    catch (error) {
+        rejected++;
+        if (error instanceof Error && error.message === "queue_full")
+            return Promise.resolve({ ok: false, error: "queue_full" });
+        console.error(JSON.stringify({ event: "fulltext_queue_error" }));
+        return Promise.resolve({ ok: false, error: "extraction_failed" });
+    }
+    return new Promise(resolve => {
+        const entry = { user, resolve, timer: setTimeout(() => {
+                listeners.get(id)?.delete(entry);
+                listenerCount--;
+                consumerTimeouts++;
+                if (!listeners.get(id)?.size)
+                    listeners.delete(id);
+                resolve({ ok: false, error: "timeout" });
+            }, 30_000) };
+        const pending = listeners.get(id) ?? new Set();
+        pending.add(entry);
+        listeners.set(id, pending);
+        listenerCount++;
+        wake();
+    });
+}
+export function authorizedFullTextRequester(id, preferred) {
+    const owner = db.prepare("SELECT user_id FROM full_text_jobs WHERE item_id=?").get(id)?.user_id;
+    const candidates = new Set([owner, preferred, ...[...(listeners.get(id) ?? [])].map(entry => entry.user)]);
+    return [...candidates].find(user => user !== undefined && canAccessItem(user, id) && hasUserCapability(user, "fulltext") && findSubscriptionFullTextMode(user, findItemById(id).feed_id) !== "never");
+}
+function finishListeners(id, result) {
+    const pending = listeners.get(id);
+    listeners.delete(id);
+    for (const entry of pending ?? []) {
+        clearTimeout(entry.timer);
+        listenerCount--;
+        const allowed = canAccessItem(entry.user, id) && hasUserCapability(entry.user, "fulltext");
+        entry.resolve(allowed ? result : { ok: false, error: "capability_not_available" });
+    }
+}
+function wake() {
+    if (stopping || scheduled)
+        return;
+    scheduled = true;
+    queueMicrotask(() => {
+        scheduled = false;
+        try {
+            pump();
+        }
+        catch {
+            console.error(JSON.stringify({ event: "fulltext_queue_error" }));
+            if (!stopping) {
+                timer = setTimeout(wake, 1000);
+                timer.unref();
             }
-            db.prepare("DELETE FROM full_text_jobs WHERE item_id = ?").run(job.item_id);
-        }));
+        }
+    });
+}
+export function configureFullTextPool() {
+    const pool = articleWorkPool();
+    pool.cooldown = host => Date.parse(db.prepare("SELECT retry_at FROM full_text_hosts WHERE host=?").get(host)?.retry_at ?? "") || 0;
+    return pool;
+}
+export function deferFullTextHost(host, retryAt) {
+    const previous = db.prepare("SELECT retry_at FROM full_text_hosts WHERE host=?").get(host);
+    const until = Math.max(Date.now() + 1000, retryAt ?? 0, Date.parse(previous?.retry_at ?? "") || 0);
+    db.prepare("INSERT INTO full_text_hosts(host,retry_at) VALUES (?,?) ON CONFLICT(host) DO UPDATE SET retry_at=excluded.retry_at")
+        .run(host, new Date(until).toISOString());
+}
+function pump() {
+    if (stopping)
+        return;
+    if (timer)
+        clearTimeout(timer);
+    const pool = configureFullTextPool();
+    if (seedTimer && db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get().n < pool.limits.concurrency * 2)
+        scheduleFullTexts();
+    const jobs = db.prepare(`SELECT j.*, i.link FROM full_text_jobs j JOIN items i ON i.id=j.item_id
+    WHERE julianday(j.available_at) <= julianday(?) ORDER BY (j.priority + CAST((julianday('now')-julianday(j.created_at))*86400000 / ? AS INTEGER)) DESC,
+    j.created_at, j.item_id LIMIT 1000`).all(new Date().toISOString(), pool.limits.agingMs);
+    for (const job of jobs) {
+        if (active.size >= pool.limits.concurrency)
+            break;
+        if (active.has(job.item_id))
+            continue;
+        let host;
+        try {
+            host = sourceHost(job.link ?? "");
+        }
+        catch {
+            db.prepare("DELETE FROM full_text_jobs WHERE item_id=?").run(job.item_id);
+            finishListeners(job.item_id, { ok: false, error: "item_has_no_link" });
+            continue;
+        }
+        const cooldown = db.prepare("SELECT retry_at FROM full_text_hosts WHERE host=?").get(host);
+        if (cooldown && Date.parse(cooldown.retry_at) > Date.now()) {
+            db.prepare("UPDATE full_text_jobs SET available_at=? WHERE item_id=?").run(cooldown.retry_at, job.item_id);
+            continue;
+        }
+        if (!pool.canStart(host))
+            continue;
+        const work = execute(job, host).finally(() => { active.delete(job.item_id); wake(); });
+        active.set(job.item_id, work);
+    }
+    if (db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get().n) {
+        timer = setTimeout(wake, 250);
+        timer.unref();
+    }
+}
+async function execute(job, host) {
+    try {
+        const result = await executeFullText(job.user_id, job.item_id, !!job.force, job.priority);
+        if (stopping)
+            return; // The SQLite row is the restart record, including its original age.
+        if (!result.ok) {
+            attemptErrors++;
+            errorCounts[result.error] = (errorCounts[result.error] ?? 0) + 1;
+        }
+        if (!result.ok && result.error === "source_busy") {
+            throttled++;
+            deferFullTextHost(result.host ?? host, result.retryAt);
+        }
+        if (!result.ok && result.retryable && (result.deferred || job.attempts < 2)) {
+            const delay = Math.min(60_000, 1000 * 2 ** job.attempts) * (0.5 + Math.random() * 0.5);
+            const retryAt = Math.max(Date.now() + delay, result.retryAt ?? 0);
+            db.transaction(() => {
+                db.prepare("UPDATE full_text_jobs SET attempts=attempts+?,available_at=? WHERE item_id=?").run(result.deferred ? 0 : 1, new Date(retryAt).toISOString(), job.item_id);
+                db.prepare("UPDATE items SET extraction_status='pending',extraction_retry_at=NULL WHERE id=?").run(job.item_id);
+            })();
+            retries++;
+            return;
+        }
+        if (!result.ok && result.retryable) {
+            const retryAt = Math.max(Date.now() + 3600_000, result.retryAt ?? 0);
+            db.prepare("UPDATE items SET extraction_status=?,extraction_retry_at=? WHERE id=?")
+                .run(result.error === "source_busy" ? "bot_blocked" : "failed", new Date(retryAt).toISOString(), job.item_id);
+        }
+        db.prepare("DELETE FROM full_text_jobs WHERE item_id=?").run(job.item_id);
+        completed++;
+        if (!result.ok)
+            failed++;
+        finishListeners(job.item_id, result);
     }
-    finally {
-        running = false;
+    catch {
+        // A database failure retains the restart record and fails the current waiters explicitly.
+        finishListeners(job.item_id, { ok: false, error: "extraction_failed" });
+        console.error(JSON.stringify({ event: "fulltext_queue_error" }));
+        await new Promise(resolve => setTimeout(resolve, 1000));
     }
 }
+export function fullTextMetrics() {
+    const queued = db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get().n;
+    const oldest = db.prepare("SELECT MIN(created_at) oldest FROM full_text_jobs").get().oldest;
+    const work = articleWorkPool().metrics();
+    return { queued, waiters: listenerCount, ...work, completed, failed, attemptErrors, errorCounts: { ...errorCounts }, consumerTimeouts, retries, throttled, rejected,
+        meanDurationMs: work.completed ? work.durationMs / work.completed : 0,
+        throughputPerSecond: completed / Math.max(1, (Date.now() - startedAt) / 1000),
+        oldestWaitMs: oldest ? Math.max(0, Date.now() - Date.parse(oldest)) : 0 };
+}
+export async function runFullTextQueue() {
+    if (stopping)
+        return;
+    if (db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get().n < 200)
+        scheduleFullTexts();
+    pump();
+    await Promise.all(active.values());
+}
 export function startFullTextScheduler() {
-    const tick = () => { void runFullTextQueue().catch(() => console.error("[fulltext] queue failed")); };
-    // One-time startup repair of old consent-only caches, without contacting publishers.
-    const cached = db.prepare("SELECT id, full_content_html FROM items WHERE full_content_html IS NOT NULL AND (lower(full_content_html) LIKE '%cookie%' OR lower(full_content_html) LIKE '%consent%') LIMIT 100").all();
+    if (seedTimer)
+        return seedTimer;
+    stopping = false;
+    restartArticleWorkPool();
+    // Older versions can insert rows without the added age column after a rollback.
+    db.prepare("UPDATE full_text_jobs SET created_at=available_at WHERE created_at=''").run();
+    const tick = () => {
+        db.prepare("DELETE FROM full_text_hosts WHERE julianday(retry_at) <= julianday(?)").run(new Date().toISOString());
+        scheduleFullTexts();
+        wake();
+        console.log(JSON.stringify({ event: "fulltext_queue", ...fullTextMetrics() }));
+    };
+    // Repair old consent-only caches once; the normal bounded planner performs network work.
+    const cached = db.prepare("SELECT id,full_content_html FROM items WHERE full_content_html IS NOT NULL AND (lower(full_content_html) LIKE '%cookie%' OR lower(full_content_html) LIKE '%consent%') LIMIT 100").all();
     db.transaction(() => {
         for (const item of cached)
             if (isCachedConsentSnippet(item.full_content_html)) {
-                db.prepare("UPDATE items SET full_content_html = NULL, extraction_status = 'pending', extraction_retry_at = NULL WHERE id = ?").run(item.id);
+                db.prepare("UPDATE items SET full_content_html=NULL,extraction_status='pending',extraction_retry_at=NULL WHERE id=?").run(item.id);
             }
     })();
     tick();
-    const timer = setInterval(tick, 15000);
-    timer.unref();
-    return timer;
+    seedTimer = setInterval(tick, 15_000);
+    seedTimer.unref();
+    return seedTimer;
+}
+export async function stopFullTextScheduler() {
+    stopping = true;
+    if (timer)
+        clearTimeout(timer);
+    if (seedTimer)
+        clearInterval(seedTimer);
+    seedTimer = undefined;
+    for (const id of listeners.keys())
+        finishListeners(id, { ok: false, error: "queue_full" });
+    await articleWorkPool().close();
+    await Promise.all(active.values());
 }
 //# sourceMappingURL=fullTextQueue.js.map

package/server/dist/feeds/imageCache.js

@@ -163,7 +163,7 @@ export class ImageCache {
                 if (buffer.length === entry.size && type) {
                     entry.used = this.now();
                     this.counters.hits++;
-                    return { buffer, mime: type.mime, path: this.file(key) };
+                    return { buffer, mime: type.mime, path: this.file(key), cacheTtlMs: Math.max(0, entry.expires - this.now()) };
                 }
             }
             catch { /* Re-fetch evicted or damaged files through the guarded downloader. */ }
@@ -182,6 +182,7 @@ export class ImageCache {
             }
             const result = { buffer: image.buffer, mime: type.mime, path: image.path };
             const ttl = Math.min(this.settings.ttlMs, image.cacheTtlMs ?? this.settings.ttlMs);
+            result.cacheTtlMs = ttl;
             if (this.diskAvailable && ttl > 0 && image.buffer.length <= this.settings.maxBytes) {
                 try {
                     await this.serialize(async () => {

package/server/dist/feeds/inbox.js

@@ -1,5 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { db } from "../db/index.js";
+import { hasUserCapability } from "../auth/capabilities.js";
+import { configureFullTextPool, deferFullTextHost } from "./fullTextQueue.js";
 import { extractArticle } from "./extractor.js";
 import { assertPublicHttpUrl, normalizeUrlCandidate } from "./ssrfGuard.js";
 import { articleImageUrls, scheduleArchive } from "./archive.js";
@@ -46,9 +48,12 @@ export async function saveToInbox(userId, input) {
         const normalized = normalizeUrlCandidate(input.url.trim());
         await assertPublicHttpUrl(normalized);
         targetUrl = normalized;
-        if (!contentHtml) {
+        if (!contentHtml && hasUserCapability(userId, "fulltext")) {
             try {
-                const extracted = await extractArticle(targetUrl);
+                configureFullTextPool();
+                const extracted = await extractArticle(targetUrl, undefined, 5, () => hasUserCapability(userId, "fulltext"));
+                if (extracted.status === "throttled")
+                    deferFullTextHost(extracted.host, extracted.retryAt);
                 if (extracted.status === "ok" || extracted.status === "partial") {
                     extractionStatus = extracted.status === "ok" ? "ready" : "partial";
                     extractedPartial = extracted.status === "partial";
@@ -62,7 +67,7 @@ export async function saveToInbox(userId, input) {
                 }
             }
             catch (err) {
-                console.warn(`[inbox] Article extraction failed for ${targetUrl}:`, err);
+                console.warn(JSON.stringify({ event: "inbox_extraction_failed", error: err instanceof Error ? err.name : "unknown" }));
             }
         }
     }

package/server/dist/feeds/poller.js

@@ -9,6 +9,7 @@ import { findFeedById, listSubscriberIds, updateFeedAfterPoll, updateFeedIcon, u
 import { db } from "../db/index.js";
 import { configurePollHostCooldowns, PollQueue } from "./pollQueue.js";
 import { feedRequests, PollDeferredError, sourceHost } from "./requestGate.js";
+import { normalizePublishedAt } from "./dates.js";
 configurePollHostCooldowns();
 function mediaAttribute(value, key) {
     if (typeof value !== "object" || value === null)
@@ -103,7 +104,7 @@ async function pollAttempt(feed, signal) {
         title: plainTitle(item.title), link: item.link,
         contentSnippet: decodeEntities(item.contentSnippet ?? item.content) ?? undefined,
         contentHtml: item.contentEncoded ?? item["content:encoded"] ?? item.content ?? null,
-        publishedAt: item.isoDate ?? item.pubDate, imageUrl: extractImageUrl(item),
+        publishedAt: normalizePublishedAt(item.isoDate ?? item.pubDate) ?? undefined, imageUrl: extractImageUrl(item),
     }));
     // No await in this transaction. Items, health/validators and job completion become visible together.
     const newItems = db.transaction(() => {
@@ -196,6 +197,8 @@ export async function stopPollingScheduler() {
         clearInterval(scheduler);
     scheduler = undefined;
     const { stopImageCache } = await import("./archive.js");
+    const { stopFullTextScheduler } = await import("./fullTextQueue.js");
+    await stopFullTextScheduler();
     await stopImageCache();
     await Promise.all([queue?.stop(), stopPushOutbox()]);
     feedRequests.stop();

package/server/dist/feeds/repository.js

@@ -234,7 +234,7 @@ export function listItemsForUser(userId, opts = {}) {
     const limit = Math.min(opts.limit ?? 50, 200);
     const offset = Math.max(opts.offset ?? 0, 0);
     params.push(limit, offset);
-    const orderBy = opts.sortByAdded ? "i.created_at DESC, i.id DESC" : opts.bookmarkedOnly ? "b.created_at DESC, i.id DESC" : "i.published_at DESC, i.created_at DESC, i.id DESC";
+    const orderBy = opts.sortByAdded ? "i.created_at DESC, i.id DESC" : opts.bookmarkedOnly ? "b.created_at DESC, i.id DESC" : "COALESCE(julianday(i.published_at), julianday(i.created_at)) DESC, i.created_at DESC, i.id DESC";
     // Page IDs before loading HTML. User/feed and user/item keys make these joins unique.
     // Keep selection and loading in one statement so they share the same read snapshot.
     return db
@@ -583,8 +583,17 @@ export function formatFtsQuery(raw) {
         return "";
     return sanitized.join(" ");
 }
+/**
+ * True when the query has letters or digits that the full-text index cannot be asked for as typed (Japanese,
+ * Cyrillic, Greek, ...). `formatFtsQuery` drops them, so a query such as "RSS 新聞" would silently search for "RSS"
+ * alone; such queries are matched word by word as substrings instead. Quoted phrases are passed through as typed.
+ */
+export function needsSubstringSearch(raw) {
+    const outsideQuotes = raw.replace(/"[^"]*"/g, " ");
+    return /(?![a-zA-Z0-9_\u00C0-\u024F\u1E00-\u1EFF])[\p{L}\p{N}]/u.test(outsideQuotes);
+}
 export function searchItemsForUser(userId, opts) {
-    const fts = formatFtsQuery(opts.query);
+    const fts = needsSubstringSearch(opts.query) ? "" : formatFtsQuery(opts.query);
     const conditions = ["(s.id IS NOT NULL OR b.item_id IS NOT NULL OR n.user_id IS NOT NULL)"];
     const params = [];
     if (opts.feedId) {
@@ -610,14 +619,18 @@ export function searchItemsForUser(userId, opts) {
         params.push(fts, userId, opts.query);
     }
     else {
-        conditions.push(`(
-      INSTR(LOWER(COALESCE(i.title, '')), LOWER(?)) > 0
-      OR INSTR(LOWER(COALESCE(i.content_snippet, '')), LOWER(?)) > 0
-      OR INSTR(LOWER(COALESCE(i.content_html, '')), LOWER(?)) > 0
-      OR INSTR(LOWER(COALESCE(i.full_content_html, '')), LOWER(?)) > 0
-      OR i.id IN (SELECT item_id FROM item_notes WHERE user_id = ? AND INSTR(LOWER(content), LOWER(?)) > 0)
-    )`);
-        params.push(opts.query, opts.query, opts.query, opts.query, userId, opts.query);
+        // Every word has to occur somewhere in the article or in the person's note.
+        const words = opts.query.trim().split(/\s+/).filter(Boolean).slice(0, 8);
+        for (const word of words) {
+            conditions.push(`(
+        INSTR(LOWER(COALESCE(i.title, '')), LOWER(?)) > 0
+        OR INSTR(LOWER(COALESCE(i.content_snippet, '')), LOWER(?)) > 0
+        OR INSTR(LOWER(COALESCE(i.content_html, '')), LOWER(?)) > 0
+        OR INSTR(LOWER(COALESCE(i.full_content_html, '')), LOWER(?)) > 0
+        OR i.id IN (SELECT item_id FROM item_notes WHERE user_id = ? AND INSTR(LOWER(content), LOWER(?)) > 0)
+      )`);
+            params.push(word, word, word, word, userId, word);
+        }
     }
     // Count total matches
     const totalRow = db.prepare(`
@@ -633,11 +646,14 @@ export function searchItemsForUser(userId, opts) {
     const total = totalRow?.count ?? 0;
     const limit = Math.min(opts.limit ?? 30, 100);
     const offset = Math.max(opts.offset ?? 0, 0);
-    let orderBy = "i.published_at DESC, i.created_at DESC, i.id DESC";
+    let orderBy = "COALESCE(julianday(i.published_at), julianday(i.created_at)) DESC, i.created_at DESC, i.id DESC";
     if (opts.sort !== "date" && fts) {
-        orderBy = "COALESCE(fts.rank, 0) ASC, i.published_at DESC";
+        orderBy = "COALESCE(fts.rank, 0) ASC, COALESCE(julianday(i.published_at), julianday(i.created_at)) DESC";
     }
+    const ranked = opts.sort !== "date" && fts;
+    // Materialize once: a virtual-table MATCH in the LEFT JOIN repeats the full ranking scan for each article.
     const querySql = `
+    ${ranked ? "WITH ranked_matches AS MATERIALIZED (SELECT rowid, rank FROM items_fts WHERE items_fts MATCH ?)" : ""}
     SELECT DISTINCT i.*,
            COALESCE(NULLIF(TRIM(s.label), ''), NULLIF(TRIM(f.title), ''), f.url) AS feed_title,
            f.site_url AS feed_site_url,
@@ -653,17 +669,17 @@ export function searchItemsForUser(userId, opts) {
     LEFT JOIN item_reads r ON r.item_id = i.id AND r.user_id = ?
     LEFT JOIN item_bookmarks b ON b.item_id = i.id AND b.user_id = ?
     LEFT JOIN item_notes n ON n.item_id = i.id AND n.user_id = ?
-    ${opts.sort !== "date" && fts ? "LEFT JOIN items_fts fts ON fts.rowid = i.id AND items_fts MATCH ?" : ""}
+    ${ranked ? "LEFT JOIN ranked_matches fts ON fts.rowid = i.id" : ""}
     WHERE ${conditions.join(" AND ")}
     ORDER BY ${orderBy}
     LIMIT ? OFFSET ?
   `;
     const queryParams = [
+        ...(ranked ? [fts] : []),
         userId,
         userId,
         userId,
         userId,
-        ...(opts.sort !== "date" && fts ? [fts] : []),
         ...params,
         limit,
         offset,

package/server/dist/oauth/routes.js

@@ -6,7 +6,7 @@ import { z } from "zod";
 import { config } from "../config.js";
 import { getUserCapabilities, hasUserCapability } from "../auth/capabilities.js";
 import { hasTotp, securityVersion, sessionIsSecure } from "../auth/security.js";
-import { verifyPassword } from "../auth/password.js";
+import { verifyPasswordOrDummy } from "../auth/password.js";
 import { findUserByEmail, findUserById } from "../auth/users.js";
 import { ISSUER, MCP_RESOURCE } from "./constants.js";
 import { getOAuthHost } from "./host.js";
@@ -115,7 +115,15 @@ function basicClientId(req) {
     if (!header?.startsWith("Basic "))
         return null;
     const [id] = Buffer.from(header.slice(6), "base64").toString("utf8").split(":");
-    return id ? decodeURIComponent(id) : null;
+    if (!id)
+        return null;
+    try {
+        return decodeURIComponent(id);
+    }
+    catch {
+        // A malformed escape is a client that does not exist, not a server error.
+        return null;
+    }
 }
 const str = (value) => (typeof value === "string" ? value : "");
 function tokenResponse(res, tokens, scope) {
@@ -312,7 +320,8 @@ oauthAuthorizeRouter.post("/oauth/authorize/login", sameOriginOnly, loginLimiter
         return;
     }
     const user = findUserByEmail(parsed.data.email);
-    if (!user || !verifyPassword(parsed.data.password, user.password_hash)) {
+    const passwordMatches = verifyPasswordOrDummy(parsed.data.password, user?.password_hash);
+    if (!user || !passwordMatches) {
         page(res, 401, renderLogin(req, { clientName: client.client_name, returnTo, failed: true }));
         return;
     }

package/server/dist/securityHeaders.js

@@ -0,0 +1,20 @@
+import helmet from "helmet";
+/**
+ * Upgrading every request to https is only right when the instance is served over https. On a plain http address
+ * (localhost, a home network) Safari would upgrade scripts and images to an https origin that does not exist.
+ */
+export function securityHeaderOptions(publicUrl) {
+    const servedOverHttps = new URL(publicUrl).protocol === "https:";
+    return {
+        contentSecurityPolicy: {
+            directives: {
+                ...helmet.contentSecurityPolicy.getDefaultDirectives(),
+                "script-src": ["'self'"],
+                "img-src": ["'self'", "data:", "https:", "http:"],
+                "upgrade-insecure-requests": servedOverHttps ? [] : null,
+            },
+        },
+        strictTransportSecurity: servedOverHttps,
+    };
+}
+//# sourceMappingURL=securityHeaders.js.map

package/server/dist/server.js

@@ -3,6 +3,7 @@ import { fileURLToPath } from "node:url";
 import { existsSync } from "node:fs";
 import express from "express";
 import helmet from "helmet";
+import { securityHeaderOptions } from "./securityHeaders.js";
 import cors from "cors";
 import cookieSession from "cookie-session";
 import rateLimit from "express-rate-limit";
@@ -42,15 +43,7 @@ if (config.clientIpHeader && !Array.isArray(config.trustProxy)) {
 }
 // Before every rate limiter, so limits count the CDN's client rather than its edge.
 app.use(clientAddressFromHeader(config.clientIpHeader, config.trustProxy));
-app.use(helmet({
-    contentSecurityPolicy: {
-        directives: {
-            ...helmet.contentSecurityPolicy.getDefaultDirectives(),
-            "script-src": ["'self'"],
-            "img-src": ["'self'", "data:", "https:", "http:"],
-        },
-    },
-}));
+app.use(helmet(securityHeaderOptions(config.publicUrl)));
 // Send X-Robots-Tag to ensure all HTTP endpoints disallow indexing even outside HTML
 app.use((_req, res, next) => {
     res.setHeader("X-Robots-Tag", "noindex, nofollow");

package/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feedkeeper/server",
-  "version": "0.23.0",
+  "version": "0.24.1",
   "private": true,
   "type": "module"
 }