feedkeeper 0.27.1 → 0.28.0

package/server/dist/sync/changeLog.js

@@ -1,8 +1,7 @@
-import { contentMetadata } from "../feeds/articleContent.js";
-import { canAccessItem, findItemById } from "../feeds/repository.js";
+import { canAccessItem } from "../feeds/repository.js";
 import { db } from "../db/index.js";
-import { findItemNote, listFoldersForUser, listMutedKeywords, listSubscriptionsForUser, } from "../feeds/repository.js";
-import { getEditionState } from "../native/editions.js";
+import { listFoldersForUser, listMutedKeywords, listSubscriptionsForUser, } from "../feeds/repository.js";
+import { getEdition, getEditionState } from "../native/editions.js";
 import { getNativePreferences } from "../native/preferences.js";
 const TOMBSTONE_DAYS = 90;
 const MAX_PAGE = 500;
@@ -26,21 +25,61 @@ export function serializeSubscription(subscription) {
             : `/api/v1/subscriptions/${subscription.id}/icon`,
     };
 }
-function itemState(userId, itemId) {
-    const exists = db.prepare("SELECT 1 FROM items WHERE id = ?").get(itemId);
-    if (!exists)
-        return null;
-    const row = db
-        .prepare(`SELECT EXISTS (SELECT 1 FROM item_reads WHERE user_id = ? AND item_id = ?) AS read,
-              (SELECT created_at FROM item_bookmarks WHERE user_id = ? AND item_id = ?) AS saved_at,
-              (SELECT position FROM item_progress WHERE user_id = ? AND item_id = ?) AS progress`)
-        .get(userId, itemId, userId, itemId, userId, itemId);
-    return {
-        read: Boolean(row.read),
-        saved: row.saved_at !== null,
-        savedAt: row.saved_at,
-        progress: row.progress,
-    };
+function loadItemStates(userId, itemIds) {
+    const result = new Map();
+    if (!itemIds.length)
+        return result;
+    const rows = db
+        .prepare(`SELECT i.id,
+              r.item_id IS NOT NULL AS read,
+              b.created_at AS saved_at,
+              p.position AS progress
+       FROM items i
+       LEFT JOIN item_reads r ON r.user_id = ? AND r.item_id = i.id
+       LEFT JOIN item_bookmarks b ON b.user_id = ? AND b.item_id = i.id
+       LEFT JOIN item_progress p ON p.user_id = ? AND p.item_id = i.id
+       WHERE i.id IN (SELECT CAST(value AS INTEGER) FROM json_each(?))`)
+        .all(userId, userId, userId, JSON.stringify(itemIds));
+    for (const row of rows) {
+        result.set(row.id, {
+            read: Boolean(row.read),
+            saved: row.saved_at !== null,
+            savedAt: row.saved_at,
+            progress: row.progress,
+        });
+    }
+    return result;
+}
+function loadItemContents(userId, itemIds) {
+    const result = new Map();
+    if (!itemIds.length)
+        return result;
+    const rows = db
+        .prepare(`SELECT i.id,
+              i.content_revision,
+              i.is_empty_gallery,
+              CASE WHEN i.full_content_html IS NOT NULL AND i.full_content_html != '' THEN 'reader'
+                   WHEN i.content_html IS NOT NULL AND i.content_html != '' THEN 'feed'
+                   ELSE 'snippet' END AS source,
+              i.extraction_status,
+              i.extraction_attempted_at,
+              i.extraction_retry_at
+       FROM items i
+       WHERE i.id IN (SELECT CAST(value AS INTEGER) FROM json_each(?))`)
+        .all(JSON.stringify(itemIds));
+    for (const row of rows) {
+        if (!canAccessItem(userId, row.id))
+            continue;
+        result.set(row.id, {
+            revision: row.content_revision,
+            source: row.source,
+            status: row.extraction_status,
+            attemptedAt: row.extraction_attempted_at,
+            retryAt: row.extraction_retry_at,
+            isEmptyGallery: Boolean(row.is_empty_gallery),
+        });
+    }
+    return result;
 }
 /**
  * Add the data a user already had before the log existed. Everything that is in the log
@@ -83,7 +122,12 @@ export class ResyncRequired extends Error {
     }
 }
 export function currentSeq(userId) {
-    return db.prepare("SELECT COALESCE(MAX(seq), 0) AS seq FROM changes WHERE user_id = ?").get(userId).seq;
+    return db
+        .prepare(`SELECT MAX(
+         COALESCE((SELECT MAX(seq) FROM changes WHERE user_id = ?), 0),
+         COALESCE((SELECT MAX(last_seq, pruned_through) FROM sync_state WHERE user_id = ?), 0)
+       ) AS seq`)
+        .get(userId, userId).seq;
 }
 /** Everything that changed after `since`, oldest first, with the current data of each object. */
 export function listChanges(userId, since, limit = 200) {
@@ -101,29 +145,63 @@ export function listChanges(userId, since, limit = 200) {
         .all(userId, since, page + 1);
     const hasMore = rows.length > page;
     const slice = rows.slice(0, page);
-    const subscriptions = new Map(listSubscriptionsForUser(userId).map((subscription) => [subscription.id, subscription]));
-    const folders = new Map(listFoldersForUser(userId).map((folder) => [folder.id, folder]));
-    const keywords = new Map(listMutedKeywords(userId).map((keyword) => [keyword.id, keyword]));
+    const hasUpsert = (entity) => slice.some((row) => row.entity === entity && row.op !== "delete");
+    const subscriptions = hasUpsert("subscription")
+        ? new Map(listSubscriptionsForUser(userId).map((subscription) => [subscription.id, subscription]))
+        : new Map();
+    const folders = hasUpsert("folder")
+        ? new Map(listFoldersForUser(userId).map((folder) => [folder.id, folder]))
+        : new Map();
+    const keywords = hasUpsert("muted_keyword")
+        ? new Map(listMutedKeywords(userId).map((keyword) => [keyword.id, keyword]))
+        : new Map();
+    const stateIds = [...new Set(slice.filter((row) => row.entity === "item_state" && row.op !== "delete").map((row) => row.entity_id))];
+    const itemStates = loadItemStates(userId, stateIds);
+    const contentIds = [...new Set(slice.filter((row) => row.entity === "item_content" && row.op !== "delete").map((row) => row.entity_id))];
+    const itemContents = loadItemContents(userId, contentIds);
+    const noteUpsertIds = [...new Set(slice.filter((row) => row.entity === "note" && row.op !== "delete").map((row) => row.entity_id))];
+    const notes = new Map();
+    if (noteUpsertIds.length) {
+        const noteRows = db
+            .prepare(`SELECT n.item_id, n.content, COALESCE(r.revision, 1) AS revision, n.created_at, n.updated_at
+         FROM item_notes n
+         LEFT JOIN item_note_revisions r ON r.user_id = n.user_id AND r.item_id = n.item_id
+         WHERE n.user_id = ? AND n.item_id IN (SELECT CAST(value AS INTEGER) FROM json_each(?))`)
+            .all(userId, JSON.stringify(noteUpsertIds));
+        for (const note of noteRows)
+            notes.set(note.item_id, note);
+    }
+    const noteDeleteIds = [...new Set(slice.filter((row) => row.entity === "note" && row.op === "delete").map((row) => row.entity_id))];
+    const noteRevisions = new Map();
+    if (noteDeleteIds.length) {
+        const revRows = db
+            .prepare(`SELECT item_id, revision FROM item_note_revisions
+         WHERE user_id = ? AND item_id IN (SELECT CAST(value AS INTEGER) FROM json_each(?))`)
+            .all(userId, JSON.stringify(noteDeleteIds));
+        for (const rev of revRows)
+            noteRevisions.set(rev.item_id, rev.revision);
+    }
     const changes = slice.map((row) => {
         const change = { seq: row.seq, entity: row.entity, id: row.entity_id, op: row.op };
         if (row.entity === "edition") {
             const state = getEditionState(userId);
-            return state.edition ? { ...change, op: "upsert", data: { ...state.edition } } :
+            const edition = getEdition(userId);
+            return edition ? { ...change, op: "upsert", data: { ...edition } } :
                 { ...change, op: "delete", data: { revision: state.revision, status: state.status } };
         }
         if (row.entity === "note" && row.op === "delete") {
-            const version = db.prepare("SELECT revision FROM item_note_revisions WHERE user_id = ? AND item_id = ?").get(userId, row.entity_id);
-            return version ? { ...change, data: { revision: version.revision } } : change;
+            const revision = noteRevisions.get(row.entity_id);
+            return revision !== undefined ? { ...change, data: { revision } } : change;
         }
         if (row.op === "delete")
             return change;
         let data;
         if (row.entity === "item_content") {
-            const item = findItemById(row.entity_id);
-            data = item && canAccessItem(userId, row.entity_id) ? contentMetadata(item) : null;
+            data = itemContents.get(row.entity_id) ?? null;
+        }
+        else if (row.entity === "item_state") {
+            data = itemStates.get(row.entity_id) ?? null;
         }
-        else if (row.entity === "item_state")
-            data = itemState(userId, row.entity_id);
         else if (row.entity === "subscription") {
             const subscription = subscriptions.get(row.entity_id);
             data = subscription && serializeSubscription(subscription);
@@ -137,7 +215,7 @@ export function listChanges(userId, since, limit = 200) {
             data = keyword && { keyword: keyword.keyword };
         }
         else if (row.entity === "note") {
-            const note = findItemNote(userId, row.entity_id);
+            const note = notes.get(row.entity_id);
             data = note && {
                 itemId: note.item_id,
                 content: note.content,
@@ -158,8 +236,11 @@ export function listChanges(userId, since, limit = 200) {
 export function pruneChangeLog() {
     const cutoff = new Date(Date.now() - TOMBSTONE_DAYS * 86_400_000).toISOString();
     return db.transaction(() => {
-        db.prepare(`UPDATE sync_state SET pruned_through = MAX(pruned_through, COALESCE(
-         (SELECT MAX(seq) FROM changes WHERE changes.user_id = sync_state.user_id AND op = 'delete' AND changed_at < ?), 0))`).run(cutoff);
+        db.prepare(`UPDATE sync_state SET
+         last_seq = MAX(last_seq, pruned_through, COALESCE(
+           (SELECT MAX(seq) FROM changes WHERE changes.user_id = sync_state.user_id), 0)),
+         pruned_through = MAX(pruned_through, COALESCE(
+           (SELECT MAX(seq) FROM changes WHERE changes.user_id = sync_state.user_id AND op = 'delete' AND changed_at < ?), 0))`).run(cutoff);
         const old = db.prepare("DELETE FROM changes WHERE op = 'delete' AND changed_at < ?").run(cutoff).changes;
         const gone = db.prepare("DELETE FROM changes WHERE entity IN ('item_state', 'item_content') AND entity_id NOT IN (SELECT id FROM items)").run().changes;
         const goneNotes = db.prepare("DELETE FROM changes WHERE entity = 'note' AND entity_id NOT IN (SELECT id FROM items)").run().changes;

package/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feedkeeper/server",
-  "version": "0.27.1",
+  "version": "0.28.0",
   "private": true,
   "type": "module"
 }