musora-content-services 2.118.1 → 2.119.0

package/src/services/sync/store/index.ts

@@ -1,4 +1,4 @@
-import { Database, Q, type Collection, type RecordId } from '@nozbe/watermelondb'
+import { Database, Q, Query, type Collection, type RecordId } from '@nozbe/watermelondb'
 import { RawSerializer, ModelSerializer } from '../serializers'
 import { ModelClass, SyncToken, SyncEntry,  SyncContext, EpochMs } from '..'
 import { SyncPullResponse, SyncPushResponse, SyncPullFetchFailureResponse, PushPayload, SyncStorePushResultSuccess, SyncStorePushResultFailure } from '../fetch'
@@ -17,7 +17,6 @@ import { type WriterInterface } from '@nozbe/watermelondb/Database/WorkQueue'
 import type LokiJSAdapter from '@nozbe/watermelondb/adapters/lokijs'
 import { SyncError } from '../errors'
 
-
 type SyncPull = (
   session: BaseSessionProvider,
   previousFetchToken: SyncToken | null,
@@ -39,6 +38,8 @@ export type SyncStoreConfig<TModel extends BaseModel = BaseModel> = {
 export default class SyncStore<TModel extends BaseModel = BaseModel> {
   static readonly PULL_THROTTLE_INTERVAL = 2_000
   static readonly PUSH_THROTTLE_INTERVAL = 1_000
+  static readonly DELETED_RECORD_GRACE_PERIOD = 60_000 // 60s
+  static readonly CLEANUP_INTERVAL = 60_000 * 60 // 1hr
 
   readonly telemetry: SyncTelemetry
   readonly context: SyncContext
@@ -60,6 +61,7 @@ export default class SyncStore<TModel extends BaseModel = BaseModel> {
   private pushCoalescer = new PushCoalescer()
 
   private emitter = new EventEmitter()
+  private cleanupTimer: NodeJS.Timeout | null = null
 
   private lastFetchTokenKey: string
 
@@ -91,12 +93,18 @@ export default class SyncStore<TModel extends BaseModel = BaseModel> {
     this.lastFetchTokenKey = `last_fetch_token:${this.model.table}`
 
     this.telemetry = telemetry
+
+    this.startCleanupTimer()
   }
 
   on = this.emitter.on.bind(this.emitter)
   off = this.emitter.off.bind(this.emitter)
   private emit = this.emitter.emit.bind(this.emitter)
 
+  destroy() {
+    this.stopCleanupTimer()
+  }
+
   async requestSync(reason: string) {
     inBoundary(ctx => {
       this.telemetry.trace(
@@ -176,6 +184,10 @@ export default class SyncStore<TModel extends BaseModel = BaseModel> {
     return this.queryRecordIds(...args)
   }
 
+  async queryAllDeletedIds(...args: Q.Clause[]) {
+    return this.queryMaybeDeletedRecordIds(...args)
+  }
+
   async queryOne(...args: Q.Clause[]) {
     const record = await this.queryRecord(...args)
     return record ? this.modelSerializer.toPlainObject(record) : null
@@ -359,6 +371,41 @@ export default class SyncStore<TModel extends BaseModel = BaseModel> {
     })
   }
 
+  async restoreOne(id: RecordId, span?: Span) {
+    return this.restoreSome([id], span).then(r => r[0])
+  }
+
+  async restoreSome(ids: RecordId[], span?: Span) {
+    return this.runScope.abortable(async () => {
+      const records = await this.telemeterizedWrite(span, async writer => {
+        const records = await writer.callReader(() => this.queryMaybeDeletedRecords(
+          Q.where('id', Q.oneOf(ids)),
+          Q.where('_status', 'deleted')
+        ))
+
+        const destroyBuilds = records.map(record => new this.model(this.collection, { id: record.id }).prepareDestroyPermanently())
+        const createBuilds = records.map(record => this.collection.prepareCreate((r) => {
+          Object.keys(record._raw).forEach((key) => {
+            r._raw[key] = record._raw[key]
+          })
+          r._raw._status = 'updated'
+        }))
+
+        await writer.batch(...destroyBuilds)
+        await writer.batch(...createBuilds)
+
+        return createBuilds
+      })
+
+      this.emit('upserted', records)
+
+      this.pushUnsyncedWithRetry(span)
+      await this.ensurePersistence()
+
+      return records.map((record) => this.modelSerializer.toPlainObject(record))
+    })
+  }
+
   async importUpsert(recordRaws: TModel['_raw'][]) {
     await this.runScope.abortable(async () => {
       await this.telemeterizedWrite(undefined, async writer => {
@@ -637,31 +684,7 @@ export default class SyncStore<TModel extends BaseModel = BaseModel> {
     return this.db.read(async () => {
       const undeletedRecords = await this.collection.query(...args).fetch()
 
-      const serializedQuery = this.collection.query(...args).serialize()
-      const adjustedQuery = {
-        ...serializedQuery,
-        description: {
-          ...serializedQuery.description,
-          where: [
-            // remove the default "not deleted" clause added by WatermelonDB
-            ...serializedQuery.description.where.filter(
-              (w) =>
-                !(
-                  w.type === 'where' &&
-                  w.left === '_status' &&
-                  w.comparison &&
-                  w.comparison.operator === 'notEq' &&
-                  w.comparison.right &&
-                  'value' in w.comparison.right &&
-                  w.comparison.right.value === 'deleted'
-                )
-            ),
-
-            // and add our own "include deleted" clause
-            Q.where('_status', Q.eq('deleted'))
-          ],
-        },
-      }
+      const adjustedQuery = this.maybeDeletedQuery(this.collection.query(...args))
 
       // NOTE: constructing models in this way is a bit of a hack,
       // but since deleted records aren't "resurrectable" in WatermelonDB anyway,
@@ -682,6 +705,54 @@ export default class SyncStore<TModel extends BaseModel = BaseModel> {
     })
   }
 
+  /**
+   * Query records including ones marked as deleted
+   * WatermelonDB by default excludes deleted records from queries
+   */
+  private async queryMaybeDeletedRecordIds(...args: Q.Clause[]) {
+    return this.db.read(async () => {
+      const undeletedRecordIds = await this.collection.query(...args).fetchIds()
+
+      const adjustedQuery = this.maybeDeletedQuery(this.collection.query(...args))
+      const deletedRecordIds = (await this.db.adapter.unsafeQueryRaw(adjustedQuery)).map(r => r.id)
+
+      return [
+        ...undeletedRecordIds,
+        ...deletedRecordIds,
+      ]
+    })
+  }
+
+  private maybeDeletedQuery(query: Query<TModel>) {
+    const serializedQuery = query.serialize()
+    const adjustedQuery = {
+      ...serializedQuery,
+      description: {
+        ...serializedQuery.description,
+        where: [
+          // remove the default "not deleted" clause added by WatermelonDB
+          ...serializedQuery.description.where.filter(
+            (w) =>
+              !(
+                w.type === 'where' &&
+                w.left === '_status' &&
+                w.comparison &&
+                w.comparison.operator === 'notEq' &&
+                w.comparison.right &&
+                'value' in w.comparison.right &&
+                w.comparison.right.value === 'deleted'
+              )
+          ),
+
+          // and add our own "include deleted" clause
+          Q.where('_status', Q.eq('deleted'))
+        ],
+      },
+    }
+
+    return adjustedQuery
+  }
+
   // Avoid lazy persistence to IndexedDB
   // to eliminate data loss risk due to tab close/crash before flush to IndexedDB
   // https://github.com/Nozbe/WatermelonDB/issues/1329
@@ -739,6 +810,9 @@ export default class SyncStore<TModel extends BaseModel = BaseModel> {
   }
 
   private async buildWriteBatchesFromEntries(writer: WriterInterface, entries: SyncEntry[], freshSync: boolean) {
+    // Clean up old deleted records during pull operations
+    await this.cleanupOldDeletedRecords(writer)
+
     // if this is a fresh sync and there are no existing records, we can skip more sophisticated conflict resolution
     if (freshSync) {
       if ((await writer.callReader(() => this.queryMaybeDeletedRecords())).length === 0) {
@@ -747,7 +821,7 @@ export default class SyncStore<TModel extends BaseModel = BaseModel> {
           .filter((e) => !e.meta.lifecycle.deleted_at)
           .forEach((entry) => resolver.againstNone(entry))
 
-        return this.prepareRecords(resolver.result)
+        return this.prepareRecords(resolver.result, new Map())
       }
     }
 
@@ -789,17 +863,26 @@ export default class SyncStore<TModel extends BaseModel = BaseModel> {
       }
     })
 
-    return this.prepareRecords(resolver.result)
+    return this.prepareRecords(resolver.result, existingRecordsMap)
   }
 
-  private prepareRecords(result: SyncResolution) {
+  private prepareRecords(result: SyncResolution, existingRecordsMap: Map<RecordId, TModel>) {
     if (Object.values(result).find((c) => c.length)) {
       this.telemetry.debug(`[store:${this.model.table}] Writing changes`, { changes: result })
     }
 
-    const destroyedBuilds = result.idsForDestroy.map((id) => {
-      return new this.model(this.collection, { id }).prepareDestroyPermanently()
-    })
+    const destroyedBuilds = result.idsForDestroy
+      .filter(id => {
+        // Only permanently delete if updated_at is older than grace period
+        const record = existingRecordsMap.get(id)
+        if (!record) return true // If no record found, safe to destroy
+
+        const gracePeriodAgo = Date.now() - SyncStore.DELETED_RECORD_GRACE_PERIOD
+        return record.updated_at < gracePeriodAgo
+      })
+      .map((id) => {
+        return new this.model(this.collection, { id }).prepareDestroyPermanently()
+      })
     const createdBuilds = result.entriesForCreate.map((entry) => {
       return this.collection.prepareCreate((r) => {
         Object.entries(entry.record!).forEach(([key, value]) => {
@@ -856,4 +939,42 @@ export default class SyncStore<TModel extends BaseModel = BaseModel> {
   private isLokiAdapter(adapter: any): adapter is LokiJSAdapter {
     return adapter._driver && 'loki' in adapter._driver
   }
+
+  private startCleanupTimer() {
+    this.cleanupTimer = setInterval(() => {
+      this.runScope.abortable(async () => {
+        this.telemeterizedWrite(undefined, async (writer) => {
+          await this.cleanupOldDeletedRecords(writer)
+        })
+      })
+    }, SyncStore.CLEANUP_INTERVAL)
+  }
+
+  private stopCleanupTimer() {
+    if (this.cleanupTimer) {
+      clearInterval(this.cleanupTimer)
+      this.cleanupTimer = null
+    }
+  }
+
+  /** Destroy permanently records past their grace period
+   * (we need to keep records around after being marked deleted
+   * for undo purposes, so we don't discard them in writeEntries
+   * (after a server push), but instead every hour or so)
+  */
+  private async cleanupOldDeletedRecords(writer: WriterInterface) {
+    const gracePeriodAgo = Date.now() - SyncStore.DELETED_RECORD_GRACE_PERIOD
+
+    const oldDeletedRecords = await writer.callReader(() => this.queryMaybeDeletedRecords(
+      Q.where('_status', 'deleted'),
+      Q.where('updated_at', Q.lt(gracePeriodAgo))
+    ))
+
+    if (oldDeletedRecords.length > 0) {
+      this.telemetry.debug(`[store:${this.model.table}] Cleaning up ${oldDeletedRecords.length} old deleted records`)
+
+      const destroyBuilds = oldDeletedRecords.map(record => record.prepareDestroyPermanently())
+      return writer.batch(...destroyBuilds)
+    }
+  }
 }

package/src/services/userActivity.js

@@ -341,7 +341,7 @@ export async function removeUserPractice(id) {
 }
 
 /**
- * Restores a previously deleted user's practice session by ID, updating both the local and remote activity context.
+ * Restores a previously deleted user's practice session by ID
  *
  * @param {number} id - The unique identifier of the practice session to be restored.
  * @returns {Promise<Object>} - A promise that resolves to the response containing the restored practice session data.
@@ -353,35 +353,7 @@ export async function removeUserPractice(id) {
  *   .catch(error => console.error(error));
  */
 export async function restoreUserPractice(id) {
-  const url = `/api/user/practices/v1/practices/restore${buildQueryString([id])}`
-  const response = await PUT(url, null)
-  if (response?.data?.length) {
-    const restoredPractice = response.data.find((p) => p.id === id)
-    if (restoredPractice) {
-      await userActivityContext.updateLocal(async function (localContext) {
-        if (!localContext.data[DATA_KEY_PRACTICES][restoredPractice.day]) {
-          localContext.data[DATA_KEY_PRACTICES][restoredPractice.day] = []
-        }
-        response.data.forEach((restoredPractice) => {
-          localContext.data[DATA_KEY_PRACTICES][restoredPractice.day].push({
-            id: restoredPractice.id,
-            duration_seconds: restoredPractice.duration_seconds,
-          })
-        })
-      })
-    }
-  }
-  const formattedMeta = await formatPracticeMeta(response.data || [])
-  const practiceDuration = formattedMeta.reduce(
-    (total, practice) => total + (practice.duration || 0),
-    0
-  )
-  return {
-    data: formattedMeta,
-    message: response.message,
-    version: response.version,
-    practiceDuration,
-  }
+  return await db.practices.restoreOne(id)
 }
 
 /**
@@ -422,25 +394,10 @@ export async function deletePracticeSession(day) {
  *   .catch(error => console.error("Restore failed:", error));
  */
 export async function restorePracticeSession(date) {
-  const url = `/api/user/practices/v1/practices/restore?date=${date}`
-  const response = await PUT(url, null)
-
-  if (response?.data) {
-    await userActivityContext.updateLocal(async function (localContext) {
-      if (!localContext.data[DATA_KEY_PRACTICES][date]) {
-        localContext.data[DATA_KEY_PRACTICES][date] = []
-      }
-
-      response.data.forEach((restoredPractice) => {
-        localContext.data[DATA_KEY_PRACTICES][date].push({
-          id: restoredPractice.id,
-          duration_seconds: restoredPractice.duration_seconds,
-        })
-      })
-    })
-  }
+  const ids = await db.practices.queryAllDeletedIds(Q.where('date', date))
+  const response = await db.practices.restoreSome(ids.data)
 
-  const formattedMeta = await formatPracticeMeta(response?.data)
+  const formattedMeta = await formatPracticeMeta(response.data)
   const practiceDuration = formattedMeta.reduce(
     (total, practice) => total + (practice.duration || 0),
     0

package/.claude/settings.local.json

@@ -1,16 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(//app/musora-platform-backend/**)",
-      "Read(//app/musora-platform-frontend/**)",
-      "Bash(find:*)",
-      "Bash(sed:*)",
-      "Read(//app/**)",
-      "Bash(cat:*)",
-      "Bash(docker exec:*)",
-      "Bash(npm config:*)"
-    ],
-    "deny": [],
-    "ask": []
-  }
-}

package/.yarnrc.yml

@@ -1 +0,0 @@
-nodeLinker: node-modules

package/check_content.js

@@ -1,30 +0,0 @@
-const { initializeService } = require('./src/services/config.js');
-const { fetchByRailContentIds } = require('./src/services/sanity.js');
-require('dotenv/config');
-
-async function checkContent() {
-  initializeService({
-    sanityConfig: {
-      token: process.env.SANITY_TOKEN,
-      projectId: process.env.SANITY_PROJECT_ID,
-      dataset: process.env.SANITY_DATASET,
-      version: process.env.SANITY_VERSION || '2021-06-07',
-    },
-    railcontentConfig: {
-      token: process.env.RAILCONTENT_TOKEN,
-      userId: process.env.RAILCONTENT_USER_ID,
-      baseUrl: process.env.RAILCONTENT_BASE_URL,
-      authToken: process.env.RAILCONTENT_AUTH_TOKEN,
-    },
-    baseUrl: process.env.RAILCONTENT_BASE_URL,
-    localStorage: null,
-    isMA: false,
-  });
-
-  console.log('Checking railcontent_id: 421814');
-  const contents = await fetchByRailContentIds([421814]);
-  console.log('Results:', JSON.stringify(contents, null, 2));
-  console.log('Found:', contents.length, 'items');
-}
-
-checkContent().catch(console.error);

package/check_content.mjs

@@ -1,32 +0,0 @@
-import { initializeService } from './src/services/config.js';
-import { fetchByRailContentIds } from './src/services/sanity.js';
-import dotenv from 'dotenv';
-
-dotenv.config();
-
-async function checkContent() {
-  initializeService({
-    sanityConfig: {
-      token: process.env.SANITY_TOKEN,
-      projectId: process.env.SANITY_PROJECT_ID,
-      dataset: process.env.SANITY_DATASET,
-      version: process.env.SANITY_VERSION || '2021-06-07',
-    },
-    railcontentConfig: {
-      token: process.env.RAILCONTENT_TOKEN,
-      userId: process.env.RAILCONTENT_USER_ID,
-      baseUrl: process.env.RAILCONTENT_BASE_URL,
-      authToken: process.env.RAILCONTENT_AUTH_TOKEN,
-    },
-    baseUrl: process.env.RAILCONTENT_BASE_URL,
-    localStorage: null,
-    isMA: false,
-  });
-
-  console.log('Checking railcontent_id: 421814');
-  const contents = await fetchByRailContentIds([421814]);
-  console.log('Results:', JSON.stringify(contents, null, 2));
-  console.log('Found:', contents.length, 'items');
-}
-
-checkContent().catch(console.error);