musora-content-services 2.89.0 → 2.92.3

package/src/services/sync/store/push-coalescer.ts

@@ -0,0 +1,57 @@
+import BaseModel from "../models/Base"
+import { RecordId } from "@nozbe/watermelondb"
+import { EpochMs } from ".."
+import { SyncPushResponse } from "../fetch"
+
+type PushIntent = {
+  promise: Promise<SyncPushResponse>
+  records: {
+    id: RecordId
+    updatedAt: EpochMs
+  }[]
+}
+
+export default class PushCoalescer {
+  private intents: PushIntent[]
+
+  constructor() {
+    this.intents = []
+  }
+
+  push(records: BaseModel[], pusher: (records: BaseModel[]) => Promise<SyncPushResponse>) {
+    const found = this.find(records)
+
+    if (found) {
+      return found.promise
+    }
+
+    return this.add(pusher(records), records)
+  }
+
+  private add(promise: Promise<SyncPushResponse>, records: BaseModel[]) {
+    const intent = {
+      promise,
+      records: records.map(record => ({
+        id: record.id,
+        updatedAt: record.updated_at
+      }))
+    }
+
+    const cleanup = () => this.intents.splice(this.intents.indexOf(intent), 1)
+    intent.promise.finally(cleanup)
+
+    this.intents.push(intent)
+
+    return intent.promise
+  }
+
+  private find(records: BaseModel[]) {
+    return this.intents.find(intent => {
+      return records.every(record => {
+        return intent.records.find(({ id, updatedAt }) => {
+          return id === record.id && updatedAt >= record.updated_at
+        })
+      })
+    })
+  }
+}

package/src/services/sync/store-configs.ts

@@ -0,0 +1,41 @@
+import { SyncStoreConfig } from "./store"
+import { ContentLike, ContentProgress, Practice, PracticeDayNote } from "./models"
+import { handlePull, handlePush, makeFetchRequest } from "./fetch"
+
+import type SyncStore from "./store"
+import type BaseModel from "./models/Base"
+
+export default function createStoresFromConfig(createStore: <TModel extends BaseModel>(config: SyncStoreConfig<TModel>) => SyncStore<TModel>) {
+  return [
+    createStore({
+      model: ContentLike,
+      pull: handlePull(makeFetchRequest('/api/content/v1/user/likes')),
+      push: handlePush(makeFetchRequest('/api/content/v1/user/likes', { method: 'POST' })),
+    }),
+
+    createStore({
+      model: ContentProgress,
+      comparator: (server, local) => {
+        if (server.record.progress_percent === 0 || local.progress_percent === 0) {
+          return server.meta.lifecycle.updated_at >= local.updated_at ? 'SERVER' : 'LOCAL'
+        } else {
+          return server.record.progress_percent >= local.progress_percent ? 'SERVER' : 'LOCAL'
+        }
+      },
+      pull: handlePull(makeFetchRequest('/content/user/progress')),
+      push: handlePush(makeFetchRequest('/content/user/progress', { method: 'POST' })),
+    }),
+
+    createStore({
+      model: Practice,
+      pull: handlePull(makeFetchRequest('/api/user/practices/v1')),
+      push: handlePush(makeFetchRequest('/api/user/practices/v1', { method: 'POST' })),
+    }),
+
+    createStore({
+      model: PracticeDayNote,
+      pull: handlePull(makeFetchRequest('/api/user/practices/v1/notes')),
+      push: handlePush(makeFetchRequest('/api/user/practices/v1/notes', { method: 'POST' })),
+    })
+  ] as unknown as SyncStore<BaseModel>[]
+}

package/src/services/sync/strategies/base.ts

@@ -0,0 +1,21 @@
+import { SyncStrategy } from "./index";
+import SyncContext from "../context";
+import SyncStore from "../store";
+
+export default abstract class BaseSyncStrategy implements SyncStrategy {
+  protected registry: { callback: (reason: string) => void, store: SyncStore }[] = []
+  abstract start(): void
+  abstract stop(): void
+
+  protected context: SyncContext
+
+  constructor(context: SyncContext) {
+    this.context = context
+    this.registry = []
+  }
+
+  onTrigger(store: SyncStore, callback: (reason: string) => void): this {
+    this.registry.push({ callback, store })
+    return this
+  }
+}

package/src/services/sync/strategies/index.ts

@@ -0,0 +1,12 @@
+import SyncStore from "../store";
+
+export interface SyncStrategy {
+  start(): void
+  stop(): void
+
+  onTrigger(store: SyncStore, callback: (reason: string) => void): this
+}
+
+export { default as BaseStrategy } from './base'
+export { default as InitialStrategy } from './initial'
+export { default as PollingStrategy } from './polling'

package/src/services/sync/strategies/initial.ts

@@ -0,0 +1,11 @@
+import { BaseStrategy } from '.'
+
+export default class InitialStrategy extends BaseStrategy {
+  start() {
+    this.registry.forEach(({ callback }) => {
+      callback('initial')
+    })
+  }
+
+  stop() {}
+}

package/src/services/sync/strategies/polling.ts

@@ -0,0 +1,54 @@
+import { DefaultTimers, Timers } from "../utils/timers";
+import { BaseStrategy } from "./index";
+import SyncContext from "../context";
+import SyncStore from "../store";
+
+export default class PollingStrategy extends BaseStrategy {
+  private active = false
+  private storeTimers = new Map<SyncStore, any>()
+
+  constructor(
+    context: SyncContext,
+    private intervalMs: number,
+    private readonly timers: Timers = DefaultTimers
+  ) {
+    super(context)
+  }
+
+  start() {
+    if (this.active) return
+    this.active = true
+
+    this.registry.forEach(({ store, callback }) => {
+      store.on('pullCompleted', () => {
+        this.resetTimer(store, callback)
+      })
+      this.startTimer(store, callback)
+    })
+  }
+
+  stop() {
+    this.active = false
+    this.storeTimers.forEach(timerId => this.timers.clearTimeout(timerId))
+    this.storeTimers.clear()
+  }
+
+  private startTimer(store: SyncStore, callback: (reason: string) => void) {
+    this.timers.clearTimeout(this.storeTimers.get(store))
+
+    const timerId = this.timers.setTimeout(() => {
+      if (this.context.visibility.getValue()) {
+        callback('polling')
+      }
+      this.startTimer(store, callback)
+    }, this.intervalMs)
+
+    this.storeTimers.set(store, timerId)
+  }
+
+  private resetTimer(store: SyncStore, callback: (reason: string) => void) {
+    if (this.active) {
+      this.startTimer(store, callback)
+    }
+  }
+}

package/src/services/sync/telemetry/index.ts

@@ -0,0 +1,140 @@
+import watermelonLogger from '@nozbe/watermelondb/utils/common/logger'
+import { SyncError, SyncUnexpectedError } from '../errors'
+
+import * as InjectedSentry from '@sentry/browser'
+export type SentryBrowserOptions = NonNullable<Parameters<typeof InjectedSentry.init>[0]>;
+
+export type SentryLike = {
+  captureException: typeof InjectedSentry.captureException
+  addBreadcrumb: typeof InjectedSentry.addBreadcrumb
+  startSpan: typeof InjectedSentry.startSpan
+}
+
+export type StartSpanOptions = Parameters<typeof InjectedSentry.startSpan>[0]
+export type Span = InjectedSentry.Span
+
+export const SYNC_TELEMETRY_TRACE_PREFIX = 'sync:'
+
+export class SyncTelemetry {
+  private static instance: SyncTelemetry | null = null
+
+  public static setInstance(instance: SyncTelemetry): SyncTelemetry {
+    SyncTelemetry.instance = instance
+    return instance
+  }
+
+  public static getInstance(): SyncTelemetry | null {
+    return SyncTelemetry.instance
+  }
+
+  public static clearInstance(): void {
+    SyncTelemetry.instance = null
+  }
+
+  private userId: string
+  private Sentry: SentryLike;
+
+  // allows us to know if Sentry shouldn't double-capture a dev-prettified console.error log
+  private _ignoreConsole = false
+
+  constructor(userId: string, { Sentry }: { Sentry: SentryLike }) {
+    this.userId = userId
+    this.Sentry = Sentry
+    watermelonLogger.log = (...messages: any[]) => this.log('[Watermelon]', ...messages);
+    watermelonLogger.warn = (...messages: any[]) => this.warn('[Watermelon]', ...messages);
+    watermelonLogger.error = (...messages: any[]) => this.error('[Watermelon]', ...messages);
+  }
+
+  trace<T>(opts: StartSpanOptions, callback: (_span: Span) => T) {
+    const options = {
+      ...opts,
+      name: `${SYNC_TELEMETRY_TRACE_PREFIX}${opts.name}`,
+      op: `${SYNC_TELEMETRY_TRACE_PREFIX}${opts.op}`,
+      attributes: {
+        ...opts.attributes,
+        userId: this.userId
+      }
+    }
+    return this.Sentry.startSpan<T>(options, (span) => {
+      let desc = span['_spanId'].slice(0, 4)
+      desc += span['_parentSpanId'] ? ` (< ${span['_parentSpanId'].slice(0, 4)})` : ''
+
+      this.debug(`[trace:start] ${options.name} (${desc})`)
+      const result = callback(span)
+      Promise.resolve(result).finally(() => this.debug(`[trace:end] ${options.name} (${desc})`))
+
+      return result
+    })
+  }
+
+  capture(err: SyncError) {
+    err.markReported()
+    this.Sentry.captureException(err, err instanceof SyncUnexpectedError ? {
+      mechanism: {
+        handled: false
+      }
+    } : undefined)
+
+
+    this._ignoreConsole = true
+    this.error(err.message)
+    this._ignoreConsole = false
+  }
+
+  // allows us to know if Sentry shouldn't double-capture a dev-prettified console.error log
+  shouldIgnoreConsole() {
+    return this._ignoreConsole
+  }
+
+  debug(...messages: any[]) {
+    console.debug(...this.formattedConsoleMessages(...messages));
+    this.recordBreadcrumb('debug', ...messages)
+  }
+
+  info(...messages: any[]) {
+    console.info(...this.formattedConsoleMessages(...messages));
+    this.recordBreadcrumb('info', ...messages)
+  }
+
+  log(...messages: any[]) {
+    console.log(...this.formattedConsoleMessages(...messages));
+    this.recordBreadcrumb('log', ...messages)
+  }
+
+  warn(...messages: any[]) {
+    console.warn(...this.formattedConsoleMessages(...messages));
+    this.recordBreadcrumb('warning', ...messages)
+  }
+
+  error(...messages: any[]) {
+    console.error(...this.formattedConsoleMessages(...messages));
+    this.recordBreadcrumb('error', ...messages)
+  }
+
+  fatal(...messages: any[]) {
+    console.error(...this.formattedConsoleMessages(...messages));
+    this.recordBreadcrumb('fatal', ...messages)
+  }
+
+  private recordBreadcrumb(level: InjectedSentry.Breadcrumb['level'], ...messages: any[]) {
+    this.Sentry.addBreadcrumb({
+      message: messages.join(', '),
+      level,
+      category: 'sync',
+    })
+  }
+
+  private formattedConsoleMessages(...messages: any[]) {
+    const date = new Date();
+    return [...this.consolePrefix(date), ...messages, ...this.consoleSuffix(date)];
+  }
+
+  private consolePrefix(date: Date) {
+    const now = Math.round(date.getTime() / 1000).toString();
+    return [`📡 SYNC: (%c${now.slice(0, 5)}%c${now.slice(5, 10)})`, 'color: #ccc', 'font-weight: bold;'];
+  }
+
+  private consoleSuffix(date: Date) {
+    return [` [${date.toLocaleTimeString()}, ${date.getTime()}]`];
+  }
+}

package/src/services/sync/telemetry/sampling.ts

@@ -0,0 +1,91 @@
+import { SyncTelemetry, SYNC_TELEMETRY_TRACE_PREFIX, type SentryBrowserOptions } from '.'
+import { SyncError } from '../errors'
+
+type ReturnsUndefined<T extends (...args: any[]) => any> = (...args: Parameters<T>) => ReturnType<T> | undefined
+
+export const syncSentryBeforeSend: ReturnsUndefined<NonNullable<SentryBrowserOptions['beforeSend']>> = (event, hint) => {
+  if (event.logger === 'console' && SyncTelemetry.getInstance().shouldIgnoreConsole()) {
+    return null
+  }
+
+  if (hint?.originalException instanceof SyncError) {
+    if (hint.originalException.isReported()) {
+      return null
+    }
+
+    // populate event with error data Sentry would otherwise ignore
+    event.extra = {
+      ...event.extra,
+      details: hint.originalException.getDetails()
+    }
+
+    return event
+  }
+
+  return undefined
+}
+
+export const syncSentryBeforeSendTransaction: ReturnsUndefined<NonNullable<SentryBrowserOptions['beforeSendTransaction']>> = (event, hint) => {
+  if (event.contexts?.trace?.op?.startsWith(SYNC_TELEMETRY_TRACE_PREFIX)) {
+    // filter out noisy empty sync traces
+    if (event.contexts.trace.op === `${SYNC_TELEMETRY_TRACE_PREFIX}sync` && event.spans?.length === 0) {
+      return null
+    }
+  }
+
+  return undefined
+}
+
+// sentry doesn't bother to expose your chosen environment in tracesSampler
+// so we have to make consumers pass in our greedy option
+export const createSyncSentryTracesSampler = (greedy = false) => {
+  const sampler: ReturnsUndefined<NonNullable<SentryBrowserOptions['tracesSampler']>> = (context) => {
+    if (!context.name.startsWith(SYNC_TELEMETRY_TRACE_PREFIX)) {
+      return undefined
+    }
+
+    if (greedy) {
+      return true
+    }
+
+    const { parentSampled, attributes } = context
+
+    if (parentSampled) {
+      return true
+    }
+
+    if (attributes?.userId) {
+      return userBucketedSampler(attributes.userId as string | number, 0.1)
+    }
+
+    return false
+  }
+
+  return sampler
+}
+
+/**
+ * Returns a sampling decision (0 or 1) based on user ID and rotation period.
+ *
+ * @param userId - string or number uniquely identifying the user
+ * @param sampleRate - fraction of users to include (0..1)
+ * @param rotationPeriodMs - rotation period in milliseconds (e.g., 1 day, 1 week)
+ */
+function userBucketedSampler(
+  userId: string | number,
+  sampleRate: number,
+  rotationPeriodMs: number = 7 * 24 * 60 * 60 * 1000 // default: 1 week
+): 0 | 1 {
+  const uid = typeof userId === 'string' ? hashString(userId) : userId;
+  const now = Date.now();
+  const bucket = (uid + Math.floor(now / rotationPeriodMs)) % 100;
+  return bucket < sampleRate * 100 ? 1 : 0;
+}
+
+function hashString(str: string): number {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    hash = (hash * 31 + str.charCodeAt(i)) >>> 0;
+  }
+  return hash;
+}

package/src/services/sync/utils/event-emitter.ts

@@ -0,0 +1,24 @@
+type EventMap = Record<string, any[]>;
+
+export default class EventEmitter<Events extends EventMap> {
+  private events: {
+    [K in keyof Events]?: Array<(...args: Events[K]) => void>
+  } = {};
+
+  on<K extends keyof Events>(event: K, fn: (...args: Events[K]) => void): () => void {
+    (this.events[event] ||= []).push(fn);
+    return () => this.off(event, fn);
+  }
+
+  off<K extends keyof Events>(event: K, fn?: (...args: Events[K]) => void) {
+    if (!fn) {
+      delete this.events[event];
+    } else {
+      this.events[event] = this.events[event]?.filter(f => f !== fn) || [];
+    }
+  }
+
+  emit<K extends keyof Events>(event: K, ...args: Events[K]) {
+    this.events[event]?.forEach(fn => fn(...args));
+  }
+}

package/src/services/sync/utils/index.ts

@@ -0,0 +1 @@
+export * from './throttle'

package/src/services/sync/utils/throttle.ts

@@ -0,0 +1,93 @@
+export interface ThrottleState<T> {
+  isWaiting: boolean
+  current: Promise<T> | null;
+  next: (() => Promise<T>) | null;
+  lastCallTime: number;
+  minIntervalMs: number;
+}
+
+export function createThrottleState<T>(minIntervalMs: number): ThrottleState<T> {
+  return {
+    isWaiting: false,
+    current: null,
+    next: null,
+    lastCallTime: 0,
+    minIntervalMs
+  };
+}
+
+export function dropThrottle<T>(
+  options: { state: ThrottleState<T>, deferOnce?: boolean },
+  fn: (..._args: any[]) => Promise<T>
+) {
+  return (...args: any[]) => {
+    const { state } = options
+
+    const wait = () => {
+      return new Promise<void>(resolve => {
+        state.isWaiting = true
+        const t = Date.now() - state.lastCallTime
+        if (t < state.minIntervalMs) {
+          setTimeout(resolve, state.minIntervalMs - t)
+        } else {
+          resolve()
+        }
+      }).finally(() => {
+        state.isWaiting = false
+      })
+    }
+
+    const run = (fn: (..._args: any[]) => Promise<T>, args: any[]) => {
+      state.lastCallTime = Date.now()
+      return fn(...args).finally(() => {
+        if (state.next) {
+          const n = state.next()
+          state.next = null
+          state.current = n
+        } else {
+          state.current = null
+        }
+      })
+    }
+
+    if (!state.current) {
+      state.current = wait().then(() => run(fn, args))
+    } else if (options.deferOnce && !state.isWaiting) {
+      state.next = () => run(fn, args)
+    }
+
+    return state.current
+  }
+}
+
+export function queueThrottle<T>(
+  options: { state: ThrottleState<T>, deferOnce?: boolean },
+  fn: (..._args: any[]) => Promise<T>
+) {
+  return (...args: any[]) => {
+    const { state } = options
+
+    const run = async () => {
+      const elapsed = Date.now() - state.lastCallTime
+      if (elapsed < state.minIntervalMs) {
+        await new Promise<void>(r => setTimeout(r, state.minIntervalMs - elapsed))
+      }
+
+      state.lastCallTime = Date.now()
+      return fn(...args).finally(() => {
+        const next = state.next
+        state.next = null
+        state.current = next ? next() : null
+      })
+    }
+
+    if (!state.current) {
+      state.current = run()
+    } else {
+      state.next = () => run()
+    }
+
+    return state.current
+  }
+
+}

package/src/services/sync/utils/timers.ts

@@ -0,0 +1,9 @@
+export interface Timers {
+  setTimeout(fn: () => void, ms: number): any
+  clearTimeout(id: any): void
+}
+
+export const DefaultTimers: Timers = {
+  setTimeout: (fn, ms) => setTimeout(fn, ms),
+  clearTimeout: (id) => clearTimeout(id),
+}