musora-content-services 2.89.0 → 2.92.3

package/src/services/sync/fetch.ts

@@ -0,0 +1,310 @@
+import { SyncToken, SyncEntry, SyncSyncable } from "./index"
+import { EpochMs } from "."
+
+import { globalConfig } from '../config.js'
+import { RecordId } from "@nozbe/watermelondb"
+import BaseModel from "./models/Base"
+import { BaseSessionProvider } from "./context/providers"
+
+interface RawPullResponse {
+  meta: {
+    since: EpochMs | null
+    max_updated_at: EpochMs | null
+    timestamp: EpochMs
+  }
+  entries: SyncEntry<BaseModel, 'client_record_id'>[]
+}
+
+interface RawPushResponse {
+  meta: {
+    timestamp: EpochMs
+  }
+  results: SyncStorePushResult<'client_record_id'>[]
+}
+
+export type SyncResponse = SyncPushResponse | SyncPullResponse
+
+export type SyncPushResponse = SyncPushSuccessResponse | SyncPushFailureResponse
+
+type SyncPushSuccessResponse = SyncPushResponseBase & {
+  ok: true
+  results: SyncStorePushResult[]
+}
+type SyncPushFailureResponse = SyncPushResponseBase & {
+  ok: false,
+  originalError: Error
+}
+interface SyncPushResponseBase extends SyncResponseBase {
+
+}
+
+type SyncStorePushResult<TRecordKey extends string = 'id'> = SyncStorePushResultSuccess<TRecordKey> | SyncStorePushResultFailure<TRecordKey>
+type SyncStorePushResultSuccess<TRecordKey extends string = 'id'> = SyncStorePushResultBase & {
+  type: 'success'
+  entry: SyncEntry<BaseModel, TRecordKey>
+}
+type SyncStorePushResultFailure<TRecordKey extends string = 'id'> = SyncStorePushResultProcessingFailure<TRecordKey> | SyncStorePushResultValidationFailure<TRecordKey>
+type SyncStorePushResultProcessingFailure<TRecordKey extends string = 'id'> = SyncStorePushResultFailureBase<TRecordKey> & {
+  failureType: 'processing'
+  error: any
+}
+type SyncStorePushResultValidationFailure<TRecordKey extends string = 'id'> = SyncStorePushResultFailureBase<TRecordKey> & {
+  failureType: 'validation'
+  errors: Record<string, string[]>
+}
+interface SyncStorePushResultFailureBase<TRecordKey extends string = 'id'> extends SyncStorePushResultBase {
+  type: 'failure'
+  failureType: string
+  ids: { [K in TRecordKey]: RecordId }
+}
+interface SyncStorePushResultBase {
+  type: 'success' | 'failure'
+}
+
+export type SyncPullResponse = SyncPullSuccessResponse | SyncPullFailureResponse
+
+type SyncPullSuccessResponse = SyncPullResponseBase & {
+  ok: true
+  entries: SyncEntry[]
+  token: SyncToken
+  previousToken: SyncToken | null
+}
+type SyncPullFailureResponse = SyncPullResponseBase & {
+  ok: false,
+  originalError: Error
+}
+interface SyncPullResponseBase extends SyncResponseBase {
+
+}
+export interface SyncResponseBase {
+  ok: boolean
+}
+
+export type PushPayload = {
+  entries: ({
+    record: SyncSyncable
+    meta: {
+      ids: {
+        id: RecordId
+      }
+      deleted: false
+    }
+  } | {
+    record: null
+    meta: {
+      ids: {
+        id: RecordId
+      }
+      deleted: true
+    }
+  })[]
+}
+
+interface ServerPushPayload {
+  entries: {
+    record: SyncSyncable<BaseModel, 'client_record_id'> | null
+    meta: {
+      ids: {
+        client_record_id: RecordId
+      },
+      deleted: boolean
+    }
+  }[]
+}
+
+export function makeFetchRequest(input: RequestInfo, init?: RequestInit): (session: BaseSessionProvider) => Request {
+  return (session) => new Request(globalConfig.baseUrl + input, {
+    ...init,
+    headers: {
+      ...init?.headers,
+      'Content-Type': 'application/json',
+      'X-Sync-Client-Id': session.getClientId(),
+      ...(session.getSessionId() ? {
+        'X-Sync-Client-Session-Id': session.getSessionId()!
+      } : {})
+    }
+  })
+}
+
+export function handlePull(callback: (session: BaseSessionProvider) => Request) {
+  return async function(session: BaseSessionProvider, lastFetchToken: SyncToken | null, signal?: AbortSignal): Promise<SyncPullResponse> {
+    const generatedRequest = callback(session)
+    const url = serializePullUrlQuery(generatedRequest.url, lastFetchToken)
+    const request = new Request(url, {
+      credentials: 'include',
+      headers: generatedRequest.headers,
+      signal
+    });
+
+    let response: Response | null = null
+    try {
+      response = await performFetch(request)
+    } catch (e) {
+      return {
+        ok: false,
+        originalError: e
+      }
+    }
+
+    const json = await response.json() as RawPullResponse
+    const data = deserializePullResponse(json)
+
+    // if no max_updated_at, at least use the server's timestamp
+    // useful for recording that we have at least tried fetching even though resultset empty
+    const token = data.meta.max_updated_at || data.meta.timestamp
+    const previousToken = data.meta.since
+
+    const entries = data.entries
+
+    return {
+      ok: true,
+      entries,
+      token,
+      previousToken
+    }
+  }
+}
+
+export function handlePush(callback: (session: BaseSessionProvider) => Request) {
+  return async function(session: BaseSessionProvider, payload: PushPayload, signal?: AbortSignal): Promise<SyncPushResponse> {
+    const generatedRequest = callback(session)
+    const serverPayload = serializePushPayload(payload)
+    const request = new Request(generatedRequest, {
+      credentials: 'include',
+      body: JSON.stringify(serverPayload),
+      signal
+    })
+
+    let response: Response | null = null
+    try {
+      response = await performFetch(request)
+    } catch (e) {
+      return {
+        ok: false,
+        originalError: e
+      }
+    }
+
+    const json = await response.json() as RawPushResponse
+    const data = deserializePushResponse(json)
+
+    return {
+      ok: true,
+      results: data.results
+    }
+  }
+}
+
+async function performFetch(request: Request) {
+  const response = await fetch(request)
+  const isRetryable = (response.status >= 500 && response.status < 504) || response.status === 429 || response.status === 408
+
+  if (isRetryable) {
+    throw new Error(`Server returned ${response.status}`)
+  }
+
+  return response
+}
+
+function serializePullUrlQuery(url: string, fetchToken: SyncToken | null) {
+  const queryString = url.replace(/^[^?]*\??/, '');
+  const searchParams = new URLSearchParams(queryString);
+  if (fetchToken) {
+    searchParams.set('since', fetchToken.toString())
+  }
+  return url + '?' + searchParams.toString()
+}
+
+function deserializePullResponse(response: RawPullResponse) {
+  return {
+    ...response,
+    entries: response.entries.map(entry => {
+      return {
+        ...entry,
+        record: deserializeRecord(entry.record),
+        meta: {
+          ...entry.meta,
+          ids: deserializeIds(entry.meta.ids)
+        }
+      }
+    })
+  }
+}
+
+function serializePushPayload(payload: PushPayload): ServerPushPayload {
+  return {
+    ...payload,
+    entries: payload.entries.map(entry => {
+      return {
+        record: serializeRecord(entry.record),
+        meta: {
+          ...entry.meta,
+          ids: serializeIds(entry.meta.ids)
+        }
+      }
+    })
+  }
+}
+
+function deserializePushResponse(response: RawPushResponse) {
+  return {
+    results: response.results.map(result => {
+      if (result.type === 'success') {
+        const entry = result.entry
+
+        return {
+          ...result,
+          entry: {
+            ...entry,
+            record: deserializeRecord(entry.record),
+            meta: {
+              ...entry.meta,
+              ids: deserializeIds(entry.meta.ids)
+            }
+          }
+        }
+      } else {
+        return {
+          ...result,
+          ids: deserializeIds(result.ids)
+        }
+      }
+    })
+  }
+}
+
+function serializeRecord(record: SyncSyncable<BaseModel, 'id'> | null): SyncSyncable<BaseModel, 'client_record_id'> | null {
+  if (record) {
+    const { id, ...rest } = record
+    return {
+      ...rest,
+      client_record_id: id
+    }
+  }
+
+  return null
+}
+
+function serializeIds(ids: { id: RecordId }): { client_record_id: RecordId } {
+  return {
+    client_record_id: ids.id
+  }
+}
+
+function deserializeRecord(record: SyncSyncable<BaseModel, 'client_record_id'> | null): SyncSyncable<BaseModel, 'id'> | null {
+  if (record) {
+    const { client_record_id: id, ...rest } = record
+    return {
+      ...rest,
+      id
+    }
+  }
+
+  return null
+}
+
+function deserializeIds(ids: { client_record_id: RecordId }): { id: RecordId } {
+  return {
+    id: ids.client_record_id
+  }
+}

package/src/services/sync/index.ts

@@ -0,0 +1,80 @@
+import './telemetry/index'
+
+import { Q, RecordId } from "@nozbe/watermelondb"
+import { type ModelSerialized, type RawSerialized } from "./serializers"
+import BaseModel from "./models/Base"
+
+export { default as db } from './repository-proxy'
+export { Q }
+
+export { default as SyncSession } from './run-scope'
+export { default as SyncRetry } from './retry'
+export { default as SyncManager } from './manager'
+export { default as SyncContext } from './context'
+export { SyncError } from './errors'
+
+type Branded<T, B extends string> = T & { __brand: B }
+export type EpochMs = Branded<number, 'EpochMs'>
+export type SyncToken = EpochMs
+
+export type SyncSyncable<TModel extends BaseModel = BaseModel, TRecordKey extends string = 'id'> = {
+  [K in TRecordKey]: RecordId
+} & Omit<RawSerialized<TModel>, 'id'>
+
+export type SyncEntry<TModel extends BaseModel = BaseModel, TRecordKey extends string = 'id'> = {
+  record: SyncSyncable<TModel, TRecordKey> | null
+  meta: {
+    ids: { [K in TRecordKey]: RecordId }
+    lifecycle: {
+      created_at: EpochMs
+      updated_at: EpochMs
+      deleted_at: EpochMs | null
+    }
+  }
+}
+export type SyncEntryNonDeleted<TModel extends BaseModel = BaseModel, TRecordKey extends string = 'id'> = {
+  record: SyncSyncable<TModel, TRecordKey>
+  meta: {
+    ids: { [K in TRecordKey]: RecordId }
+    lifecycle: {
+      created_at: EpochMs
+      updated_at: EpochMs
+      deleted_at: null
+    }
+  }
+}
+
+export type SyncExistsDTO<_TModel extends BaseModel, T extends boolean | boolean[]> = {
+  data: T
+  status: 'fresh' | 'stale'
+  pullStatus: 'success' | 'pending' | 'failure' | null
+  lastFetchToken: SyncToken | null
+}
+
+export type SyncReadData<T extends BaseModel> = ModelSerialized<T> | ModelSerialized<T>[] | RecordId | RecordId[] | null
+export type SyncReadDTO<T extends BaseModel, TData extends SyncReadData<T>> = {
+  data: TData
+  status: 'fresh' | 'stale'
+  pullStatus: 'success' | 'pending' | 'failure' | null
+  lastFetchToken: SyncToken | null
+}
+
+export type SyncWriteData<T extends BaseModel> = SyncWriteRecordData<T> | SyncWriteIdData<T>
+export type SyncWriteRecordData<T extends BaseModel> = ModelSerialized<T> | ModelSerialized<T>[]
+export type SyncWriteIdData<_T extends BaseModel> = RecordId | RecordId[]
+export type SyncWriteDTO<T extends BaseModel, TData extends SyncWriteData<T>> = {
+  data: TData
+  status: 'synced' | 'unsynced'
+  pushStatus: 'success' | 'pending' | 'failure'
+}
+
+export type SyncRemoteWriteDTO<T extends BaseModel> = {
+  data: ModelSerialized<T> | null
+  status: 'synced'
+  pushStatus: 'success'
+}
+
+export type ModelClass<T extends BaseModel = BaseModel> = {
+  new (...args: any[]): T
+  table: string
+}

package/src/services/sync/manager.ts

@@ -0,0 +1,139 @@
+import BaseModel from './models/Base'
+import { Database } from '@nozbe/watermelondb'
+import SyncRunScope from './run-scope'
+import { SyncStrategy } from './strategies'
+import { default as SyncStore, SyncStoreConfig } from './store'
+
+import { ModelClass } from './index'
+import SyncRetry from './retry'
+import SyncContext from './context'
+import { SyncError } from './errors'
+import { SyncConcurrencySafetyMechanism } from './concurrency-safety'
+import { SyncTelemetry } from './telemetry/index'
+import { inBoundary } from './errors/boundary'
+import createStoresFromConfig from './store-configs'
+
+export default class SyncManager {
+  private static instance: SyncManager | null = null
+
+  public static assignAndSetupInstance(instance: SyncManager) {
+    if (SyncManager.instance) {
+      throw new SyncError('SyncManager already initialized')
+    }
+    SyncManager.instance = instance
+    const teardown = instance.setup()
+    return async () => {
+      await teardown()
+      SyncManager.instance = null
+    }
+  }
+
+  public static getInstance(): SyncManager {
+    if (!SyncManager.instance) {
+      throw new SyncError('SyncManager not initialized')
+    }
+    return SyncManager.instance
+  }
+
+  public telemetry: SyncTelemetry
+  private database: Database
+  private context: SyncContext
+  private storesRegistry: Record<string, SyncStore<any>>
+  private runScope: SyncRunScope
+  private retry: SyncRetry
+  private strategyMap: { stores: SyncStore<any>[]; strategies: SyncStrategy[] }[]
+  private safetyMap: { stores: SyncStore<any>[]; mechanisms: (() => void)[] }[]
+
+  constructor(context: SyncContext, initDatabase: () => Database) {
+    this.telemetry = SyncTelemetry.getInstance()!
+    this.context = context
+
+    this.database = this.telemetry.trace({ name: 'db:init' }, () => inBoundary(initDatabase))
+
+    this.runScope = new SyncRunScope()
+    this.retry = new SyncRetry(this.context, this.telemetry)
+
+    this.storesRegistry = this.registerStores(createStoresFromConfig(this.createStore.bind(this)))
+
+    this.strategyMap = []
+    this.safetyMap = []
+  }
+
+  createStore<TModel extends BaseModel>(config: SyncStoreConfig<TModel>) {
+    return new SyncStore<TModel>(config, this.context, this.database, this.retry, this.runScope, this.telemetry)
+  }
+
+  registerStores<TModel extends BaseModel>(stores: SyncStore<TModel>[]) {
+    return Object.fromEntries(stores.map(store => {
+      return [store.model.table, store]
+    })) as Record<string, SyncStore<TModel>>
+  }
+
+  storesForModels(models: ModelClass[]) {
+    return models.map(model => this.storesRegistry[model.table])
+  }
+
+  createStrategy<T extends SyncStrategy, U extends any[]>(
+    strategyClass: new (context: SyncContext, ...args: U) => T,
+    ...args: U
+  ): T {
+    return new strategyClass(this.context, ...args)
+  }
+
+  syncStoresWithStrategies(stores: SyncStore<any>[], strategies: SyncStrategy[]) {
+    this.strategyMap.push({ stores, strategies })
+  }
+
+  protectStores(
+    stores: SyncStore<any>[],
+    mechanisms: SyncConcurrencySafetyMechanism[]
+  ) {
+    const teardowns = mechanisms.map(mechanism => mechanism(this.context, stores))
+    this.safetyMap.push({ stores, mechanisms: teardowns })
+  }
+
+  setup() {
+    this.telemetry.debug('[SyncManager] Setting up')
+
+    this.context.start()
+    this.retry.start()
+
+    this.strategyMap.forEach(({ stores, strategies }) => {
+      strategies.forEach(strategy => {
+        stores.forEach(store => {
+          strategy.onTrigger(store, reason => {
+            store.requestSync(reason)
+          })
+        })
+        strategy.start()
+      })
+    })
+
+    const teardown = async () => {
+      this.telemetry.debug('[SyncManager] Tearing down')
+      this.runScope.abort()
+      this.strategyMap.forEach(({ strategies }) => strategies.forEach(strategy => strategy.stop()))
+      this.safetyMap.forEach(({ mechanisms }) => mechanisms.forEach(mechanism => mechanism()))
+      this.retry.stop()
+      this.context.stop()
+      await this.database.write(() => this.database.unsafeResetDatabase())
+    }
+    return teardown
+  }
+
+  getStore<TModel extends BaseModel>(model: ModelClass<TModel>) {
+    const store = this.storesRegistry[model.table]
+    if (!store) {
+      throw new SyncError(`Store not found`, { table: model.table })
+    }
+    return store as unknown as SyncStore<TModel>
+  }
+
+  getTelemetry() {
+    return this.telemetry
+  }
+
+  getContext() {
+    return this.context
+  }
+}

package/src/services/sync/models/Base.ts

@@ -0,0 +1,47 @@
+import { Model, Collection, RawRecord } from '@nozbe/watermelondb'
+import { EpochMs } from '..'
+
+export default abstract class BaseModel<ExtraRaw extends object = {}> extends Model {
+  declare _raw: RawRecord & ExtraRaw & {
+    created_at: EpochMs
+    updated_at: EpochMs
+  }
+
+  get created_at() {
+    return this._getRaw('created_at') as EpochMs
+  }
+
+  get updated_at() {
+    return this._getRaw('updated_at') as EpochMs
+  }
+
+  static _prepareCreate(
+    collection: Collection<Model>,
+    recordBuilder: (record: Model) => void
+  ) {
+    return super._prepareCreate(collection, (record: Model) => {
+      const now = Date.now() as EpochMs
+      record._raw['created_at'] = now
+      record._raw['updated_at'] = now
+      recordBuilder(record)
+    })
+  }
+
+  prepareUpdate(recordBuilder: ((record: this) => void) | undefined) {
+    return super.prepareUpdate((record: this) => {
+      record._raw['updated_at'] = Date.now() as EpochMs
+      if (recordBuilder) recordBuilder(record)
+    })
+  }
+
+  prepareMarkAsDeleted() {
+    return super.prepareUpdate((record: this) => {
+      if (record._raw._status === 'deleted') {
+        return
+      }
+
+      record._raw['updated_at'] = Date.now() as EpochMs
+      record._raw._status = 'deleted'
+    })
+  }
+}

package/src/services/sync/models/ContentLike.ts

@@ -0,0 +1,16 @@
+import { SYNC_TABLES } from '../schema'
+import BaseModel from './Base'
+
+export default class ContentLike extends BaseModel<{
+  content_id: number
+}> {
+  static table = SYNC_TABLES.CONTENT_LIKES
+
+  get content_id() {
+    return this._getRaw('content_id') as number
+  }
+
+  set content_id(value: number) {
+    this._setRaw('content_id', value)
+  }
+}

package/src/services/sync/models/ContentProgress.ts

@@ -0,0 +1,69 @@
+import BaseModel from './Base'
+import { SYNC_TABLES } from '../schema'
+
+export enum COLLECTION_TYPE {
+  SKILL_PACK = 'skill-pack',
+  LEARNING_PATH = 'learning-path',
+  PLAYLIST = 'playlist',
+}
+
+export enum STATE {
+  STARTED = 'started',
+  COMPLETED = 'completed'
+}
+
+export default class ContentProgress extends BaseModel<{
+  content_id: number
+  collection_type: COLLECTION_TYPE | null
+  collection_id: number | null
+  state: STATE
+  progress_percent: number
+  resume_time_seconds: number
+}> {
+  static table = SYNC_TABLES.CONTENT_PROGRESS
+
+  get content_id() {
+    return this._getRaw('content_id') as number
+  }
+  get content_brand() {
+    return this._getRaw('content_brand') as string
+  }
+  get state() {
+    return this._getRaw('state') as STATE
+  }
+  get progress_percent() {
+    return this._getRaw('progress_percent') as number
+  }
+  get collection_type() {
+    return (this._getRaw('collection_type') as COLLECTION_TYPE) || null
+  }
+  get collection_id() {
+    return (this._getRaw('collection_id') as number) || null
+  }
+  get resume_time_seconds() {
+    return this._getRaw('resume_time_seconds') as number
+  }
+
+  set content_id(value: number) {
+    this._setRaw('content_id', value)
+  }
+  set content_brand(value: string) {
+    this._setRaw('content_brand', value)
+  }
+  set state(value: STATE) {
+    this._setRaw('state', value)
+  }
+  set progress_percent(value: number) {
+    this._setRaw('progress_percent', value)
+  }
+  set collection_type(value: COLLECTION_TYPE | null) {
+    this._setRaw('collection_type', value)
+  }
+  set collection_id(value: number | null) {
+    this._setRaw('collection_id', value)
+  }
+  set resume_time_seconds(value: number) {
+    this._setRaw('resume_time_seconds', value)
+  }
+
+}