@cuboapp/crdt 1.0.5 → 1.0.7

package/src/server/index.ts

@@ -0,0 +1,582 @@
+import { cloneDeep } from '@cuboapp/utils'
+import { WsServerSocket } from '@cuboapp/ws'
+
+import { CUBO_CRDT_EVENT } from '../constants'
+import { CuboCrdtAction } from '../types'
+import { checkRowIsSutable } from '../utils'
+
+import { CuboCrdtClientDocOrigin } from '../client'
+import { CuboCrdtServerDocument } from './document'
+import {
+  CuboCrdtServerDocumentIncomingAction,
+  CuboCrdtServerDocumentOrigin,
+  CuboCrdtServerOptions,
+  CuboCrdtServerSubscribe,
+  CuboCrdtServerSubscribeDto,
+  CuboCrdtServerUnsubscribeDto,
+  CuboCrdtSocketClient
+} from './types'
+
+export * from './document'
+export * from './types'
+
+export class CuboCrdtServer<M, A = {}, E extends Extract<keyof M, string> = Extract<keyof M, string>> {
+  constructor(private options: CuboCrdtServerOptions<M, A, E>) {}
+
+  private clients = new Map<string, CuboCrdtSocketClient<A>>()
+  private queue = new Map<string, Set<any>>()
+  private sendTimeouts = new Map<string, any>()
+
+  private subscribes = new Map<string, CuboCrdtServerSubscribe>()
+  private subscribesByClient = new Map<string, Set<string>>()
+  private subscribesByEntity = new Map<E, Set<string>>()
+  private subscribesIniting = new Map<string, Promise<void>>()
+  private subscribesUpgrading = new Map<string, Promise<void>>()
+
+  private documents = new Map<string, CuboCrdtServerDocument>()
+  private subscribesByDocument = new Map<string, Set<string>>()
+
+  public async init() {
+    this.entities.forEach((e) => this.subscribesByEntity.set(e, new Set()))
+
+    this.ws.registerHandler(CUBO_CRDT_EVENT.SUBSCRIBE, async ({ client, message }) => {
+      const { subscribe_id: id, entity, filters } = message.data as CuboCrdtServerSubscribeDto
+
+      if (this.debug) {
+        console.log('[CRDT] subscribe', entity, filters)
+      }
+
+      const subscribe = {
+        id,
+        client_id: client.id,
+        entity,
+        filters
+      }
+
+      this.subscribes.set(id, subscribe)
+
+      // создаём мапу подписок по клиенту
+      this.subscribesByClient.get(client.id)?.add(id)
+
+      // создаём мапу подписок по сущности
+      this.subscribesByEntity.get(entity as E)?.add(id)
+
+      // инициализируем подписку
+      this.initSubscribe(client, subscribe)
+    })
+
+    this.ws.registerHandler(CUBO_CRDT_EVENT.UPGRADE, async ({ client, message }) => {
+      const { subscribe_id: id, entity, filters } = message.data as CuboCrdtServerSubscribeDto
+
+      if (!this.debug) {
+        console.log('[CRDT] upgrade', entity, filters)
+      }
+
+      const subscribe = this.subscribes.get(id)
+      if (!subscribe) {
+        return
+      }
+
+      // обновляем подписку
+      this.upgradeSubscribe(client, subscribe, { filters })
+    })
+
+    this.ws.registerHandler(CUBO_CRDT_EVENT.UNSUBSCRIBE, async ({ message }) => {
+      const { subscribe_id } = message.data as CuboCrdtServerUnsubscribeDto
+
+      this.cleanSubscribe(subscribe_id)
+    })
+
+    this.ws.registerHandler(CUBO_CRDT_EVENT.EVENT, async ({ message }) => {
+      let data = message.data as any
+      if (!Array.isArray(data)) {
+        data = [data]
+      }
+
+      if (this.debug) {
+        console.log('[CRDT] incoming event', data)
+      }
+
+      data.forEach((row: any) => {
+        const update = new Uint8Array(row.data)
+
+        switch (row.action) {
+          case 'create':
+            this.onDocumentExternalCreate(update, row)
+            break
+          case 'update':
+            this.onDocumentExternalUpdate(update, row)
+            break
+          case 'delete':
+            this.onDocumentExternalDelete(update, row)
+            break
+        }
+      })
+    })
+
+    this.ws.registerHttpHandler('GET', '/stats', () => {
+      return {
+        clients: this.clients.size,
+        subscribes: {
+          size: this.subscribes.size,
+          byClient: this.subscribesByClient.size,
+          byEntity: Array.from(this.subscribesByEntity)
+            .map((m) => ({
+              entity: m[0],
+              size: m[1].size
+            }))
+            .filter((s) => s.size > 0)
+        },
+        documents: {
+          size: this.documents.size,
+          rows: Array.from(this.documents).map((d) => ({
+            id: d[0],
+            subs: Array.from(this.subscribesByDocument.get(d[0]) || [])
+          }))
+        }
+      }
+    })
+  }
+
+  public async destroy() {
+    this.ws.deleteHandler(CUBO_CRDT_EVENT.SUBSCRIBE)
+    this.ws.deleteHandler(CUBO_CRDT_EVENT.UNSUBSCRIBE)
+    this.ws.deleteHandler(CUBO_CRDT_EVENT.EVENT)
+
+    this.subscribes.forEach((sub) => this.ws.deleteHandler(sub.id))
+  }
+
+  public addClient(client: WsServerSocket<{ auth?: A }>) {
+    // добавляем клиента
+    this.clients.set(client.id, client)
+
+    // создаем мапу подписок по клиенту
+    this.subscribesByClient.set(client.id, new Set())
+
+    // создаём очередь под клиента
+    this.queue.set(client.id, new Set())
+  }
+
+  public removeClient(client: WsServerSocket) {
+    // удаляем все подписки по клиенту
+    this.subscribesByClient.get(client.id)?.forEach((subscribe_id) => this.cleanSubscribe(subscribe_id))
+
+    // удаляем мапу подписок по клиенту
+    this.subscribesByClient.delete(client.id)
+
+    // удаляем очередь под клиента
+    this.queue.delete(client.id)
+
+    // и в принципе клиента
+    this.clients.delete(client.id)
+  }
+
+  private sendQueueToClient(client: CuboCrdtSocketClient<A>) {
+    const queue = this.queue.get(client.id)
+
+    // const data = Array.from(queue)
+    const data = [...new Map((Array.from(queue) || []).map((item) => [JSON.stringify(item), item])).values()]
+
+    queue.clear()
+
+    client.send(
+      JSON.stringify({
+        method: CUBO_CRDT_EVENT.EVENT,
+        data
+      })
+    )
+
+    queue.clear()
+  }
+
+  public sendToClient(client_id: string, data: any) {
+    const client = this.clients.get(client_id)
+    if (!client) {
+      return
+    }
+
+    // сбрасываем таймаут если он есть
+    clearTimeout(this.sendTimeouts.get(client_id))
+
+    // добавляем в очередь
+    this.queue.get(client_id).add(data)
+
+    // если очередь >10 то сразу отправляем
+    if (this.queue.size > 10) {
+      this.sendQueueToClient(client)
+    }
+    // если нет - ставим таймаут
+    else {
+      this.sendTimeouts.set(
+        client_id,
+        setTimeout(() => this.sendQueueToClient(client), 100)
+      )
+    }
+  }
+
+  // если у документа нет подписок - сносим его
+  private checkDocumentNeedRemove(name: string) {
+    if (!this.subscribesByDocument.get(name)?.size) {
+      if (this.debug) {
+        console.log('[CRDT] delete document', name)
+      }
+
+      this.documents.get(name)?.destroy()
+      this.documents.delete(name)
+    }
+  }
+
+  // очистка подписки (при отписке клиента - через emit-метод или options-хук)
+  private cleanSubscribe(subscribe_id: string) {
+    const subscribe = this.subscribes.get(subscribe_id)
+    if (subscribe) {
+      if (this.debug) {
+        console.log('[CRDT] unsubscribe', subscribe.entity, subscribe.filters)
+      }
+
+      // удаляем подписку по сущности
+      this.subscribesByEntity.get(subscribe.entity as E)?.delete(subscribe.id)
+
+      // отписываем все документы от подписки
+      this.subscribesByDocument.forEach((subscribes, docName) => {
+        subscribes.delete(subscribe_id)
+
+        this.checkDocumentNeedRemove(docName)
+      })
+
+      // удаляем подписки
+      this.subscribes.delete(subscribe_id)
+      this.subscribesByClient.get(subscribe.client_id)?.delete(subscribe_id)
+      this.subscribesByEntity.get(subscribe.entity as E)?.delete(subscribe_id)
+    }
+  }
+
+  public getDocument(entity: string, entity_id: number) {
+    return this.documents.get(`${entity}:${entity_id}`)
+  }
+
+  public getSutableSubscribes(entity: string, row: any) {
+    return Array.from(this.subscribesByEntity.get(entity as E) || [])
+      .filter((m) => {
+        const subscribe = this.subscribes.get(m[1])
+
+        let state = subscribe && checkRowIsSutable(row, subscribe)
+        if (this.options.checkRowIsSutable) {
+          state = this.options.checkRowIsSutable(state, { entity: entity as any, subscribe, row })
+        }
+
+        return state
+      })
+      .map((m) => m[1])
+  }
+
+  public ensureDocument(entity: E, row: any) {
+    // todo: врапнуть всё это в Promise
+    const entity_id = row.id
+    const documentName = `${entity}:${entity_id}`
+
+    // проверяем документ
+    let document = this.documents.get(documentName)
+    if (!document) {
+      document = new CuboCrdtServerDocument({
+        name: documentName,
+        initialState: row,
+        onUpdate: (data: Uint8Array, origin?: CuboCrdtServerDocumentOrigin) => {
+          // пушим обновление документа
+          // console.log('push document update', document.name)
+
+          this.pushDocumentAction('update', entity, document!, data, origin)
+        },
+        onStore: (item: object, origin: CuboCrdtServerDocumentOrigin) => {
+          const subscribe = origin.subscribe_id && this.subscribes.get(origin.subscribe_id)
+          const client = subscribe && this.clients.get(subscribe.client_id)
+
+          if (client) {
+            return this.options?.storeRow?.(entity as any, entity_id, item as any, { client, origin })
+          }
+        }
+      })
+
+      // создаём документ
+      this.documents.set(documentName, document)
+
+      // создаём пул подписок по документу
+      this.subscribesByDocument.set(documentName, new Set())
+
+      // пушим создание документа
+      this.pushDocumentAction('create', entity, document, document.stateAsUpdate)
+
+      // console.log('create document', documentName)
+    } else {
+      // console.log('update document', documentName)
+
+      // пушим обновление документа
+      this.pushDocumentAction('upsert', entity, document, document.stateAsUpdate)
+    }
+
+    return document
+  }
+
+  public deleteDocument(entity: E, row: any, origin?: CuboCrdtClientDocOrigin) {
+    if (this.debug) {
+      console.log('[CRDT] delete document', entity, row.id, origin)
+    }
+
+    const documentName = `${entity}:${row.id}`
+    const document = this.documents.get(documentName)
+
+    if (document) {
+      // сначала пушим удаление документа
+      this.pushDocumentAction('delete', entity, document, null, origin)
+
+      // удаляем документ
+      this.documents.delete(documentName)
+
+      // дестроим документ
+      document.destroy()
+    }
+  }
+
+  private onDocumentExternalCreate(update: Uint8Array, action: CuboCrdtServerDocumentIncomingAction) {
+    console.warn('[CRDT] onDocumentExternalCreate not implemented')
+  }
+
+  private onDocumentExternalUpdate(update: Uint8Array, action: CuboCrdtServerDocumentIncomingAction) {
+    const document = this.getDocument(action.entity, action.entity_id)
+
+    // console.log('onDocumentExternalUpdate', action)
+
+    if (!document) {
+      console.warn('[CRDT] onDocumentExternalUpdate - document not found:', action.entity, action.entity_id)
+    } else {
+      document.applyUpdate(new Uint8Array(update), action.origin)
+    }
+  }
+
+  private onDocumentExternalDelete(update: Uint8Array, action: CuboCrdtServerDocumentIncomingAction) {
+    console.warn('[CRDT] onDocumentExternalDelete not implemented')
+  }
+
+  private checkSubscribeIsSutable(origin: CuboCrdtServerDocumentOrigin, subscribe?: CuboCrdtServerSubscribe) {
+    if (!subscribe) {
+      return false
+    }
+
+    const noStrategyResolved = !origin?.expose || origin?.expose === 'all'
+    const strategyOtherResolved = (origin?.expose === 'other' && origin.subscribe_id !== subscribe?.id) || false
+    const strategyClientResolved = origin?.expose === 'client' && subscribe?.client_id === origin.client_id
+    const strategySubscribeResolved = origin?.expose === 'subscribe' && subscribe?.id === origin.subscribe_id
+
+    return noStrategyResolved || strategyOtherResolved || strategyClientResolved || strategySubscribeResolved
+  }
+
+  private checkRowIsSutable(row: any, subscribe: CuboCrdtServerSubscribe, entity: E) {
+    const baseState = checkRowIsSutable(row, subscribe.filters || {})
+
+    if (!this.options.checkRowIsSutable) {
+      return baseState
+    }
+
+    return this.options.checkRowIsSutable(baseState, { entity, subscribe, row: row as any })
+  }
+
+  private pushDocumentAction(
+    action: CuboCrdtAction,
+    entity: E,
+    document: CuboCrdtServerDocument,
+    data?: Uint8Array | null,
+    origin?: CuboCrdtServerDocumentOrigin
+  ) {
+    const row = document.getJson()
+
+    // берём все подписки по сущности
+    const sutableSubscribes = Array.from(this.subscribesByEntity.get(entity))
+      .map((subscribe_id) => {
+        const subscribe = this.subscribes.get(subscribe_id)
+
+        const subscribeSutable = this.checkSubscribeIsSutable(origin, subscribe)
+        const documentExistsInSubscribe = this.subscribesByDocument.get(document.name)?.has(subscribe.id)
+        const rowSutable = this.checkRowIsSutable(row, subscribe, entity)
+
+        const sutable = subscribeSutable && (rowSutable || documentExistsInSubscribe)
+
+        if (sutable) {
+          if (this.debug && !rowSutable) {
+            console.warn('row is not sutable for subscribe', {
+              entity,
+              row,
+              subscribe,
+              subscribeSutable,
+              documentExistsInSubscribe,
+              rowSutable
+            })
+          }
+
+          let sutableAction = action
+          let sutableData: any = data ? Array.from(data as any).slice(0) : undefined
+
+          if (!rowSutable && action === 'update') {
+            if (this.debug) {
+              console.log('switch action', action + ' -> delete', { name: document.name, subscribe: subscribe.id })
+            }
+
+            sutableAction = 'delete'
+            sutableData = null
+          } else if (action === 'update' && !documentExistsInSubscribe) {
+            if (this.debug) {
+              console.log('switch action', 'update -> create', { name: document.name, subscribe: subscribe.id })
+            }
+
+            sutableAction = 'create'
+            sutableData = Array.from(document.stateAsUpdate)
+          } else if (action === 'upsert' && !documentExistsInSubscribe) {
+            if (this.debug) {
+              console.log('switch action', 'upsert -> create', { name: document.name, subscribe: subscribe.id })
+            }
+
+            sutableAction = 'create'
+            sutableData = Array.from(document.stateAsUpdate)
+          } else if (action === 'delete' && !documentExistsInSubscribe) {
+            if (this.debug) {
+              console.warn('document to delete is not exist', { name: document.name, subscribe: subscribe.id })
+            }
+          } else if (action === 'create' && documentExistsInSubscribe) {
+            if (this.debug) {
+              console.warn('document to create is already exists in subscribe', { name: document.name, subscribe: subscribe.id })
+            }
+            sutableAction = 'update'
+          }
+
+          return {
+            subscribe,
+            action: sutableAction,
+            data: sutableData
+          }
+        }
+
+        return null
+      })
+      .filter((i) => !!i) as { subscribe: CuboCrdtServerSubscribe; action: CuboCrdtAction; data: undefined | number[] }[]
+
+    if (sutableSubscribes.length) {
+      const documentSubscribes = this.subscribesByDocument.get(document.name)
+
+      sutableSubscribes.forEach(({ subscribe, action, data }) => {
+        this.sendToClient(subscribe.client_id, {
+          action,
+          entity: subscribe.entity,
+          entity_id: row.id,
+          data,
+          subscribe_id: subscribe.id
+        })
+
+        switch (action) {
+          case 'create':
+          case 'upsert':
+          case 'update':
+            documentSubscribes.add(subscribe.id)
+            break
+          case 'delete':
+            documentSubscribes.delete(subscribe.id)
+            this.checkDocumentNeedRemove(document.name)
+            break
+        }
+      })
+    }
+  }
+
+  private async pushSubscribeDocuments(client: WsServerSocket, subscribe: CuboCrdtServerSubscribe) {
+    // получаем список всех документов
+    const rows = await this.options.fetchRows?.(client, subscribe)
+
+    // пушим их в сокеты
+    for (const row of rows || []) {
+      // this.ensureDocument(subscribe.entity as E, row, { subscribes_ids: [subscribe.id] })
+      this.ensureDocument(subscribe.entity as E, row)
+    }
+  }
+
+  // апгрейд подписки (фильтры поменялись например)
+  private async upgradeSubscribe(
+    client: WsServerSocket,
+    subscribe: CuboCrdtServerSubscribe,
+    upgrade: Partial<Pick<CuboCrdtServerSubscribe, 'filters'>>
+  ) {
+    if (this.subscribesUpgrading.has(subscribe.id)) {
+      return this.subscribesUpgrading.get(subscribe.id)
+    }
+
+    this.subscribesUpgrading.set(
+      subscribe.id,
+      new Promise<void>(async (resolve, reject) => {
+        try {
+          if (this.debug) {
+            console.log('[CRDT] upgrade subscribe ' + client.id, subscribe)
+          }
+
+          if (upgrade.filters) {
+            subscribe.filters = cloneDeep(upgrade.filters || {})
+          }
+
+          await this.pushSubscribeDocuments(client, subscribe)
+
+          resolve()
+        } catch (e) {
+          if (this.debug) {
+            console.error('[CRDT] upgrade subscribe', e)
+          }
+
+          reject(e)
+        } finally {
+          this.subscribesUpgrading.delete(subscribe.id)
+        }
+      })
+    )
+
+    return this.subscribesUpgrading.get(subscribe.id)
+  }
+
+  // инциализация подписки
+  private async initSubscribe(client: WsServerSocket, subscribe: CuboCrdtServerSubscribe) {
+    if (this.subscribesIniting.has(subscribe.id)) {
+      return this.subscribesIniting.get(subscribe.id)
+    }
+
+    this.subscribesIniting.set(
+      subscribe.id,
+      new Promise<void>(async (resolve, reject) => {
+        try {
+          if (this.debug) {
+            console.log('[CRDT] init subscribe ' + client.id, subscribe)
+          }
+
+          await this.pushSubscribeDocuments(client, subscribe)
+
+          resolve()
+        } catch (e) {
+          if (this.debug) {
+            console.error('[CRDT] init subscribe', e)
+          }
+
+          reject(e)
+        } finally {
+          this.subscribesIniting.delete(subscribe.id)
+        }
+      })
+    )
+
+    return this.subscribesIniting.get(subscribe.id)
+  }
+
+  private get debug() {
+    return this.options.debug
+  }
+
+  private get ws() {
+    return this.options.ws
+  }
+
+  private get entities() {
+    return this.options.entities
+  }
+}

package/src/server/types/document.ts

@@ -0,0 +1,56 @@
+import { CuboCrdtAction, CuboCrdtExposeStrategy } from '../../types'
+
+export type CuboCrdtServerDocumentOptions = {
+  name: string
+  onStore?: (body: object, origin: CuboCrdtServerDocumentOrigin) => void
+  // onCreate?: (data: Uint8Array, origin?: CuboCrdtServerDocumentOrigin) => void
+  onUpdate?: (data: Uint8Array, origin?: CuboCrdtServerDocumentOrigin) => void
+  // onInit?: (document: CuboCrdtServerDocument) => void
+
+  initialState?: object
+
+  yjsOptions?: {
+    guid?: string
+    collectionid?: string
+    gc?: boolean
+    gcFilter?: () => true
+    meta?: any
+  }
+}
+
+export type CuboCrdtServerDocumentIncomingAction = {
+  // действие
+  action: CuboCrdtAction
+
+  // сущность
+  entity: string
+
+  // ид сущности
+  entity_id: number
+
+  // контент апдейта
+  data: number[]
+
+  // ориджин
+  origin: CuboCrdtServerDocumentOrigin
+}
+
+export type CuboCrdtServerDocumentOrigin = {
+  // сохранять ли на бэке в дебаунсе
+  store?: boolean
+
+  // реагировать ли в onUpdate (на бэке)
+  react?: boolean
+
+  // обновлённые ключи
+  keys?: string[]
+
+  // куда раскатывать обновления - всем или всем кроме себя
+  expose?: CuboCrdtExposeStrategy
+
+  // id подписки-исходника
+  subscribe_id?: string
+
+  // id клиента
+  client_id?: string
+}

package/src/server/types/index.ts

@@ -0,0 +1,25 @@
+import { WsServer, WsServerSocket } from '@cuboapp/ws'
+import { CuboCrdtServerDocumentOrigin } from './document'
+import { CuboCrdtServerSubscribe } from './subscribe'
+
+export * from './document'
+export * from './subscribe'
+
+export type CuboCrdtServerOptions<M, A, E extends Extract<keyof M, string>> = {
+  ws: WsServer
+  entities: E[]
+  debug?: boolean
+  fetchRows?: (client: WsServerSocket, subscribe: CuboCrdtServerSubscribe) => Promise<M[E][]>
+  checkRowIsSutable?: (baseState: boolean, ctx: { entity: E; subscribe: CuboCrdtServerSubscribe; row: any }) => boolean
+  storeRow?: <K extends E>(
+    entity: K,
+    entity_id: number,
+    row: M[K],
+    opts: {
+      client: CuboCrdtSocketClient<A>
+      origin: CuboCrdtServerDocumentOrigin
+    }
+  ) => Promise<void> | void
+}
+
+export type CuboCrdtSocketClient<A> = WsServerSocket<{ auth?: A }>

package/src/server/types/subscribe.ts

@@ -0,0 +1,15 @@
+export type CuboCrdtServerUnsubscribeDto = {
+  subscribe_id: string
+}
+export type CuboCrdtServerSubscribeDto = {
+  subscribe_id: string
+  entity: string
+  filters: {
+    [K in 'id' | string]: any
+  }
+}
+
+export type CuboCrdtServerSubscribe = {
+  id: string
+  client_id: string
+} & Pick<CuboCrdtServerSubscribeDto, 'entity' | 'filters'>

package/src/types/index.ts

@@ -1,11 +1,5 @@
-import { WsServer, WsServerRequest } from '@cuboapp/ws'
+export type CuboCrdtKey<K, M> = K extends Extract<keyof M, string> ? M[K] : any
 
-export type CuboCrdtOptions<A> = {
-  host?: string
-  port?: number
-  debug?: boolean
+export type CuboCrdtAction = 'create' | 'upsert' | 'update' | 'delete'
 
-  authHandler?: (ctx: { server: WsServer<{ auth: A }>; request: WsServerRequest<{ auth: A }> }) => Promise<A> | A
-
-  beforeLoadDocument?: (name: string) => Promise<any> | any
-}
+export type CuboCrdtExposeStrategy = 'all' | 'other' | 'subscribe' | 'client'