@cuboapp/crdt 1.0.5 → 1.0.6

package/src/server/index.ts

@@ -0,0 +1,449 @@
+import { WsServerSocket } from '@cuboapp/ws'
+
+import { CUBO_CRDT_EVENT } from '../constants'
+import { CuboCrdtAction } from '../types'
+import { checkRowIsSutable } from '../utils'
+
+import { CuboCrdtServerDocument } from './document'
+import {
+  CuboCrdtServerDocumentIncomingAction,
+  CuboCrdtServerDocumentOrigin,
+  CuboCrdtServerOptions,
+  CuboCrdtServerSubscribe,
+  CuboCrdtServerSubscribeDto,
+  CuboCrdtServerUnsubscribeDto
+} from './types'
+
+export * from './document'
+export * from './types'
+
+export class CuboCrdtServer<M, A = {}> {
+  constructor(private options: CuboCrdtServerOptions<M, A>) {}
+
+  private clients = new Map<string, WsServerSocket<{ auth?: A }>>()
+
+  private subscribes = new Map<string, CuboCrdtServerSubscribe>()
+  private subscribesByClient = new Map<string, Map<string, CuboCrdtServerSubscribe>>()
+  private subscribesByEntity = new Map<string, Map<string, CuboCrdtServerSubscribe>>()
+  private subscribesIniting = new Map<string, Promise<void>>()
+
+  private documents = new Map<string, CuboCrdtServerDocument>()
+  private documentsSubsrcibes = new Map<string, Set<string>>()
+
+  public async init() {
+    this.entities.forEach((e) => this.subscribesByEntity.set(e, new Map()))
+
+    this.ws.registerHandler(CUBO_CRDT_EVENT.SUBSCRIBE, async ({ client, message }) => {
+      const { subscribe_id: id, entity, filters } = message.data as CuboCrdtServerSubscribeDto
+
+      if (this.debug) {
+        console.log('[CRDT] subscribe', entity, filters)
+      }
+
+      const subscribe = {
+        id,
+        client_id: client.id,
+        entity,
+        filters
+      }
+
+      this.subscribes.set(id, subscribe)
+
+      // создаём мапу подписок по клиенту
+      this.subscribesByClient.get(client.id)?.set(id, subscribe)
+
+      // создаём мапу подписок по сущности
+      this.subscribesByEntity.get(entity)?.set(id, subscribe)
+
+      this.initSubscribe(client, subscribe)
+    })
+
+    this.ws.registerHandler(CUBO_CRDT_EVENT.UNSUBSCRIBE, async ({ message }) => {
+      const { subscribe_id } = message.data as CuboCrdtServerUnsubscribeDto
+
+      this.cleanSubscribe(subscribe_id)
+    })
+
+    this.ws.registerHandler(CUBO_CRDT_EVENT.EVENT, async ({ message }) => {
+      let data = message.data as any
+      if (!Array.isArray(data)) {
+        data = [data]
+      }
+
+      if (this.debug) {
+        console.log('[CRDT] incoming event', data)
+      }
+
+      data.forEach((row: any) => {
+        const update = new Uint8Array(row.data)
+
+        switch (row.action) {
+          case 'create':
+            this.onDocumentExternalCreate(update, row)
+            break
+          case 'update':
+            this.onDocumentExternalUpdate(update, row)
+            break
+          case 'delete':
+            this.onDocumentExternalDelete(update, row)
+            break
+        }
+      })
+    })
+
+    this.ws.registerHttpHandler('GET', '/stats', () => {
+      return {
+        clients: this.clients.size,
+        subscribes: {
+          size: this.subscribes.size,
+          byClient: this.subscribesByClient.size,
+          byEntity: Array.from(this.subscribesByEntity)
+            .map((m) => ({
+              entity: m[0],
+              size: m[1].size
+            }))
+            .filter((s) => s.size > 0)
+        },
+        documents: {
+          size: this.documents.size,
+          rows: Array.from(this.documents).map((d) => ({
+            id: d[0],
+            subs: Array.from(this.documentsSubsrcibes.get(d[0]) || [])
+          }))
+        }
+      }
+    })
+  }
+
+  public async destroy() {
+    this.ws.deleteHandler(CUBO_CRDT_EVENT.SUBSCRIBE)
+    this.ws.deleteHandler(CUBO_CRDT_EVENT.UNSUBSCRIBE)
+    this.ws.deleteHandler(CUBO_CRDT_EVENT.EVENT)
+
+    this.subscribes.forEach((sub) => this.ws.deleteHandler(sub.id))
+  }
+
+  public addClient(client: WsServerSocket<{ auth?: A }>) {
+    // добавляем клиента
+    this.clients.set(client.id, client)
+
+    // создаем мапу подписок по клиенту
+    this.subscribesByClient.set(client.id, new Map())
+  }
+
+  public removeClient(client: WsServerSocket) {
+    // удаляем все подписки по клиенту
+    this.subscribesByClient.get(client.id)?.forEach((sub) => this.cleanSubscribe(sub.id))
+
+    // удаляем мапу подписок по клиенту
+    this.subscribesByClient.delete(client.id)
+
+    // и в принципе клиента
+    this.clients.delete(client.id)
+  }
+
+  private cleanSubscribe(subscribe_id: string) {
+    const subscribe = this.subscribes.get(subscribe_id)
+    if (subscribe) {
+      if (this.debug) {
+        console.log('[CRDT] unsubscribe', subscribe.entity, subscribe.filters)
+      }
+
+      // удаляем подписку по сущности
+      this.subscribesByEntity.get(subscribe.entity)?.delete(subscribe.id)
+
+      // отписываем все документы
+      this.documentsSubsrcibes.forEach((subs, docName) => {
+        subs.delete(subscribe_id)
+
+        if (!subs?.size) {
+          if (this.debug) {
+            console.log('[CRDT] delete document', docName)
+          }
+
+          this.documents.get(docName)?.destroy()
+          this.documents.delete(docName)
+        }
+      })
+
+      // удаляем подписки
+      this.subscribes.delete(subscribe_id)
+      this.subscribesByClient.get(subscribe.client_id)?.delete(subscribe_id)
+      this.subscribesByEntity.get(subscribe.entity)?.delete(subscribe_id)
+    }
+  }
+
+  public getDocument(entity: string, entity_id: number) {
+    return this.documents.get(`${entity}:${entity_id}`)
+  }
+
+  public getSutableSubscribes(entity: string, row: any) {
+    return Array.from(this.subscribesByEntity.get(entity) || [])
+      .filter((m) => checkRowIsSutable(row, m[1].filters))
+      .map((m) => m[1])
+  }
+
+  public ensureDocument(entity: string, row: any, subscribes_ids?: string[]) {
+    const entity_id = row.id
+    const documentName = `${entity}:${entity_id}`
+
+    // проверяем документ
+    let document = this.documents.get(documentName)
+    if (!document) {
+      document = new CuboCrdtServerDocument({
+        name: documentName,
+        onCreate: (data: Uint8Array, origin?: CuboCrdtServerDocumentOrigin) => {
+          this.pushDocumentAction('create', document!, data, origin)
+        },
+        onUpdate: (data: Uint8Array, origin?: CuboCrdtServerDocumentOrigin) => {
+          this.pushDocumentAction('update', document!, data, origin)
+        },
+        onStore: (item: object, origin: CuboCrdtServerDocumentOrigin) => {
+          const subscribe = origin.subscribe_id && this.subscribes.get(origin.subscribe_id)
+          const client = subscribe && this.clients.get(subscribe.client_id)
+
+          if (client) {
+            return this.options?.storeRow?.(entity as any, entity_id, item as any, { client, origin })
+          }
+        }
+      })
+
+      this.documents.set(documentName, document)
+
+      this.documentsSubsrcibes.set(documentName, new Set())
+      if (subscribes_ids?.length) {
+        const subs = this.documentsSubsrcibes.get(documentName)
+        subscribes_ids?.forEach((id) => {
+          // добавляем подписку к документу
+          subs?.add(id)
+        })
+      }
+
+      // инициализируем документ
+      document.init(row)
+    } else {
+      // добавляем подписки к документу
+      const subs = this.documentsSubsrcibes.get(documentName)
+      subscribes_ids?.forEach((subscribe_id) => {
+        subs?.add(subscribe_id)
+
+        // пушим документ в сокеты (в текущем состоянии)
+        this.pushDocumentAction('update', document!, document!.stateAsUpdate, { subscribe_id, expose: 'subscribe' })
+      })
+    }
+
+    return this.documents.get(documentName)
+  }
+
+  public deleteDocument(entity: string, row: any) {
+    if (this.debug) {
+      console.log('[CRDT] delete document', entity, row.id)
+    }
+
+    const documentName = `${entity}:${row.id}`
+    const document = this.documents.get(documentName)
+
+    if (document) {
+      // отписываемся от всех подписок на документ
+      this.pushDocumentAction('delete', document, null, { expose: 'subscribe' })
+
+      // удаляем все подписки по документу
+      this.documentsSubsrcibes.delete(documentName)
+
+      // удаляем документ
+      this.documents.delete(documentName)
+
+      // дестроим документ
+      document.destroy()
+    }
+  }
+
+  private onDocumentExternalCreate(update: Uint8Array, action: CuboCrdtServerDocumentIncomingAction) {
+    console.warn('[CRDT] onDocumentExternalCreate not implemented')
+  }
+
+  private onDocumentExternalUpdate(update: Uint8Array, action: CuboCrdtServerDocumentIncomingAction) {
+    const document = this.getDocument(action.entity, action.entity_id)
+
+    // console.log('onDocumentExternalUpdate', action)
+
+    if (!document) {
+      console.warn('[CRDT] onDocumentExternalUpdate - document not found:', action.entity, action.entity_id)
+    } else {
+      document.applyUpdate(new Uint8Array(update), action.origin)
+    }
+  }
+
+  private onDocumentExternalDelete(update: Uint8Array, action: CuboCrdtServerDocumentIncomingAction) {
+    console.warn('[CRDT] onDocumentExternalDelete not implemented')
+  }
+
+  private documentUnsubscribe(docName: string, subscribe_id: string, push = true) {
+    const document = this.documents.get(docName)
+
+    if (document) {
+      // сначала отсылаем, затем удаляем подписки (иначе не будет куда слать)
+      if (push) {
+        this.pushDocumentAction('delete', document, null, { subscribe_id, expose: 'subscribe' })
+      }
+
+      const documentSubscribes = this.documentsSubsrcibes.get(docName)
+      documentSubscribes?.delete(subscribe_id)
+
+      // если нет больше подписок - удаляем документ
+      if (!documentSubscribes?.size) {
+        if (this.debug) {
+          console.log('[CRDT] delete document', name)
+        }
+
+        document?.destroy()
+        this.documents.delete(document.name)
+      }
+    }
+  }
+
+  private pushDocumentAction(
+    action: CuboCrdtAction,
+    document: CuboCrdtServerDocument,
+    data?: Uint8Array | null,
+    origin?: CuboCrdtServerDocumentOrigin
+  ) {
+    const row = document.getJson()
+
+    if (this.debug) {
+      console.log('[CRDT] pushDocumentAction ' + action, {
+        action,
+        row_id: row.id,
+        subscribes: Array.from(this.documentsSubsrcibes.get(document.name)!)
+      })
+    }
+
+    switch (action) {
+      case 'delete': {
+        this.documentsSubsrcibes.get(document.name)?.forEach((subscribe_id) => {
+          const subscribe = this.subscribes.get(subscribe_id)
+          if (subscribe) {
+            const client = this.clients.get(subscribe.client_id)
+
+            if (client) {
+              if (this.debug) {
+                console.log('[CRDT] send action ' + action, subscribe.entity, row.id, subscribe_id)
+              }
+
+              // todo: send сделать через очередь (с группировкой по клиенту)
+              client.send(
+                JSON.stringify({
+                  method: CUBO_CRDT_EVENT.EVENT,
+                  data: {
+                    action,
+                    entity: subscribe.entity,
+                    entity_id: row.id,
+                    subscribe_id: subscribe.id
+                  }
+                })
+              )
+            }
+          }
+        })
+        break
+      }
+      case 'update':
+      case 'create': {
+        this.documentsSubsrcibes.get(document.name)?.forEach((subscribe_id) => {
+          const subscribe = this.subscribes.get(subscribe_id)
+
+          if (subscribe) {
+            const client = this.clients.get(subscribe.client_id)
+
+            if (client) {
+              const noStrategyResolved = !origin?.expose || origin?.expose === 'all'
+              const strategyOtherResolved = (origin?.expose === 'other' && origin.subscribe_id !== subscribe_id) || false
+              const strategyClientResolved = origin?.expose === 'client' && subscribe.client_id === origin.client_id
+              const strategySubscribeResolved = origin?.expose === 'subscribe' && subscribe.id === origin.subscribe_id
+
+              if (noStrategyResolved || strategyOtherResolved || strategyClientResolved || strategySubscribeResolved) {
+                const isSutable = checkRowIsSutable(document.getJson(), subscribe.filters || {})
+
+                // если документ перестал быть доступным в рамках указанных фильтров - удаляем его
+                if (!isSutable) {
+                  this.documentUnsubscribe(document.name, subscribe.id)
+                } else {
+                  if (this.debug) {
+                    console.log('[CRDT] send action ' + action, subscribe.entity, row.id, origin, {
+                      noStrategyResolved,
+                      strategyOtherResolved,
+                      strategyClientResolved,
+                      strategySubscribeResolved
+                    })
+                  }
+
+                  // todo: send сделать через очередь (с группировкой по клиенту)
+                  client.send(
+                    JSON.stringify({
+                      method: CUBO_CRDT_EVENT.EVENT,
+                      data: {
+                        action,
+                        entity: subscribe.entity,
+                        entity_id: row.id,
+                        data: Array.from(data as any),
+                        subscribe_id: subscribe.id
+                      }
+                    })
+                  )
+                }
+              }
+            }
+          }
+        })
+      }
+    }
+  }
+
+  private async initSubscribe(client: WsServerSocket, subscribe: CuboCrdtServerSubscribe) {
+    if (this.subscribesIniting.has(subscribe.id)) {
+      return this.subscribesIniting.get(subscribe.id)
+    }
+
+    this.subscribesIniting.set(
+      subscribe.id,
+      new Promise<void>(async (resolve, reject) => {
+        try {
+          if (this.debug) {
+            console.log('[CRDT] initSubscribe ' + client.id, subscribe)
+          }
+
+          // получаем список всех документов
+          const rows = await this.options.fetchRows?.(client, subscribe)
+
+          for (const row of rows || []) {
+            this.ensureDocument(subscribe.entity, row, [subscribe.id])
+          }
+
+          resolve()
+        } catch (e) {
+          if (this.debug) {
+            console.error(`[CRDT] initSubscribe`, e)
+          }
+
+          reject(e)
+        } finally {
+          this.subscribesIniting.delete(subscribe.id)
+        }
+      })
+    )
+
+    return this.subscribesIniting.get(subscribe.id)
+  }
+
+  private get debug() {
+    return this.options.debug
+  }
+
+  private get ws() {
+    return this.options.ws
+  }
+
+  private get entities() {
+    return this.options.entities
+  }
+}

package/src/server/old/document/index.ts

@@ -0,0 +1,107 @@
+import { applyUpdate, Doc, encodeStateAsUpdate } from 'yjs'
+
+export class CuboCrdtDocument {
+  private ydoc: Doc
+  private subs = new Map<string, (data: Uint8Array<ArrayBufferLike>, origin?: any) => void>()
+
+  constructor(
+    public opts: {
+      name: string
+      initialState?: any
+      autoRemove?: boolean
+      onDelete?: () => void
+      yjsOptions?: {
+        guid?: string
+        collectionid?: string
+        gc?: boolean
+        gcFilter?: () => true
+        meta?: any
+      }
+    }
+  ) {}
+
+  get stateAsUpdate() {
+    return encodeStateAsUpdate(this.ydoc)
+  }
+
+  async init() {
+    this.ydoc = new Doc({
+      ...(this.opts.yjsOptions || {}),
+      autoLoad: false
+    })
+
+    const state = this.opts?.initialState ?? {}
+
+    return new Promise<Uint8Array<ArrayBufferLike>>((resolve) => {
+      this.ydoc.once('update', () => {
+        this.ydoc.on('update', (data, origin) => {
+          this.subs.forEach((sub) => sub(data, origin))
+        })
+
+        resolve(this.stateAsUpdate)
+      })
+
+      const map = this.getMap()
+
+      this.ydoc.transact(() => {
+        if (Object.keys(state).length) {
+          Object.entries(state).forEach(([key, value]) => {
+            map.set(key, value)
+          })
+        }
+
+        this.getState().setAttribute('inited', 'Y')
+      })
+    })
+  }
+
+  get inited() {
+    return this.getState().getAttribute('inited') === 'Y'
+  }
+
+  get id() {
+    return this.ydoc.guid
+  }
+
+  applyUpdate(update: Uint8Array, origin?: any) {
+    return applyUpdate(this.ydoc, update, origin)
+  }
+
+  subscribe(id: string, cb: (data: Uint8Array<ArrayBufferLike>, origin?: any) => void) {
+    this.subs.set(id, cb)
+  }
+
+  unsubscribe(id: string) {
+    this.subs.delete(id)
+
+    if (!this.subs.size && this.opts?.autoRemove !== false) {
+      this.opts?.onDelete?.()
+    }
+  }
+
+  getState() {
+    return this.ydoc.getText('state')
+  }
+
+  getMap() {
+    return this.ydoc.getMap()
+  }
+
+  getXmlFragment() {
+    return this.ydoc.getXmlFragment()
+  }
+
+  async writeMap(dto: object, origin?: any) {
+    return new Promise<void>(async (resolve) => {
+      const map = this.getMap()
+
+      this.ydoc.transact(() => {
+        Object.entries(dto).forEach(([key, value]) => {
+          map.set(key, value)
+        })
+
+        resolve()
+      }, origin)
+    })
+  }
+}

package/src/server/old/index.ts

@@ -0,0 +1,49 @@
+import { CuboCrdtDocument } from './document'
+
+export * from './document'
+
+export class CuboCrdtServer<T> {
+  constructor(private options: { debug?: boolean }) {}
+
+  public documents = new Map<string, CuboCrdtDocument>()
+  private loading = new Map<string, Promise<CuboCrdtDocument>>()
+
+  public async getDocument(name: string, opts?: { autoCreate?: boolean; initialState?: any }) {
+    // существующий документ
+    if (!this.documents.has(name)) {
+      if (!this.loading.has(name) && opts?.autoCreate !== false) {
+        if (this.options.debug) {
+          console.log('CRDT create document load start', name)
+        }
+
+        this.loading.set(
+          name,
+          new Promise(async (resolve) => {
+            const document = new CuboCrdtDocument({
+              name,
+              initialState: opts?.initialState,
+              autoRemove: true,
+              onDelete: () => {
+                this.documents.delete(name)
+              }
+            })
+
+            await document.init()
+
+            resolve(document)
+          })
+        )
+      }
+
+      const document = await this.loading.get(name)
+
+      this.loading.delete(name)
+
+      if (document) {
+        this.documents.set(name, document)
+      }
+    }
+
+    return this.documents.get(name)
+  }
+}

package/src/server/types/document.ts

@@ -0,0 +1,54 @@
+import { CuboCrdtAction, CuboCrdtExposeStrategy } from '../../types'
+
+export type CuboCrdtServerDocumentOptions = {
+  name: string
+  onStore?: (body: object, origin: CuboCrdtServerDocumentOrigin) => void
+  onCreate?: (data: Uint8Array, origin?: CuboCrdtServerDocumentOrigin) => void
+  onUpdate?: (data: Uint8Array, origin?: CuboCrdtServerDocumentOrigin) => void
+  // onInit?: (document: CuboCrdtServerDocument) => void
+
+  yjsOptions?: {
+    guid?: string
+    collectionid?: string
+    gc?: boolean
+    gcFilter?: () => true
+    meta?: any
+  }
+}
+
+export type CuboCrdtServerDocumentIncomingAction = {
+  // действие
+  action: CuboCrdtAction
+
+  // сущность
+  entity: string
+
+  // ид сущности
+  entity_id: number
+
+  // контент апдейта
+  data: number[]
+
+  // ориджин
+  origin: CuboCrdtServerDocumentOrigin
+}
+
+export type CuboCrdtServerDocumentOrigin = {
+  // сохранять ли на бэке в дебаунсе
+  store?: boolean
+
+  // реагировать ли в onUpdate (на бэке)
+  react?: boolean
+
+  // обновлённые ключи
+  keys?: string[]
+
+  // куда раскатывать обновления - всем или всем кроме себя
+  expose?: CuboCrdtExposeStrategy
+
+  // id подписки-исходника
+  subscribe_id?: string
+
+  // id клиента
+  client_id?: string
+}

package/src/server/types/index.ts

@@ -0,0 +1,22 @@
+import { WsServer, WsServerSocket } from '@cuboapp/ws'
+import { CuboCrdtServerDocumentOrigin } from './document'
+import { CuboCrdtServerSubscribe } from './subscribe'
+
+export * from './document'
+export * from './subscribe'
+
+export type CuboCrdtServerOptions<M, A> = {
+  ws: WsServer
+  entities: Extract<keyof M, string>[]
+  debug?: boolean
+  fetchRows?: (client: WsServerSocket, subscribe: CuboCrdtServerSubscribe) => Promise<M[Extract<keyof M, string>][]>
+  storeRow?: <K extends Extract<keyof M, string>>(
+    entity: K,
+    entity_id: number,
+    row: M[K],
+    opts: {
+      client: WsServerSocket<{ auth?: A }>
+      origin: CuboCrdtServerDocumentOrigin
+    }
+  ) => Promise<void> | void
+}

package/src/server/types/subscribe.ts

@@ -0,0 +1,15 @@
+export type CuboCrdtServerUnsubscribeDto = {
+  subscribe_id: string
+}
+export type CuboCrdtServerSubscribeDto = {
+  subscribe_id: string
+  entity: string
+  filters: {
+    [K in 'id' | string]: any
+  }
+}
+
+export type CuboCrdtServerSubscribe = {
+  id: string
+  client_id: string
+} & Pick<CuboCrdtServerSubscribeDto, 'entity' | 'filters'>