@cuboapp/api-backend 1.0.9 → 1.0.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cuboapp/api-backend",
-  "version": "1.0.9",
+  "version": "1.0.12",
   "description": "Backend Api for CuboApp",
   "main": "src/index.ts",
   "repository": "git@github.com:cuboapp/api-backend.git",
@@ -8,15 +8,17 @@
   "license": "MIT",
   "type": "module",
   "dependencies": {
-    "@cuboapp/constants": "2.0.4",
-    "@cuboapp/database": "1.0.1",
-    "@cuboapp/types": "2.0.5",
+    "@cuboapp/constants": "^2.0.5",
+    "@cuboapp/database": "^1.0.1",
+    "@cuboapp/types": "^2.0.6",
     "@cuboapp/utils": "1.0.4",
+    "@hocuspocus/extension-database": "^3.2.3",
+    "@hocuspocus/server": "^3.2.3",
     "pg": "^8.16.3",
-    "sequelize": "^6.37.7"
+    "yjs": "^13.6.27"
   },
   "devDependencies": {
-    "@types/node": "^22.0.0",
-    "typescript": "^5.8.2"
+    "@types/node": "^24.10.1",
+    "typescript": "^5.9.3"
   }
 }

package/src/constants/index.ts

@@ -18,7 +18,8 @@ export const CUBO_CRUD_QUERY_CONDITION = {
   GT: 'gt',
   LT: 'lt',
   GTE: 'gte',
-  LTE: 'lte'
+  LTE: 'lte',
+  BTW: 'btw'
 }
 
 export const CUBO_CRUD_DEFAULT_FIELDS: Pick<CuboEntityField, 'type' | 'alias' | 'extra'>[] = [

package/src/crdt/index.ts

@@ -0,0 +1,418 @@
+import { QueryTypes } from '@cuboapp/database'
+import { CuboApiEntitiesMap, CuboEntity } from '@cuboapp/types'
+import { Database } from '@hocuspocus/extension-database'
+import {
+  beforeSyncPayload,
+  Document,
+  Hocuspocus,
+  onChangePayload,
+  onLoadDocumentPayload,
+  onStoreDocumentPayload,
+  Server
+} from '@hocuspocus/server'
+import { IncomingHttpHeaders } from 'http'
+import * as Y from 'yjs'
+
+import { CuboBackendApi } from '..'
+import { DirectConnection } from './types'
+
+export * from './types'
+
+export class CuboCrdt<T extends CuboApiEntitiesMap<T>> {
+  public server: Server
+  private documentPrefix = 'cubo'
+  private defaultCrdtFieldAlias = 'crdt_data'
+
+  constructor(public api: CuboBackendApi<T>) {
+    this.init()
+  }
+
+  private init() {
+    const { onAuthenticate, onLoadDocument, onChange, onStoreDocument, afterLoadDocument, connected, ...config } = this.api.options.crdt
+
+    this.server = new Server({
+      port: config.port,
+      extensions: [
+        new Database({
+          fetch: async ({ documentName, context, document, instance, requestHeaders, requestParameters, socketId, connectionConfig }) => {
+            try {
+              const [prefix] = documentName.split('.')
+              if (prefix !== this.documentPrefix) {
+                return null
+              }
+
+              const { entityAlias, id } = this.parseDocumentName(documentName)
+
+              console.log('fetch', entityAlias, id)
+              if (id <= 0) {
+                return null
+              }
+
+              const entity = await this.api.helpers.getEntityByAlias(entityAlias as keyof T)
+              if (!entity) {
+                throw new Error(`Entity not found: "${entityAlias}"`)
+              }
+
+              const isSupported = this.isCrdtSupported(entity)
+              if (!isSupported) {
+                throw new Error(`Entity "${entityAlias}" does not support CRDT`)
+              }
+
+              const fieldKey = this.getCrdtField(entity)?.alias
+
+              if (!fieldKey) {
+                return null
+              }
+
+              const db = await this.api.helpers.getConnection('write')
+
+              const [data] = await db.connection.query(`SELECT ${fieldKey} from ${entityAlias} where id=:id`, {
+                type: QueryTypes.SELECT,
+                replacements: {
+                  id
+                }
+              })
+
+              if (!data) {
+                throw new Error(`Entity "${entityAlias}" with id=${id} not found`)
+              }
+
+              const raw = (data as any)[fieldKey]
+              // console.log('raw', raw)
+
+              if (!raw) {
+                return null
+              }
+
+              return raw
+            } catch (error) {
+              console.log('error', error)
+            }
+          },
+          store: async ({ documentName, document, state, context }) => {
+            const [prefix] = documentName.split('.')
+            if (prefix !== this.documentPrefix) {
+              return null
+            }
+
+            const { entityAlias, id } = this.parseDocumentName(documentName)
+
+            if (id <= 0) {
+              return
+            }
+
+            const entity = await this.api.helpers.getEntityByAlias(entityAlias as keyof T)
+            if (!entity) {
+              throw new Error(`Entity not found: "${entityAlias}"`)
+            }
+
+            const isSupported = this.isCrdtSupported(entity)
+            if (!isSupported) {
+              throw new Error(`Entity "${entityAlias}" does not support CRDT`)
+            }
+
+            const entityMap = document.getMap(entityAlias)
+            const body: Record<string, any> = {}
+
+            const fieldKey = this.getCrdtField(entity)?.alias
+
+            for (const [key, value] of entityMap.entries()) {
+              if (key === fieldKey) {
+                continue
+              }
+
+              body[key] = value
+            }
+
+            // console.log('state', state)
+
+            body[fieldKey] = state
+
+            body.modified_at = new Date().toISOString()
+
+            await this.api.updateOne(
+              entityAlias as Extract<keyof T, string>,
+              {
+                query: { id },
+                body
+              },
+              {
+                crdt: false,
+                extra: {
+                  auth: context.auth
+                },
+                excludeHooks: [],
+                queryOptions: { withDeleted: true }
+              }
+            )
+          }
+        })
+      ],
+      onAuthenticate: async (data) => {
+        try {
+          return onAuthenticate?.(this.api, data)
+        } catch (err) {
+          throw new Error('Ошибка авторизации: ' + err)
+        }
+      },
+      onLoadDocument: async (data) => {
+        const { documentName } = data
+
+        const [prefix] = documentName.split('.')
+        if (prefix === this.documentPrefix) {
+          const { entityAlias, id } = this.parseDocumentName(documentName)
+
+          if (entityAlias) {
+            const augmentation = this.api.options.augmentations?.[entityAlias]
+
+            if (augmentation?.onLoadDocument) {
+              await augmentation.onLoadDocument(data)
+            }
+          }
+        }
+
+        return onLoadDocument?.(this.api, data)
+      },
+      afterLoadDocument: async (data) => {
+        const { documentName, document, instance } = data
+
+        return afterLoadDocument?.(this.api, data)
+      },
+      connected: async (data) => {
+        const { documentName, instance } = data
+
+        const [prefix] = documentName?.split('.')
+        if (prefix === this.documentPrefix) {
+          const document = instance.documents.get(documentName)
+
+          const { entityAlias, id } = this.parseDocumentName(documentName)
+
+          if (entityAlias && document) {
+            const entityMap = document.getMap(entityAlias)
+            const id = entityMap.get('id')
+
+            if (!id) {
+              document.connections.forEach(({ connection }) => {
+                connection.close({
+                  reason: 'Document not loaded',
+                  code: 500
+                })
+              })
+              instance.unloadDocument(document)
+            }
+          }
+        }
+
+        return connected?.(this.api, data)
+      },
+      onChange: async (data) => {
+        const { documentName } = data
+
+        const [prefix] = documentName.split('.')
+        if (prefix === this.documentPrefix) {
+          const { entityAlias, id } = this.parseDocumentName(documentName)
+
+          if (entityAlias) {
+            const augmentation = this.api.options.augmentations?.[entityAlias]
+
+            if (augmentation?.onChangeDocument) {
+              await augmentation.onChangeDocument(data)
+            }
+          }
+        }
+
+        return onChange?.(this.api, data)
+      },
+      onStoreDocument: async (data) => {
+        const { documentName } = data
+
+        const [prefix] = documentName.split('.')
+        if (prefix === this.documentPrefix) {
+          const { entityAlias, id } = this.parseDocumentName(documentName)
+
+          if (entityAlias) {
+            const augmentation = this.api.options.augmentations?.[entityAlias]
+
+            if (augmentation?.onStoreDocument) {
+              await augmentation.onStoreDocument(data)
+            }
+          }
+        }
+
+        return onStoreDocument?.(this.api, data)
+      },
+
+      ...config
+    })
+  }
+
+  public async listen() {
+    await this.server.listen()
+  }
+
+  private parseDocumentName(documentName: string) {
+    const parts = documentName.split('.')
+
+    const [prefix, entityAlias, idRaw] = parts
+
+    if (prefix !== this.documentPrefix) {
+      throw new Error(`Invalid prefix: "${prefix}", expected "${this.documentPrefix}"`)
+    }
+
+    if (!entityAlias) {
+      throw new Error(`Missing entity alias in documentName: "${documentName}"`)
+    }
+
+    const id = Number(idRaw)
+    if (Number.isNaN(id)) {
+      throw new Error(`Invalid id in documentName: "${documentName}"`)
+    }
+
+    return { entityAlias: entityAlias as Extract<keyof T, string>, id }
+  }
+
+  public createDocumentName(entityAlias: string, id: number | string, prefix?: string) {
+    const parts = [prefix || this.documentPrefix, entityAlias, id]
+
+    return parts.join('.')
+  }
+
+  public isCrdtSupported(entity: CuboEntity) {
+    const hasCrdtField = entity.fields.some((f) => f.alias === this.api.options.crdt?.fieldAlias || this.defaultCrdtFieldAlias)
+
+    return hasCrdtField
+  }
+
+  public getCrdtField(entity: CuboEntity) {
+    const crdtField = entity.fields.find((f) => f.alias === this.api.options.crdt?.fieldAlias || this.defaultCrdtFieldAlias)
+
+    return crdtField
+  }
+
+  public getDocumentByEntity(entityAlias: string, id: number, requestHeaders: IncomingHttpHeaders) {
+    const documentName = this.createDocumentName(entityAlias, id)
+    const fakeSocketId = 'api-' + crypto.randomUUID()
+
+    return this.server.hocuspocus.createDocument(documentName, { headers: requestHeaders, url: '/collaboration' }, fakeSocketId, {
+      readOnly: false,
+      isAuthenticated: false
+    })
+  }
+
+  public async openDirectConnection(entityAlias: string, id: number, context?: any) {
+    const documentName = this.createDocumentName(entityAlias, id)
+    if (!documentName) {
+      return
+    }
+
+    return this.server.hocuspocus.openDirectConnection(documentName, context)
+  }
+
+  public async connectionDisconnect(connection: DirectConnection) {
+    const { document, instance, context } = connection
+
+    if (document) {
+      document.removeDirectConnection()
+
+      await instance.storeDocumentHooks(document, {
+        clientsCount: document.getConnectionsCount(),
+        context: context,
+        document: document,
+        documentName: document.name,
+        instance: instance,
+        requestHeaders: {},
+        requestParameters: new URLSearchParams(),
+        socketId: 'server'
+      })
+
+      // // If the direct connection was the only connection to the document
+      // // then we should trigger the onDisconnect hook for
+      // // this doc and unload the document
+      // if (document.getConnectionsCount() === 0) {
+      //   await instance.hooks('onDisconnect', {
+      //     instance: instance,
+      //     clientsCount: document.getConnectionsCount(),
+      //     context: context,
+      //     document: document,
+      //     socketId: 'server',
+      //     documentName: document.name,
+      //     requestHeaders: {},
+      //     requestParameters: new URLSearchParams()
+      //   })
+
+      //   await instance.unloadDocument(document)
+      // }
+
+      connection.document = null
+    }
+  }
+
+  public async updateDocument(params: { entityAlias: string; id: number; entity: CuboEntity }, dto: Record<string, any>, context?: any) {
+    const { entityAlias, id, entity } = params
+
+    const connection = await this.openDirectConnection(entityAlias, id, context)
+    if (!connection) {
+      return
+    }
+
+    const { document } = connection
+    if (!document) {
+      return
+    }
+
+    const entityMap = document.getMap(entityAlias)
+
+    const fieldKey = this.getCrdtField(entity)?.alias
+
+    document.transact(() => {
+      for (const [key, value] of Object.entries(dto)) {
+        if (key === fieldKey) {
+          continue
+        }
+
+        let preparedValue = value
+
+        if (value instanceof Date) {
+          preparedValue = value.toISOString()
+        }
+
+        entityMap.set(key, preparedValue)
+      }
+    })
+
+    return this.connectionDisconnect(connection)
+  }
+
+  public async createState(params: { entityAlias: string; id: number; entity: CuboEntity }, dto: Record<string, any>) {
+    const { entityAlias, id, entity } = params
+
+    const document = new Y.Doc()
+    if (!document) {
+      return
+    }
+
+    const entityMap = document.getMap(entityAlias)
+
+    const fieldKey = this.getCrdtField(entity)?.alias
+
+    document.transact(() => {
+      for (const [key, value] of Object.entries(dto)) {
+        if (key === fieldKey) {
+          continue
+        }
+
+        let preparedValue = value
+
+        if (value instanceof Date) {
+          preparedValue = value.toISOString()
+        }
+
+        entityMap.set(key, preparedValue)
+      }
+    })
+
+    return Buffer.from(Y.encodeStateAsUpdate(document))
+  }
+}
+
+export { beforeSyncPayload, DirectConnection, Document, Hocuspocus, onChangePayload, onLoadDocumentPayload, onStoreDocumentPayload }

package/src/crdt/types.ts

@@ -0,0 +1,34 @@
+import { CuboApiEntitiesMap } from '@cuboapp/types'
+import {
+  DirectConnection as DirectConnectionInterface,
+  Document,
+  Hocuspocus,
+  ServerConfiguration,
+  afterLoadDocumentPayload,
+  connectedPayload,
+  onAuthenticatePayload,
+  onChangePayload,
+  onLoadDocumentPayload,
+  onStoreDocumentPayload
+} from '@hocuspocus/server'
+
+import { CuboBackendApi } from '..'
+
+export type CuboCrdtConfiguration<T extends CuboApiEntitiesMap<T>> = Omit<
+  Partial<ServerConfiguration>,
+  'onAuthenticate' | 'onLoadDocument' | 'onChange' | 'onStoreDocument' | 'afterLoadDocument' | 'connected'
+> & {
+  onAuthenticate?: (api: CuboBackendApi<T>, data: onAuthenticatePayload) => Promise<any>
+  onLoadDocument?: (api: CuboBackendApi<T>, data: onLoadDocumentPayload) => Promise<any>
+  onChange?: (api: CuboBackendApi<T>, data: onChangePayload) => Promise<any>
+  onStoreDocument?: (api: CuboBackendApi<T>, data: onStoreDocumentPayload) => Promise<any>
+  afterLoadDocument?: (api: CuboBackendApi<T>, data: afterLoadDocumentPayload) => Promise<any>
+  connected?: (api: CuboBackendApi<T>, data: connectedPayload) => Promise<any>
+  fieldAlias?: string
+}
+
+export type DirectConnection = {
+  document: Document | null
+  instance: Hocuspocus
+  context: any
+} & DirectConnectionInterface

package/src/helpers/convert.ts

@@ -1,10 +1,10 @@
 import { CuboApiEntitiesMap, CuboEntity, CuboEntityField } from '@cuboapp/types'
 import { get, set, unset } from '@cuboapp/utils'
+import { CUBO_ENTITY_FIELD_TYPE } from '@cuboapp/constants'
 
 import { CUBO_CRUD_DEFAULT_FIELDS } from '../constants'
 import { CuboCrudFindQuery, CuboCrudRequest } from '../types'
 
-import { CUBO_ENTITY_FIELD_TYPE } from '@cuboapp/constants'
 import { ApiHelpers } from '.'
 
 export class ApiHelpersConvert<T extends CuboApiEntitiesMap<T>> {
@@ -98,6 +98,7 @@ export class ApiHelpersConvert<T extends CuboApiEntitiesMap<T>> {
         console.warn('field type "file" is not implemented yet')
         return undefined
       case CUBO_ENTITY_FIELD_TYPE.JSON:
+      case CUBO_ENTITY_FIELD_TYPE.BINARY:
         return value
       case CUBO_ENTITY_FIELD_TYPE.NUMBER:
         return ![undefined, null].includes(value) ? parseFloat(value) : null

package/src/helpers/data.ts

@@ -128,6 +128,7 @@ export class ApiHelpersData<T extends CuboApiEntitiesMap<T>> {
             errors.push(`"${key}": file field type is not already implemented`)
             break
           case CUBO_ENTITY_FIELD_TYPE.JSON:
+          case CUBO_ENTITY_FIELD_TYPE.BINARY:
             if (value === null) {
               prepared[key] = value
             } else if (Buffer.byteLength(JSON.stringify(value)) >= 65535 * 50) {

package/src/helpers/index.ts

@@ -1,8 +1,8 @@
 import { CUBO_ENTITY_FIELD_TYPE } from '@cuboapp/constants'
-import { Database, createDatabase, type Options } from '@cuboapp/database'
+import { Database, createDatabase } from '@cuboapp/database'
 import { CuboApiEntitiesMap, CuboEntity, CuboEntityField, CuboEntityFieldExtra, CuboEntityFieldType } from '@cuboapp/types'
 
-import { CuboBackendApi, CuboBackendApiDbConnectionType, CuboCrudFindQuery, CuboCrudRequest } from '../..'
+import { CuboBackendApi } from '../..'
 import {
   CUBO_CRUD_DEFAULT_FIELDS,
   CUBO_CRUD_DEFAULT_PAGE_LIMIT,
@@ -11,6 +11,7 @@ import {
   CUBO_CRUD_QUERY_KEY_REGEX,
   CUBO_CRUD_QUERY_SYMBOL_NOT
 } from '../constants'
+import { CuboBackendApiDbConnectionType, CuboCrudFindQuery, CuboCrudRequest } from '../types'
 
 import { ApiHelpersConvert } from './convert'
 import { ApiHelpersData } from './data'
@@ -65,7 +66,10 @@ export class ApiHelpers<T extends CuboApiEntitiesMap<T>> {
     // condition parts
     switch (parts.length) {
       case 3:
-        if (parts[0] !== CUBO_CRUD_QUERY_SYMBOL_NOT) {
+        if (parts[0] === 'btw') {
+          condition = parts[0]
+          value = `${parts[1]}:${parts[2]}`
+        } else if (parts[0] !== CUBO_CRUD_QUERY_SYMBOL_NOT) {
           throw new Error(`"${field.alias}": three-level value allow only "not" condition at first level (${value})`)
         } else if (!Object.values(CUBO_CRUD_QUERY_CONDITION).includes(parts[1])) {
           throw new Error(`"${field.alias}": incorrect condition "${parts[1]}" in second level (${value})`)
@@ -140,9 +144,18 @@ export class ApiHelpers<T extends CuboApiEntitiesMap<T>> {
             } else {
               if ([CUBO_CRUD_QUERY_CONDITION.IN, CUBO_CRUD_QUERY_CONDITION.NIN].includes(condition)) {
                 value = value.split(',').map(Number)
+              } else if (condition === CUBO_CRUD_QUERY_CONDITION.BTW) {
+                const [fromStr, toStr] = value.split(':')
+                const from = precision === 0 ? parseInt(fromStr) : parseFloat(fromStr)
+                const to = precision === 0 ? parseInt(toStr) : parseFloat(toStr)
+
+                if (isNaN(from) || isNaN(to)) {
+                  throw new Error(`"${field.alias}": both values for "btw" must be valid numbers`)
+                }
+
+                value = precision > 0 ? `${from.toFixed(precision)}:${to.toFixed(precision)}` : `${from}:${to}`
               } else {
                 const potentialValue = precision === 0 ? parseInt(value) : parseFloat(value)
-
                 if (isNaN(potentialValue)) {
                   throw new Error('"' + field.alias + '" must be number')
                 } else {
@@ -211,7 +224,7 @@ export class ApiHelpers<T extends CuboApiEntitiesMap<T>> {
   ): { conditions: string[]; replacements: Record<string, any> } {
     const conditions: string[] = []
     const replacements: Record<string, any> = {}
-
+    // console.log(field, condition, value)
     switch (condition) {
       case CUBO_CRUD_QUERY_CONDITION.ILIKE:
         conditions.push(`${field} ${is_not ? 'not ilike' : 'ilike'} :${replacement}`)
@@ -253,6 +266,22 @@ export class ApiHelpers<T extends CuboApiEntitiesMap<T>> {
         conditions.push(`${field} ${is_not ? '>=' : '<='} :${replacement}`)
         replacements[replacement] = value
         break
+      case CUBO_CRUD_QUERY_CONDITION.BTW: {
+        const [from, to] = String(value).split(':')
+        const fromKey = `${replacement}_from`
+        const toKey = `${replacement}_to`
+
+        if (!from || !to) {
+          throw new Error(`Invalid "btw" value: "${value}"`)
+        }
+
+        const expr = `${field} >= :${fromKey} AND ${field} <= :${toKey}`
+        conditions.push(is_not ? `NOT (${expr})` : `(${expr})`)
+
+        replacements[fromKey] = from
+        replacements[toKey] = to
+        break
+      }
       case CUBO_CRUD_QUERY_CONDITION.EQ:
       default:
         if (value === null) {
@@ -287,75 +316,13 @@ export class ApiHelpers<T extends CuboApiEntitiesMap<T>> {
     }
 
     this.connecting[type] = new Promise(async (success) => {
-      let database: string = ''
-      let host: string = ''
-      let port: number = 0
-      let username: string = ''
-      let password: string = ''
-      let schema: string = ''
-      let dialect: string = ''
-      let dialectModule: any = undefined
-
-      if (!this.api.options.db.options) {
-        throw 'DB connection is not configured'
-      }
-
-      const dbConfig = this.api.options.db.options
-
-      let cfg: Options = {}
-      if (dbConfig) {
-        if (dbConfig[type]) {
-          cfg = dbConfig[type]
-        } else {
-          cfg = dbConfig as Options
-        }
-      }
-
-      database = cfg.database as string
-      dialectModule = cfg.dialectModule
-      host = cfg.host as string
-      port = cfg.port as number
-      username = cfg.username as string
-      password = cfg.password as string
-      schema = cfg.schema as string
-      dialect = cfg.dialect as string
-
-      const config: Options = {
-        host,
-        port,
-        username,
-        password,
-        database,
-        dialect: dialect as any,
-        timezone: '+03:00',
-        logging: false,
-        schema
-      }
-
-      if (dialectModule) {
-        config.dialectModule = dialectModule
-      }
-
-      // cert for yandex cloud
-      if (/yandexcloud/.test(config.host!)) {
-        config.dialectOptions = {
-          ...(config.dialectOptions || {}),
-          ...{
-            ssl: {
-              ca: await fetch('https://storage.yandexcloud.net/cloud-certs/CA.pem').then((res) => res.text()),
-              rejectUnauthorized: true,
-              target_session_attrs: 'read-write',
-              require: true
-            }
-          }
-        }
-      }
+      const { iaas, pk_key, pk_type, ...config } = this.api.options.db.options
 
-      this.connections[type] = createDatabase(config)
+      this.connections[type] = createDatabase(config, JSON.parse(JSON.stringify({ iaas, pk_key, pk_type })))
       await this.connections[type].connect()
 
-      if (dialect === 'postgres') {
-        await this.connections[type].connection.query(`SET schema '${schema}';`)
+      if (config.dialect === 'postgres' && config.schema) {
+        await this.connections[type].connection.query(`SET schema '${config.schema}';`)
       }
 
       success(this.connections[type])