@teamplay/backend 0.1.5

package/CHANGELOG.md

@@ -0,0 +1,9 @@
+# v0.1.5 (Mon May 27 2024)
+
+#### 🚀 Enhancement
+
+- feat: move backend implementation from startupjs. Add an example app. ([@cray0000](https://github.com/cray0000))
+
+#### Authors: 1
+
+- Pavel Zhukov ([@cray0000](https://github.com/cray0000))

package/README.md

@@ -0,0 +1,70 @@
+# Teamplay Backend
+
+> Create a new ShareDB backend instance
+
+## Installation
+
+```sh
+yarn add @teamplay/backend
+```
+
+## Requirements
+
+## Configuration
+
+The package can be configured using the following environment variables:
+
+- `MONGO_URL`: Specifies the URL for the MongoDB connection. MongoDB is used as the primary database unless overridden by setting `NO_MONGO` to `true`.
+- `NO_MONGO`: When set to `true`, this variable disables the use of MongoDB. In this case, the package will utilize a Mingo database, persisting data to SQLite.
+- `DB_PATH`: Defines the file path for the SQLite database. This setting is relevant when `NO_MONGO` is `true`. If `DB_PATH` is not specified, the default file 'sqlite.db' will be used.
+- `DB_LOAD_SNAPSHOT`: An optional variable that can be set with a path to a SQLite snapshot file. This setting is relevant when `NO_MONGO` is `true`. If provided, the SQLite database will be initialized from this snapshot.
+- `DB_READONLY`: Set to `true` to disable persistence to SQLite.
+- `REDIS_URL`: URL for the Redis connection.
+- `NO_REDIS`: Set to `true` to use a mocked Redis client.
+
+### Database Initialization
+
+The backend toggles between MongoDB and Mingo for database operations, influenced by environment settings:
+
+- **MongoDB**: Used when `MONGO_URL` is set and `NO_MONGO` is `false`.
+- **Mingo and SQLite**: Activated by setting `NO_MONGO` to `true`. Mingo handles operations, while SQLite is used solely for data persistence, initialized from `DB_PATH` if provided.
+- **SQLite Snapshot**: When `DB_LOAD_SNAPSHOT` is set, SQLite is initialized with this pre-populated data snapshot and pulls data to Mingo.
+
+This setup ensures flexibility in database management based on environment configurations.
+Data persistence can be disabled by setting `DB_READONLY` to `true`.
+
+### Cloning Existing SQLite Snapshots
+
+In scenarios where there is a need to create new SQLite database snapshots based on existing ones, the combination of `DB_LOAD_SNAPSHOT` and `DB_PATH` environment variables can be effectively utilized for cloning. This feature is particularly useful for initializing new environments or testing purposes.
+
+To clone an existing snapshot, set `DB_LOAD_SNAPSHOT` to the path of the existing snapshot file (e.g., `snapshot.db`) and specify a new file name in `DB_PATH` for the cloned database (e.g., `clone-snapshot.db`).
+
+For example:
+```bash
+NO_MONGO=true DB_LOAD_SNAPSHOT=snapshot.db DB_PATH=clone-snapshot.db startupjs server
+```
+
+Upon server initialization with these settings, the system will clone the data from `snapshot.db` into a new database file named `clone-snapshot.db`. Subsequently, the server will continue operations using the new `clone-snapshot.db` as its database source. This cloning process provides a seamless way to replicate and utilize existing database states in new instances.
+
+## Usage
+
+To use the backend package in your StartupJS project, import and initialize it as follows:
+
+```js
+import getBackend from '@teamplay/backend'
+
+export default async function runServer () {
+  const backend = await getBackend(options)
+  // ...
+}
+```
+
+where `options` are:
+
+- `pollDebounce`: the minimum delay between subsequent database polls. It is used individually for each collection in the database. This is used to batch updates and reduce load on the database.
+
+## License
+
+MIT
+
+(c) Decision Mapper - http://decisionmapper.com

package/db/index.js

@@ -0,0 +1,24 @@
+export const {
+  db,
+  mongo, // (optional, only if mongo is used) mongoClient.db()
+  mongoClient, // (optional, only if mongo is used)
+  createMongoIndex = () => {}, // (optional, only if mongo is used; mock provided) create mongo indexes
+  sqlite // (optional, only if mingo-sqlite is used) sqlite3 db instance
+} = await getDb({
+  mongoUrl: process.env.MONGO_URL,
+  disableMongo: process.env.NO_MONGO,
+  isReadonly: process.env.DB_READONLY
+})
+
+async function getDb ({ mongoUrl, disableMongo, isReadonly }) {
+  if (mongoUrl && !disableMongo) {
+    console.log('Database: mongo')
+    return await import('./mongo.js')
+  } else if (isReadonly) {
+    console.log('Database: mingo-memory (no data persistency)')
+    return await import('./mingo-memory.js')
+  } else {
+    console.log('Database: mingo-sqlite (persist data to a local SQLite file)')
+    return await import('./mingo-sqlite.js')
+  }
+}

package/db/mingo-memory.js

@@ -0,0 +1,17 @@
+import ShareDbMingoMemory from 'sharedb-mingo-memory'
+import { getExistingSqliteDb, loadSqliteDbToMingo } from './utils.js'
+
+export const db = await getMingoDb({
+  loadSnapshotPath: process.env.DB_LOAD_SNAPSHOT
+})
+
+async function getMingoDb ({ loadSnapshotPath }) {
+  const db = new ShareDbMingoMemory()
+
+  if (loadSnapshotPath) {
+    const sqliteDb = getExistingSqliteDb(loadSnapshotPath)
+    await loadSqliteDbToMingo(sqliteDb, db)
+  }
+
+  return db
+}

package/db/mingo-sqlite.js

@@ -0,0 +1,167 @@
+import ShareDbMingoMemory from 'sharedb-mingo-memory'
+import { resolve } from 'path'
+import sqlite3 from 'sqlite3'
+import { v4 as uuid } from 'uuid'
+import { loadSqliteDbToMingo, getExistingSqliteDb } from './utils.js'
+
+const DEFAULT_DB_PATH = './local.db'
+
+export const { db, sqlite } = await getMingoSqliteDb({
+  dbPath: process.env.DB_PATH,
+  loadSnapshotPath: process.env.DB_LOAD_SNAPSHOT
+})
+
+async function getMingoSqliteDb ({ dbPath, loadSnapshotPath }) {
+  const db = new ShareDbMingoMemory()
+  dbPath = resolve(dbPath || DEFAULT_DB_PATH)
+  const sqliteDb = await getOrCreateSqliteDb(dbPath)
+
+  await deleteExpiredDocumentsOps(sqliteDb)
+
+  if (loadSnapshotPath) {
+    const snapshotSqliteDb = getExistingSqliteDb(loadSnapshotPath)
+    await cloneSqliteDb(snapshotSqliteDb, sqliteDb)
+  }
+
+  await loadSqliteDbToMingo(sqliteDb, db)
+
+  patchMingoForSQLitePersistence(sqliteDb, db)
+
+  return { db, sqlite: sqliteDb }
+}
+
+// override the commit method to save changes to SQLite
+function patchMingoForSQLitePersistence (sqliteDb, shareDbMingo) {
+  const originalCommit = shareDbMingo.commit
+
+  shareDbMingo.commit = function (collection, docId, op, snapshot, options, callback) {
+    originalCommit.call(this, collection, docId, op, snapshot, options, async err => {
+      if (err) return callback(err)
+
+      try {
+        await new Promise((resolve, reject) => {
+          sqliteDb.run(`
+            REPLACE INTO documents (collection, id, data) VALUES (?, ?, ?)
+          `, [collection, docId, JSON.stringify(snapshot)], err => err ? reject(err) : resolve())
+        })
+        await new Promise((resolve, reject) => {
+          sqliteDb.run(`
+            INSERT INTO ops (id, collection, documentId, op) VALUES (?, ?, ?, ?)
+          `, [uuid(), collection, docId, JSON.stringify(op)], err => err ? reject(err) : resolve())
+        })
+      } catch (err) {
+        throw Error('Error saving to SQLite:\n', err.message)
+      }
+
+      callback(null, true)
+    })
+  }
+}
+
+async function deleteExpiredDocumentsOps (sqliteDb) {
+  return await new Promise((resolve, reject) => {
+    sqliteDb.run(`
+      DELETE FROM ops
+      WHERE
+          json_extract(op, '$.m.ts') < (strftime('%s', 'now') - 24 * 60 * 60 ) * 1000
+          AND
+          json_extract(op, '$.v') < (
+              SELECT (json_extract(data, '$.v') - 1)
+              FROM documents
+              WHERE documents.id = ops.documentId AND documents.collection = ops.collection
+          );
+    `, err => err ? reject(err) : resolve())
+  })
+}
+
+async function getOrCreateSqliteDb (dbPath) {
+  const sqliteDb = new sqlite3.Database(dbPath)
+
+  try {
+    await new Promise((resolve, reject) => {
+      sqliteDb.run(`
+        CREATE TABLE IF NOT EXISTS documents (
+          collection TEXT,
+          id TEXT,
+          data TEXT,
+          PRIMARY KEY (collection, id)
+        )
+      `, err => err ? reject(err) : resolve())
+    })
+    await new Promise((resolve, reject) => {
+      sqliteDb.run(`
+        CREATE TABLE IF NOT EXISTS ops (
+          id UUID PRIMARY KEY,
+          collection TEXT,
+          documentId TEXT,
+          op TEXT
+        )
+      `, err => err ? reject(err) : resolve())
+    })
+    await new Promise((resolve, reject) => {
+      sqliteDb.run(`
+        CREATE TABLE IF NOT EXISTS files (
+          id TEXT PRIMARY KEY,
+          data BLOB,
+          createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+      `, err => err ? reject(err) : resolve())
+    })
+  } catch (err) {
+    throw Error('Error creating SQLite file db and/or tables:\n' + err.message)
+  }
+  console.log('Using SQLite DB from file:', dbPath)
+  return sqliteDb
+}
+
+// clone each table from source to target
+// TODO: Do it in batches of 100 to avoid memory issues.
+async function cloneSqliteDb (source, target) {
+  try {
+    { // clone 'documents'
+      const rows = await new Promise((resolve, reject) => {
+        source.all('SELECT * FROM documents', [], (err, rows) => err ? reject(err) : resolve(rows))
+      })
+      const promises = []
+      for (const row of rows) {
+        promises.push(new Promise((resolve, reject) => {
+          target.run(`
+            INSERT INTO documents (collection, id, data) VALUES (?, ?, ?)
+          `, [row.collection, row.id, row.data], err => err ? reject(err) : resolve())
+        }))
+      }
+      await Promise.all(promises)
+    }
+    { // clone 'ops'
+      const rows = await new Promise((resolve, reject) => {
+        source.all('SELECT * FROM ops', [], (err, rows) => err ? reject(err) : resolve(rows))
+      })
+      const promises = []
+      for (const row of rows) {
+        promises.push(new Promise((resolve, reject) => {
+          target.run(`
+            INSERT INTO ops (id, collection, documentId, op) VALUES (?, ?, ?, ?)
+          `, [row.id, row.collection, row.documentId, row.op], err => err ? reject(err) : resolve())
+        }))
+      }
+      await Promise.all(promises)
+    }
+    { // clone 'files'
+      // TODO: Clone them one by one to avoid memory issues, since files are large.
+      const rows = await new Promise((resolve, reject) => {
+        source.all('SELECT * FROM files', [], (err, rows) => err ? reject(err) : resolve(rows))
+      })
+      const promises = []
+      for (const row of rows) {
+        promises.push(new Promise((resolve, reject) => {
+          target.run(`
+            INSERT INTO files (id, data) VALUES (?, ?)
+          `, [row.id, row.data], err => err ? reject(err) : resolve())
+        }))
+        await Promise.all(promises)
+      }
+    }
+  } catch (err) {
+    throw Error('Error cloning SQLite DB:\n' + err.message)
+  }
+}

package/db/mongo.js

@@ -0,0 +1,40 @@
+import fs from 'fs'
+import { MongoClient } from 'mongodb'
+import ShareDbMongo from 'sharedb-mongo'
+
+export const { db, mongo, mongoClient, createMongoIndex } = getMongoDb({
+  url: process.env.MONGO_URL,
+  optsString: process.env.MONGO_OPTS,
+  sslKeyPath: process.env.MONGO_SSL_KEY_PATH,
+  sslCertPath: process.env.MONGO_SSL_CERT_PATH,
+  sslCaPath: process.env.MONGO_SSL_CA_PATH
+})
+
+function getMongoDb ({ url, optsString, sslKeyPath, sslCertPath, sslCaPath }) {
+  const options = { useUnifiedTopology: true }
+
+  if (typeof optsString === 'string') {
+    const { key, cert, ca } = JSON.parse(optsString)
+    options.sslKey = fs.readFileSync(key)
+    options.sslCert = fs.readFileSync(cert)
+    options.sslCA = fs.readFileSync(ca)
+  } else if (sslKeyPath) {
+    options.sslKey = fs.readFileSync(sslKeyPath)
+    options.sslCert = fs.readFileSync(sslCertPath)
+    options.sslCA = fs.readFileSync(sslCaPath)
+  }
+
+  const mongoClient = new MongoClient(url, options)
+  const mongo = mongoClient.db()
+  return {
+    db: ShareDbMongo({
+      mongo: callback => callback(null, mongoClient),
+      allowAllQueries: true
+    }),
+    mongo,
+    mongoClient,
+    createMongoIndex (collection, keys, options) {
+      return mongo.collection(collection).createIndex(keys, options)
+    }
+  }
+}

package/db/utils.js

@@ -0,0 +1,45 @@
+import sqlite3 from 'sqlite3'
+import { existsSync } from 'fs'
+import { resolve } from 'path'
+
+export async function loadSqliteDbToMingo (sqliteDb, mingo) {
+  return new Promise((resolve, reject) => {
+    sqliteDb.all('SELECT collection, id, data FROM documents', [], (err, docsRows) => {
+      if (err) return reject(err)
+
+      sqliteDb.all('SELECT collection, documentId, op FROM ops', [], (err, opsRows) => {
+        if (err) return reject(err)
+
+        const docsOpsByDocId = opsRows.reduce((byId, row) => {
+          const values = byId[`${row.collection}.${row.documentId}`] || []
+          const op = JSON.parse(row.op)
+          return { ...byId, [`${row.collection}.${row.documentId}`]: [...values, op] }
+        }, {})
+
+        for (const row of docsRows) {
+          if (!mingo.docs[row.collection]) {
+            mingo.docs[row.collection] = {}
+            mingo.ops[row.collection] = {}
+          }
+
+          const docOps = (docsOpsByDocId[`${row.collection}.${row.id}`] || []).sort((a, b) => a.v - b.v)
+          const firstOp = docOps[0]
+
+          mingo.ops[row.collection][row.id] = firstOp?.v > 0 ? [...new Array(firstOp.v), ...docOps] : docOps
+          mingo.docs[row.collection][row.id] = JSON.parse(row.data)
+        }
+        resolve()
+        console.log('DB data was loaded from SQLite to shareDbMingo')
+      })
+    })
+  })
+}
+
+export function getExistingSqliteDb (dbPath) {
+  dbPath = resolve(dbPath)
+  console.log('[mingo] Getting existing sqlite db from:', dbPath)
+  if (!existsSync(dbPath)) {
+    throw Error('[mingo] SQLite db file doesn\'t exist:', dbPath)
+  }
+  return new sqlite3.Database(dbPath)
+}

package/features/serverAggregate.js

@@ -0,0 +1,38 @@
+import _serverAggregate from '@teamplay/server-aggregate'
+import { isAggregationFunction } from '@teamplay/utils/aggregation'
+
+export default function serverAggregate (backend, { customCheck, models = {} } = {}) {
+  _serverAggregate(backend, { customCheck })
+
+  for (const modelPattern in models) {
+    for (const aggregationName in models[modelPattern]) {
+      const aggregation = models[modelPattern][aggregationName]
+      if (!isAggregationFunction(aggregation)) continue
+      // support only top-level collections
+      const collection = modelPattern
+      if (/\./.test(collection)) throw Error(ERRORS.onlyTopLevelCollections(modelPattern, aggregationName))
+      backend.addAggregate(
+        collection,
+        aggregationName,
+        (queryParams, shareRequest) => {
+          const session = shareRequest.agent.connectSession
+          // TODO: rewrite to use $ here, or create a separate root $ for each user
+          // const model = global.__clients[userId].model
+          const context = { session, collection }
+          return aggregation(queryParams, context)
+        }
+      )
+    }
+  }
+
+  console.log('✓ Security: only server-side aggregations are allowed')
+}
+
+const ERRORS = {
+  onlyTopLevelCollections: (modelPattern, aggregationName) => `
+    serverAggregate: you can only define aggregations in the top-level collection models
+      (i.e. 'model/items.js')
+      Found aggregation '${aggregationName}' in '${modelPattern}'.
+      Move it to the top-level collection model: 'models/${modelPattern.split('.')[0]}.js'
+  `
+}

package/features/validateSchema.js

@@ -0,0 +1,41 @@
+import sharedbSchema from '@teamplay/sharedb-schema'
+import { transformSchema } from '@teamplay/schema'
+
+export default function validateSchema (backend, { models = {} } = {}) {
+  const schemaPerCollection = { schemas: {}, formats: {}, validators: {} }
+
+  for (const modelPattern in models) {
+    let { schema, factory } = models[modelPattern]
+
+    if (factory) {
+      // TODO: implement getting schema from factory
+      // schemaPerCollection.schemas[modelPattern.replace('.*', '')] = ORM[path].OrmEntity
+      throw Error('factory model: NOT IMPLEMENTED')
+    } else if (schema) {
+      const collectionName = modelPattern
+      if (/\./.test(collectionName)) throw Error(ERRORS.onlyTopLevelCollections(modelPattern))
+      // transform schema from simplified format to full format
+      schema = transformSchema(schema, { collectionName })
+      schemaPerCollection.schemas[collectionName] = schema
+    }
+
+    // allow any 'service' collection structure
+    // since 'service' collection is used in our startupjs libraries
+    // and we don't have a tool to collect scheme from all packages right now
+    schemaPerCollection.schemas.service = transformSchema({
+      type: 'object', properties: {}, additionalProperties: true
+    })
+  }
+
+  sharedbSchema(backend, schemaPerCollection)
+  console.log('✓ Security: JSON-schema validation of DB collections on backend is enabled')
+}
+
+const ERRORS = {
+  onlyTopLevelCollections: (modelPattern) => `
+    validateSchema: you can only define schema in the top-level collection models
+      (i.e. 'model/items.js')
+      Found schema in '${modelPattern}'.
+      Move it to the top-level collection model: 'models/${modelPattern.split('.')[0]}.js'
+  `
+}

package/index.d.ts

@@ -0,0 +1,48 @@
+import { type Redis } from 'ioredis'
+import { type Db, type MongoClient, type CreateIndexesOptions } from 'mongodb'
+import { type Database as SqliteDatabase } from 'sqlite3'
+
+export interface BackendOptions {
+  secure?: boolean
+  ee?: any // EventEmitter instance, replace 'any' with more specific type if available
+  pollDebounce?: number
+  flushRedis?: boolean
+  extraDbs?: any // Replace 'any' with actual type
+  hooks?: (backend: any) => void // Replace 'any' with Backend class type
+  accessControl?: boolean
+  serverAggregate?: boolean | { customCheck: () => any } // TODO: remove customCheck support
+  validateSchema?: boolean
+  silentLogs?: boolean
+}
+
+// Accommodate both the specific MongoDB index creation signature and a dummy function
+interface CreateMongoIndexFunction {
+  (collection: string, keys: Record<string, number | string>, options?: CreateIndexesOptions): void
+  (): void
+}
+
+export const createMongoIndex: CreateMongoIndexFunction
+
+export interface RedisExports {
+  redis: Redis
+  redlock: any
+  Redlock: any
+}
+
+export interface DbExports {
+  db: any // This refers to the ShareDB connection, not MongoDB connection
+  mongo: Db // MongoDB Db instance from MongoClient.db()
+  mongoClient: MongoClient
+  sqlite: SqliteDatabase // sqlite3 Database instance
+}
+
+export function createBackend (options?: BackendOptions): any // Replace 'any' with Backend class type
+
+// Exports instances and constructors for Redis and database connections
+export const redis: RedisExports['redis']
+export const redlock: RedisExports['redlock']
+export const Redlock: RedisExports['Redlock']
+export const db: DbExports['db']
+export const mongo: DbExports['mongo'] | undefined
+export const mongoClient: DbExports['mongoClient'] | undefined
+export const sqlite: DbExports['sqlite'] | undefined

package/index.js

@@ -0,0 +1,115 @@
+import ShareDbAccess, {
+  registerOrmRules
+} from '@teamplay/sharedb-access'
+import ShareDB from 'sharedb'
+import shareDbHooks from 'sharedb-hooks'
+import { pubsub } from './redis/index.js'
+import { db } from './db/index.js'
+import maybeFlushRedis from './redis/maybeFlushRedis.js'
+import initValidateSchema from './features/validateSchema.js'
+import initServerAggregate from './features/serverAggregate.js'
+
+export { redis, redlock, Redlock } from './redis/index.js'
+export { db, mongo, mongoClient, createMongoIndex, sqlite } from './db/index.js'
+
+const usersConnectionCounter = {}
+global.__clients = {}
+
+export default function createBackend ({
+  secure = false,
+  pollDebounce,
+  flushRedis = true,
+  extraDbs,
+  hooks,
+  accessControl = secure,
+  serverAggregate = secure,
+  validateSchema = secure,
+  models,
+  verbose = true
+} = {}) {
+  // pollDebounce is the minimum time in ms between query polls in sharedb
+  if (pollDebounce) db.pollDebounce = pollDebounce
+
+  // Maybe flush redis when starting the app.
+  // When running in cluster this should only run on one instance and once a day
+  // so redlock is used to guarantee that.
+  if (flushRedis) maybeFlushRedis()
+
+  const backend = new ShareDB({
+    db,
+    pubsub,
+    extraDbs
+  })
+
+  // sharedb-hooks
+  shareDbHooks(backend)
+
+  if (hooks) hooks(backend)
+
+  const ORM = global.STARTUP_JS_ORM || {}
+
+  // sharedb-access
+  if (accessControl) {
+    // eslint-disable-next-line
+    new ShareDbAccess(backend, { dontUseOldDocs: true })
+
+    for (const path in ORM) {
+      const ormEntity = ORM[path].OrmEntity
+
+      const { access } = ormEntity
+      const isFactory = !!ormEntity.factory
+
+      // TODO:
+      //   - move registerOrmRulesFromFactory and registerOrmRules to this library
+      //   - rewrite factories check to not use model anymore
+      if (isFactory) {
+        throw Error('Sharedb-access does not support ORM factories yet')
+        // registerOrmRulesFromFactory(backend, path, ormEntity)
+      } else if (access) {
+        registerOrmRules(backend, path, access)
+      }
+    }
+
+    console.log('sharedb-access is working')
+  }
+
+  if (serverAggregate) {
+    initServerAggregate(backend, { customCheck: serverAggregate?.customCheck, models })
+  }
+
+  if (validateSchema && process.env.NODE_ENV !== 'production') {
+    initValidateSchema(backend, { models })
+  }
+
+  backend.on('client', (client, reject) => {
+    const req = client.upgradeReq
+    if (!req) return
+
+    const userId = client.session?.userId || req.session?.userId
+
+    // TODO: rewrite to use $ here, or create a separate root $ for each user
+    // if (!global.__clients[userId]) {
+    //   const model = backend.createModel()
+    //   global.__clients[userId] = { model }
+    // }
+
+    usersConnectionCounter[userId] = ~~usersConnectionCounter[userId] + 1
+
+    const userAgent = req.headers && req.headers['user-agent']
+    if (verbose) console.log('[WS OPENED]:', userId, userAgent)
+
+    client.once('close', () => {
+      if (verbose) console.log('[WS CLOSED]', userId)
+
+      usersConnectionCounter[userId] -= 1
+
+      // TODO: rewrite to use $ here, or create a separate root $ for each user
+      // if (usersConnectionCounter[userId] <= 0) {
+      //   global.__clients[userId].model.close()
+      //   delete global.__clients[userId]
+      // }
+    })
+  })
+
+  return backend
+}

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@teamplay/backend",
+  "version": "0.1.5",
+  "description": "Create new ShareDB backend instance",
+  "type": "module",
+  "main": "index.js",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": "./index.js"
+  },
+  "dependencies": {
+    "@teamplay/schema": "^0.1.5",
+    "@teamplay/server-aggregate": "^0.1.5",
+    "@teamplay/sharedb-access": "^0.1.5",
+    "@teamplay/sharedb-schema": "^0.1.5",
+    "@teamplay/utils": "^0.1.5",
+    "@types/ioredis-mock": "^8.2.5",
+    "ioredis": "^5.3.2",
+    "ioredis-mock": "^8.9.0",
+    "mongodb": "^6.0.0",
+    "redlock": "^3.0.0",
+    "sharedb": "^5.0.0",
+    "sharedb-hooks": "~4.0.0",
+    "sharedb-mingo-memory": "^3.0.0",
+    "sharedb-mongo": "^4.1.2",
+    "sharedb-redis-pubsub": "^2.0.1",
+    "sqlite3": "^5.1.6",
+    "uuid": "^9.0.1"
+  },
+  "gitHead": "ad85edf62088d1e8d28e9b5e22f2dfb5dfc00bec"
+}

package/redis/index.js

@@ -0,0 +1,94 @@
+import { readFileSync } from 'fs'
+import Redis from 'ioredis'
+import RedisMock from 'ioredis-mock'
+import Redlock from 'redlock'
+import redisPubSub from 'sharedb-redis-pubsub'
+
+export const {
+  redis,
+  redisObserver
+} = getUniversalRedis({
+  disableRedis: process.env.NO_REDIS,
+  redisOpts: process.env.REDIS_OPTS,
+  redisUrl: process.env.REDIS_URL,
+  keyPrefix: generatePrefix({
+    mongoUrl: process.env.MONGO_URL,
+    baseUrl: process.env.BASE_URL
+  })
+})
+
+export const pubsub = redisPubSub({
+  client: redis,
+  observer: redisObserver
+})
+
+export const redlock = getRedlock(redis)
+
+export { Redlock }
+
+function getUniversalRedis ({ disableRedis, redisOpts, redisUrl, keyPrefix }) {
+  if (!disableRedis) {
+    if (typeof redisOpts === 'string') {
+      redisOpts = JSON.parse(redisOpts)
+      let tls = {}
+
+      if (redisOpts.key) {
+        tls = {
+          key: readFileSync(redisOpts.key),
+          cert: readFileSync(redisOpts.cert),
+          ca: readFileSync(redisOpts.ca)
+        }
+      }
+
+      const options = {
+        sentinels: redisOpts.sentinels,
+        sslPort: redisOpts.ssl_port || '6380',
+        tls,
+        name: 'mymaster',
+        db: redisOpts.db || 0,
+        password: redisOpts.password,
+        keyPrefix
+      }
+
+      return {
+        redis: new Redis(options),
+        redisObserver: new Redis(options)
+      }
+    } else if (redisUrl) {
+      return {
+        redis: new Redis(redisUrl, { keyPrefix }),
+        redisObserver: new Redis(redisUrl, { keyPrefix })
+      }
+    }
+  }
+  return {
+    redis: new RedisMock({ keyPrefix }),
+    redisObserver: new RedisMock({ keyPrefix })
+  }
+}
+
+function getRedlock (redis) {
+  return new Redlock([redis], {
+    driftFactor: 0.01,
+    retryCount: 2,
+    retryDelay: 10,
+    retryJitter: 10
+  })
+}
+
+// Use prefix for ShareDB's pubsub. This prevents issues with multiple
+// projects using the same redis db.
+// We use a combination of MONGO_URL and BASE_URL to generate a simple
+// hash because together they are going to be unique no matter whether
+// it's run on localhost or on the production server.
+// ref: https://github.com/share/sharedb/issues/420
+function generatePrefix ({ mongoUrl, baseUrl }) {
+  return '_' + simpleNumericHash('' + mongoUrl + baseUrl)
+}
+
+// ref: https://gist.github.com/hyamamoto/fd435505d29ebfa3d9716fd2be8d42f0?permalink_comment_id=2694461#gistcomment-2694461
+function simpleNumericHash (s) {
+  let i, h
+  for (i = 0, h = 0; i < s.length; i++) h = Math.imul(31, h) + s.charCodeAt(i) | 0
+  return h
+}

package/redis/maybeFlushRedis.js

@@ -0,0 +1,54 @@
+import { redis, redlock, Redlock } from './index.js'
+
+export default function maybeFlushRedis () {
+  redis.once('connect', _maybeFlushRedis)
+}
+
+async function _maybeFlushRedis () {
+  // Always flush redis db in development or if a force env flag is specified.
+  if (process.env.NODE_ENV !== 'production' || process.env.FORCE_REDIS_FLUSH) {
+    await flushRedis()
+    return
+  }
+
+  // In production we flush redis db once a day using locking in redis itself.
+  const ONE_DAY = 1000 * 60 * 60 * 24
+  const LOCK_FLUSH_DB_KEY = 'startupjs_service_flushdb'
+
+  try {
+    try {
+      await redlock.lock(LOCK_FLUSH_DB_KEY, ONE_DAY)
+    } catch (err) {
+      if (err instanceof Redlock.LockError) {
+        console.log('>> No need to do Redis Flush DB yet (lock for one day is still present)')
+        return
+      } else { throw err }
+    }
+
+    console.log('>>> FLUSHING REDIS DB (this should happen only once a day)')
+    await flushRedis()
+
+    // Re-lock right away.
+    console.log('>>> RE-LOCK REDIS DB FLUSH CHECK (for one day)')
+    try {
+      await redlock.lock(LOCK_FLUSH_DB_KEY, ONE_DAY)
+    } catch (err) {
+      console.error('Error while re-locking flushdb redis lock!\n' + err)
+    }
+  } catch (err) {
+    console.error('Error while performing redis DB flushing!\n' + err)
+  }
+}
+
+function flushRedis () {
+  return new Promise(resolve => {
+    redis.flushdb((err, didSucceed) => {
+      if (err) {
+        console.error('Redis flushdb err:', err)
+      } else {
+        console.log('Redis flushdb success:', didSucceed)
+      }
+      resolve()
+    })
+  })
+}