@live-change/db-server 0.9.83 → 0.9.84

package/Dockerfile

@@ -9,6 +9,7 @@ RUN npm install -g levelup
 RUN npm install -g memdown
 RUN npm install -g node-lmdb
 RUN npm install -g subleveldown
+RUN npm install -g typescript
 
 COPY package.json .
 RUN npm install -g @live-change/db-server@`echo "console.log(require('./package.json').version)" | node`
@@ -16,6 +17,7 @@ RUN npm install -g @live-change/db-client@`echo "console.log(require('./package.
 RUN rm package.json
 
 RUN cd /usr/local/lib/node_modules/@live-change/db-server/node_modules/@live-change/db-admin/; npm run build
+RUN cd /usr/local/lib/node_modules/@live-change/db-server/; npm run build
 
 EXPOSE 9417
 

package/build-docker-and-upload.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 VERSION=`echo "console.log(require('./package.json').version)" | node`
-PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7"
-#PLATFORMS="linux/amd64"
+#PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7"
+PLATFORMS="linux/amd64"
 echo building docker image for version $VERSION
 docker buildx build --debug \
   --platform $PLATFORMS \

package/dist/bin/server.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/bin/server.js

@@ -0,0 +1,181 @@
+#!/usr/bin/env node
+import path from 'path';
+import fs from 'fs';
+import service from "os-service";
+import Server from '../lib/Server.js';
+import { client as WSClient } from "@live-change/dao-websocket";
+import ReactiveDao from '@live-change/dao';
+import * as db from "@live-change/db";
+import profileOutput from "../lib/profileOutput.js";
+import { performance } from 'perf_hooks';
+import yargs from 'yargs';
+import { fileURLToPath } from 'url';
+import { SsrServer, createLoopbackDao } from "@live-change/server";
+process.on('unhandledRejection', (reason, promise) => {
+    console.error('Unhandled Promise Rejection', "reason", reason, "stack", reason.stack, "promise", promise);
+    //process.exit(1) // TODO: database should not fail because of it, but it should be logged somewhere
+});
+process.on('uncaughtException', function (err) {
+    console.error('uncaughtException', err);
+});
+function serverOptions(yargs) {
+    yargs.option('port', {
+        describe: 'port to bind on',
+        type: 'number',
+        default: 9417
+    });
+    yargs.option('host', {
+        describe: 'bind host',
+        type: 'string',
+        default: '::'
+    });
+    yargs.option('master', {
+        describe: 'replicate from master',
+        type: 'string',
+        default: null
+    });
+    yargs.option('slowStart', {
+        type: 'boolean',
+        description: 'start indexes one after another(better for debugging)'
+    });
+    yargs.option('profileLog', {
+        type: 'string',
+        description: 'profiling log file path'
+    });
+}
+function storeOptions(yargs, defaults = {}) {
+    yargs.option('dbRoot', {
+        describe: 'server root directory',
+        type: 'string',
+        default: defaults.dbRoot || '.'
+    });
+    yargs.option('backend', {
+        describe: 'database backend engine ( lmdb | leveldb | rocksdb | memdown | mem )',
+        type: "string",
+        default: defaults.backend || 'lmdb'
+    });
+    yargs.option('backendUrl', {
+        describe: 'remote database backend address',
+        type: "string"
+    });
+}
+function serviceOptions(yargs) {
+    yargs.option('serviceName', {
+        describe: 'name of service',
+        type: 'string',
+        default: 'liveChangeDb'
+    });
+}
+const argv = yargs(process.argv.slice(2)) // eslint-disable-line
+    .command('create', 'create database root', (yargs) => {
+    storeOptions(yargs);
+}, (argv) => create(argv))
+    .command('serve', 'start server', (yargs) => {
+    serverOptions(yargs);
+    storeOptions(yargs);
+    yargs.option('service', {
+        describe: 'run as service',
+        type: 'boolean'
+    });
+}, (argv) => {
+    if (argv.service) {
+        if (argv.verbose) {
+            console.info('running as service!');
+        }
+        service.run(function () {
+            service.stop(0);
+        });
+    }
+    serve(argv);
+})
+    .command('install', 'install system service', (yargs) => {
+    serviceOptions(yargs);
+    serverOptions(yargs);
+    storeOptions(yargs, { dbRoot: '/var/db/liveChangeDb' });
+}, async (argv) => {
+    const programArgs = ["serve", '--service',
+        '--port', argv.port, '--host', argv.host,
+        '--dbRoot', argv.dbRoot, '--backend', argv.backend];
+    if (argv.verbose)
+        console.info(`creating system service ${argv.serviceName}`);
+    await fs.promises.mkdir(argv.dbRoot, { recursive: true });
+    service.add(argv.serviceName, { programArgs }, function (error) {
+        if (error)
+            console.trace(error);
+    });
+})
+    .command('uninstall', 'remove system service', (yargs) => {
+    serviceOptions(yargs);
+}, (argv) => {
+    if (argv.verbose)
+        console.info(`removing system service ${argv.serviceName}`);
+    service.remove(argv.serviceName, function (error) {
+        if (error)
+            console.trace(error);
+    });
+})
+    .option('verbose', {
+    alias: 'v',
+    type: 'boolean',
+    description: 'Run with verbose logging'
+}).argv;
+async function create({ dbRoot, backend, verbose }) {
+    await fs.promises.mkdir(dbRoot, { recursive: true });
+    if (verbose)
+        console.info(`creating database in ${path.resolve(dbRoot)}`);
+    let server = new Server({ dbRoot, backend });
+    await server.initialize({ forceNew: true });
+    if (verbose)
+        console.info(`database server root directory created.`);
+}
+async function serve(argv) {
+    const { dbRoot, backend, backendUrl, verbose, host, port, master, slowStart, profileLog } = argv;
+    if (profileLog) {
+        const out = profileOutput(profileLog);
+        await db.profileLog.startLog(out, performance);
+    }
+    const profileOp = await db.profileLog.begin({ operation: "startingDbServer", ...argv });
+    if (verbose)
+        console.info(`starting server in ${path.resolve(dbRoot)}`);
+    let server = new Server({
+        dbRoot, backend, backendUrl, master,
+        slowStart
+    });
+    process.on('unhandledRejection', (reason, promise) => {
+        if (reason.stack && reason.stack.match(/\s(userCode:([a-z0-9_.\/-]+):([0-9]+):([0-9]+))\n/i)) {
+            server.handleUnhandledRejectionInQuery(reason, promise);
+        }
+        else {
+            console.error('Unhandled Promise Rejection', (reason && reason.stack) || reason, "Promise:", promise);
+            //process.exit(1) // TODO: database should not fail because of it, but it should be logged somewhere
+        }
+    });
+    await server.initialize();
+    if (verbose)
+        console.info(`database initialized!`);
+    if (verbose)
+        console.info(`listening on: ${argv.host}:${argv.port}`);
+    const ssrRoot = path.dirname(fileURLToPath(import.meta.resolve("@live-change/db-admin/front/vite.config.js")));
+    const http = await server.getHttp();
+    const { app } = http;
+    const dev = await fs.promises.access(path.resolve(ssrRoot, './dist'), fs.constants.R_OK)
+        .then(r => false).catch(r => true);
+    if (dev)
+        console.log("STARTING ADMIN IN DEV MODE!");
+    const manifest = (dev || argv.spa)
+        ? null
+        : JSON.parse(fs.readFileSync((path.resolve(ssrRoot, 'dist/client/.vite/ssr-manifest.json'))));
+    const admin = new SsrServer(app, manifest, {
+        dev,
+        fastAuth: true,
+        root: ssrRoot,
+        daoFactory: async (credentials, ip) => {
+            return await createLoopbackDao(credentials, () => server.apiServer.daoFactory(credentials, ip));
+        }
+    });
+    admin.start();
+    http.server.listen(port, host);
+    if (verbose)
+        console.info(`server started!`);
+    await db.profileLog.end(profileOp);
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export default Server;
+import Server from './lib/Server.js';

package/dist/index.js

@@ -0,0 +1,2 @@
+import Server from './lib/Server.js';
+export default Server;

package/dist/lib/Replicator.d.ts

@@ -0,0 +1,13 @@
+export default Replicator;
+declare class Replicator {
+    constructor(server: any);
+    server: any;
+    dao: any;
+    listObservable: any;
+    databases: Map<any, any>;
+    start(): void;
+    close(): void;
+    set(dbList: any): void;
+    push(name: any): void;
+    remove(dbName: any): Promise<void>;
+}

package/dist/lib/Replicator.js

@@ -0,0 +1,370 @@
+const bucketSize = 100;
+const LOG_NOTSTARTED = 0;
+const LOG_UPDATING = 2;
+const LOG_READY = 3;
+const LOG_CLOSED = 4;
+class LogReplicator {
+    constructor(databaseName, server, logName, destination) {
+        this.server = server;
+        this.databaseName = databaseName;
+        this.logName = logName;
+        this.destination = destination;
+        this.dao = server.masterDao;
+        this.state = LOG_NOTSTARTED;
+        this.lastUpdateId = null;
+        this.observable = null;
+    }
+    observeMore() {
+        if (this.observable)
+            this.observable.unobserve(this);
+        this.observable = this.dao.observable(['database', 'logRange', this.databaseName, this.logName, {
+                gt: this.lastUpdateId,
+                limit: bucketSize
+            }]);
+        this.observable.observe(this);
+    }
+    async set(entries) {
+        for (let entry of entries) {
+            await this.destination.data.put(entry);
+            this.lastUpdateId = entry.id;
+        }
+        if (this.observable.list.length == bucketSize) {
+            this.observeMore();
+        }
+    }
+    putByField(_fd, id, entry, _reverse, oldObject) {
+        this.destination.data.put(entry);
+        this.lastUpdateId = entry.id;
+        if (this.observable.list.length == bucketSize) {
+            this.observeMore();
+        }
+    }
+    async updateAll() {
+        let entries;
+        do {
+            entries = await this.dao.get(['database', 'logRange', this.databaseName, this.logName, {
+                    gt: this.lastUpdateId,
+                    limit: bucketSize
+                }]);
+            for (let entry of entries) {
+                await this.destination.data.put(entry);
+                this.lastUpdateId = entry.id;
+            }
+        } while (entries.length >= bucketSize && this.state != LOG_CLOSED);
+        this.state = LOG_READY;
+        this.observeMore();
+    }
+    async start() {
+        const lastLogOperations = await this.destination.rangeGet({ reverse: true, limit: 1 });
+        const lastLogOperation = lastLogOperations[0];
+        if (!lastLogOperation) {
+            this.lastUpdateId = '';
+        }
+        else {
+            this.lastUpdateId = lastLogOperation.id; // one second overlap
+        }
+        this.state = LOG_UPDATING;
+        this.updateAll();
+    }
+    close() {
+        this.state = LOG_CLOSED;
+        if (this.observable)
+            this.observable.unobserve(this);
+    }
+}
+const TABLE_NOTSTARTED = 0;
+const TABLE_COPYING = 1;
+const TABLE_UPDATING = 2;
+const TABLE_READY = 3;
+const TABLE_CLOSED = 4;
+class TableReplicator {
+    constructor(databaseName, server, tableName, destination) {
+        this.server = server;
+        this.databaseName = databaseName;
+        this.tableName = tableName;
+        this.destination = destination;
+        this.dao = server.masterDao;
+        this.state = TABLE_NOTSTARTED;
+        this.lastUpdateId = null;
+        this.observable = null;
+    }
+    observeMore() {
+        if (this.observable)
+            this.observable.unobserve(this);
+        this.observable = this.dao.observable(['database', 'tableOpLogRange', this.databaseName, this.tableName, {
+                gt: this.lastUpdateId,
+                limit: bucketSize
+            }]);
+        this.observable.observe(this);
+    }
+    async set(ops) {
+        for (let op of ops) {
+            if (op.type == 'put')
+                await this.destination.put(op.object);
+            if (op.type == 'delete')
+                await this.destination.delete(op.object);
+            if (op.type == 'clearOpLog')
+                await this.destination.clearOpLog(op.to);
+            this.lastUpdateId = op.id;
+        }
+        if (this.observable.list.length == bucketSize) {
+            this.observeMore();
+        }
+    }
+    async putByField(_fd, id, op, _reverse, oldObject) {
+        if (op.type == 'put')
+            this.destination.put(op.object);
+        if (op.type == 'delete')
+            this.destination.delete(op.object);
+        if (op.type == 'clearOpLog')
+            await this.destination.clearOpLog(op.to);
+        this.lastUpdateId = op.id;
+        if (this.observable.list.length == bucketSize) {
+            this.observeMore();
+        }
+    }
+    async applyOp(op) {
+        if (op.operation.type == 'put')
+            await this.destination.data.put(op.operation.object);
+        if (op.operation.type == 'delete')
+            await this.destination.data.delete(op.operation.object);
+        await this.destination.opLog.put(op);
+        this.lastUpdateId = op.id;
+    }
+    async updateAll() {
+        // console.log("UPDATE TABLE", this.databaseName, this.tableName, "FROM", this.lastUpdateId)
+        let ops;
+        do {
+            ops = await this.dao.get(['database', 'tableOpLogRange', this.databaseName, this.tableName, {
+                    gt: this.lastUpdateId,
+                    limit: bucketSize
+                }]);
+            //console.log("GOT TABLE OPS", this.databaseName, this.tableName, ":", JSON.stringify(ops, null, "  "))
+            for (let op of ops) {
+                await this.applyOp(op);
+            }
+        } while (ops.length >= bucketSize && this.state != TABLE_CLOSED);
+        this.state = TABLE_READY;
+        this.observeMore();
+    }
+    async copyAll() {
+        await Promise.all([
+            this.destination.opLog.rangeDelete({}),
+            this.destination.data.rangeDelete({}),
+        ]);
+        //console.log("COPY TABLE", this.databaseName, this.tableName)
+        let copyPosition = '';
+        let data;
+        do {
+            data = await this.dao.get(['database', 'tableRange', this.databaseName, this.tableName, {
+                    gt: copyPosition,
+                    limit: bucketSize
+                }]);
+            for (let obj of data) {
+                await this.destination.put(obj);
+            }
+        } while (data.length >= bucketSize && this.state != TABLE_CLOSED);
+        this.state = TABLE_READY;
+        this.observeMore();
+    }
+    async start() {
+        const lastTableOperation = await this.destination.opLog.rangeGet({ reverse: true, limit: 1 })[0];
+        const firstSourceOperation = await this.dao.get(['database', 'tableOpLogRange', this.databaseName, this.tableName, {
+                limit: 1
+            }]);
+        if (!lastTableOperation // If empty table, or opLog desynchronized(cleared)
+            || (firstSourceOperation && firstSourceOperation.id > lastTableOperation.id)) { // Copy data first
+            let tableCreateTimestamp = Date.now();
+            this.state = TABLE_COPYING;
+            this.lastUpdateId = ('' + (tableCreateTimestamp - 1000)).padStart(16, '0'); // one second overlay
+            this.copyAll();
+        }
+        else {
+            this.state = TABLE_UPDATING;
+            this.lastUpdateId = lastTableOperation.id; // one second overlap
+            this.updateAll();
+        }
+    }
+    close() {
+        this.state = TABLE_CLOSED;
+        if (this.observable)
+            this.observable.unobserve(this);
+    }
+}
+class ListReplicator {
+    constructor(databaseName, server, list, type, copy, factory) {
+        this.name = databaseName;
+        this.database = null;
+        this.server = server;
+        this.dao = server.masterDao;
+        this.list = list;
+        this.type = type;
+        this.copy = copy;
+        this.factory = factory;
+        this.observable = null;
+        this.objects = new Map();
+    }
+    start() {
+        this.database = this.server.databases.get(this.name);
+        /// Observe system table because it is organized by table uids which is better for rename operations
+        this.observable = this.dao.observable(['database', 'tableRange', 'system', this.name + '_' + this.list, {}]);
+        this.observable.observe(this);
+    }
+    async createObject(obj) {
+        let foundName = null;
+        const configList = this.database.config[this.list];
+        for (let k in configList)
+            if (configList[k].uid == obj.id)
+                foundName = k;
+        if (foundName) {
+            if (foundName != obj.name) {
+                await this.database['rename' + this.type](foundName, obj.name);
+                await this.server.databases.get('system').table(this.name + '_' + this.list).update(obj.id, [
+                    { op: 'merge', property: 'name', value: obj.name }
+                ]);
+            }
+            /// TODO: compare configurations?
+        }
+        else {
+            if (configList[obj.name]) { // Object with this name already exists, overwrite?!
+                console.error("NAME", obj.name, "EXISTS WITH ANOTHER UID", configList[obj.name].uid, "!=", obj.id, "IN DB", this.name);
+            }
+            try {
+                const object = await this.factory(obj);
+                await this.server.databases.get('system').table(this.name + '_' + this.list).put(obj);
+            }
+            catch (e) {
+                console.log("CREATE OBJ", obj, "IN DB", this.name, "FAILED");
+                console.error(e);
+            }
+        }
+        if (this.copy) {
+            const objRep = this.objects.get(obj.id);
+            if (objRep) {
+                objRep.close();
+                this.objects.delete(obj.id);
+            }
+            const objectReplicator = this.copy(obj);
+            this.objects.set(obj.id, objectReplicator);
+            objectReplicator.start();
+        }
+    }
+    set(objects) {
+        for (const obj of objects) {
+            this.createObject(obj);
+        }
+        for (const key in this.database.config[this.list]) {
+            const value = this.database.config[this.list][key];
+            const obj = objects.find(o => o.id == value.uid);
+            if (!obj) {
+                this.database['delete' + this.type](key);
+                this.server.databases.get('system').table(this.name + '_' + this.list).delete(value.uid);
+            }
+        }
+    }
+    putByField(_fd, uid, object, _reverse, oldObject) {
+        this.createObject(obj);
+    }
+    removeByField(_fd, uid, oldObject) {
+        this.database['delete' + this.type](oldObject.name);
+        this.server.databases.get('system').table(this.name + '_' + this.list).delete(uid);
+        if (this.copy) {
+            this.objects.get(uid).close();
+            this.objects.delete(uid);
+        }
+    }
+}
+class DatabaseReplicator {
+    constructor(databaseName, server) {
+        this.name = databaseName;
+        this.server = server;
+        this.dao = server.masterDao;
+        this.tables = null;
+        this.indexes = null;
+        this.logs = null;
+    }
+    async start() {
+        this.databaseConfig = this.dao.get(['database', 'databaseConfig', this.name]);
+        if (!this.server.metadata.databases[this.name]) {
+            this.server.metadata.databases[this.name] = this.databaseConfig;
+            const database = await this.server.initDatabase(this.name, this.databaseConfig);
+            this.server.databases.set(this.name, database);
+            this.server.databasesListObservable.push(this.name);
+            await Promise.all([
+                this.server.databases.get('system').createTable(this.name + "_tables"),
+                this.server.databases.get('system').createTable(this.name + "_logs"),
+                this.server.databases.get('system').createTable(this.name + "_indexes")
+            ]);
+            await this.server.saveMetadata();
+        }
+        const database = this.server.databases.get(this.name);
+        this.tables = new ListReplicator(this.name, this.server, 'tables', 'Table', (obj) => new TableReplicator(this.name, this.server, obj.name, database.table(obj.name)), (obj) => database.createTable(obj.name, obj.config));
+        this.indexes = new ListReplicator(this.name, this.server, 'indexes', 'Index', false, (obj) => {
+            //console.log("CREATE INDEX", obj.id, obj.name, "WITH UID", obj.config.uid)
+            return database.createIndex(obj.name, obj.config.code, obj.config.parameters, obj.config);
+        });
+        this.logs = new ListReplicator(this.name, this.server, 'logs', 'Log', (obj) => new LogReplicator(this.name, this.server, obj.name, database.log(obj.name)), (obj) => database.createLog(obj.name, obj.config));
+        this.tables.start();
+        this.indexes.start();
+        this.logs.start();
+    }
+    close() {
+        this.tables.close();
+        this.indexes.close();
+        this.logs.close();
+    }
+}
+class Replicator {
+    constructor(server) {
+        this.server = server;
+        this.dao = server.masterDao;
+        this.listObservable = null;
+        this.databases = new Map();
+    }
+    start() {
+        this.listObservable = this.dao.observable(['database', 'databasesList']);
+        this.listObservable.observe(this);
+    }
+    close() {
+        if (this.listObservable)
+            this.listObservable.unobserve(this);
+        for (let replicator of this.databases.values())
+            replicator.close();
+    }
+    set(dbList) {
+        if (!Array.isArray(dbList))
+            return;
+        for (let [name, db] of this.server.databases.entries()) {
+            if (name != 'system' && dbList.indexOf(name) == -1) { // Database removed
+                this.remove(name);
+            }
+        }
+        for (let name of dbList) {
+            this.push(name);
+        }
+    }
+    push(name) {
+        if (!this.databases.get(name)) {
+            const replicator = new DatabaseReplicator(name, this.server);
+            this.databases.set(name, replicator);
+            replicator.start();
+        }
+    }
+    async remove(dbName) {
+        const replicator = this.databases.get(dbName);
+        if (replicator) {
+            replicator.close();
+        }
+        delete this.server.metadata.databases[dbName];
+        this.server.databases.get(dbName).delete();
+        this.server.databaseStores.get(dbName).delete();
+        this.server.databasesListObservable.remove(dbName);
+        await Promise.all([
+            this.server.databases.get('system').deleteTable(dbName + "_tables"),
+            this.server.databases.get('system').deleteTable(dbName + "_logs"),
+            this.server.databases.get('system').deleteTable(dbName + "_indexes")
+        ]);
+        await this.server.saveMetadata();
+    }
+}
+export default Replicator;

package/dist/lib/Server.d.ts

@@ -0,0 +1,68 @@
+/// <reference types="node" resolution-mode="require"/>
+export default Server;
+declare class Server {
+    constructor(config: any);
+    config: any;
+    databases: Map<any, any>;
+    metadata: any;
+    databaseStores: Map<any, any>;
+    metadataSavePromise: Promise<void>;
+    databasesListObservable: any;
+    apiServer: any;
+    backends: {};
+    masterDao: import("@live-change/dao/lib/Dao.js").default;
+    replicator: Replicator;
+    createDaoConfig(session: any): {
+        database: {
+            type: string;
+            source: any;
+        };
+        serverDatabase: {
+            type: string;
+            source: any;
+        };
+        store: {
+            type: string;
+            source: any;
+        };
+        version: {
+            type: string;
+            source: any;
+        };
+        metadata: {
+            type: string;
+            source: any;
+        };
+    };
+    createDao(session: any): import("@live-change/dao/lib/Dao.js").default;
+    initialize(initOptions?: {}): Promise<void>;
+    checkInfoIntegrity(): Promise<void>;
+    initDatabase(dbName: any, dbConfig: any): Promise<Database>;
+    initializingDatabase: Database;
+    doSaveMetadata(): Promise<void>;
+    saveMetadata(): Promise<void>;
+    getHttp(): Promise<{
+        app: any;
+        sockJsServer: import("@live-change/sockjs/lib/server.js");
+        wsServer: any;
+        server: http.Server<typeof http.IncomingMessage, typeof http.ServerResponse>;
+    }>;
+    httpPromise: Promise<{
+        app: any;
+        sockJsServer: import("@live-change/sockjs/lib/server.js");
+        wsServer: any;
+        server: http.Server<typeof http.IncomingMessage, typeof http.ServerResponse>;
+    }>;
+    http: {
+        app: any;
+        sockJsServer: import("@live-change/sockjs/lib/server.js");
+        wsServer: any;
+        server: http.Server<typeof http.IncomingMessage, typeof http.ServerResponse>;
+    };
+    listen(...args: any[]): Promise<void>;
+    close(): Promise<void>;
+    handleUnhandledRejectionInQuery(reason: any, promise: any): void;
+}
+import Replicator from './Replicator.js';
+import { Database } from '@live-change/db';
+import http from 'http';