@vandenberghinc/volt 1.1.4 → 1.1.6

package/backend/src/database.ts

@@ -9,13 +9,23 @@
 import { ChildProcess, spawn } from "child_process";
 import { deserialize, serialize } from "bson";
 import { Transform } from 'stream';
-import { MongoClient, Collection as MongoCollection, ObjectId } from 'mongodb';
+import { MongoClient, Collection as MongoCollection, ObjectId, ServerApiVersion } from 'mongodb';
 import { logger, LogSource } from "./logger.js";
 import { Status } from "./status.js";
 import { vlib } from "@vinc";
+import { resolve } from "path";
 
 const log_source = new LogSource("Database")
 
+interface BaseOptions {
+    options?: Record<string, any>;
+    commit_quorum?: any;
+    forced?: boolean;
+}
+type IndexOptions =
+    (BaseOptions & { key: string; keys?: never }) |
+    (BaseOptions & { key?: never; keys: string[] });
+
 // ---------------------------------------------------------
 // Collection.
 // Path based collection, so "myfile", "mydir/myfile".
@@ -39,7 +49,7 @@ class Collection {
     static constructor_scheme = {
         name: "string",
         uid_based: "boolean",
-        ttl: {type: "number", default: null},
+        ttl: {type: "number", default: null}, // ttl in msec
         indexes: {
             type: "array",
             default: [],
@@ -48,7 +58,7 @@ class Collection {
                 scheme: {
                     key: {type: "string", required: (data) => data.key == null && data.keys == null },
                     keys: {
-                        type: ["string", "array"],
+                        type: ["array"],
                         required: (data) => data.key == null && data.keys == null, value_scheme: "string",
                         postprocess: (keys) => typeof keys === "string" ? [keys] : keys,
                     },
@@ -58,11 +68,6 @@ class Collection {
                 },
                 postprocess: (info) => {
                     if (typeof info === "string") return {keys: [info]}
-                    if (typeof info === "object" && info.key) {
-                        info.keys = info.key;
-                        delete info.key;
-                        return info;
-                    }
                     return info;
                 },
             },
@@ -74,19 +79,14 @@ class Collection {
     col: MongoCollection;
     name: string;
     uid_based: boolean;
-    ttl: number | null;
+    ttl: number | null; // ttl in msec
     ttl_enabled: boolean;
 
     constructor(
         name: string,
         collection: MongoCollection,
-        ttl: number | null = null,
-        indexes: Array<{
-            keys: string[] | string | Record<string, any>,
-            options?: Record<string, any>,
-            commit_quorum?: any,
-            forced?: boolean
-        }> = [],
+        ttl: number | null = null, // ttl in msec
+        indexes: IndexOptions[] = [],
         uid_based: boolean = false,
     ) {
 
@@ -120,7 +120,8 @@ class Collection {
         if (this.ttl_enabled) {
             this.col.dropIndex("_ttl_timestamp_1")
             .catch(err => {
-                if (err.codeName !== 'IndexNotFound') {
+                if (err.codeName !== 'IndexNotFound' && err.codeName !== 'NamespaceNotFound') {
+                    err.stack = new Error(err.message).stack;
                     throw err;
                 }    
             })
@@ -134,13 +135,8 @@ class Collection {
 
         // Create indexes.
         if (Array.isArray(indexes) && indexes.length > 0) {
-            for (const index of indexes) {
-                this.create_index({
-                    keys: index.keys,
-                    options: index.options,
-                    commit_quorum: index.commit_quorum,
-                    forced: index.forced,
-                })
+            for (let item of indexes) {
+                this.create_index(item)
             }
         }
     }
@@ -269,31 +265,27 @@ class Collection {
             @desc: The `commitQuorum` argument for the orignal mongodb `createIndex()` function.
      */
     async create_index({
-        keys, 
-        options = null, 
-        commit_quorum = null, 
+        key = undefined,
+        keys = undefined, // following the IndexOptions "key" or "keys" must be defined in typescript but not in raw js.
+        options = undefined, 
+        commit_quorum = undefined, 
         forced = false
-    }: {
-        keys: string[] | string | Record<string, any>, 
-        options?: Record<string, any> | null,
-        commit_quorum?: any,
-        forced?: boolean
-    }): Promise<string> {
+    }: IndexOptions): Promise<string> {
+
+        // Create keys objs per input type.
         let keys_obj: Record<string, number> = {};
-        if (typeof keys === "string") {
+        if (key) {
             keys_obj = {};
-            keys_obj[keys] = 1;
+            keys_obj[key] = 1;
         }
-        else if (Array.isArray(keys)) {
+        else if (Array.isArray(keys) && keys.length > 0) {
             keys_obj = {};
             for (const key of keys) {
                 keys_obj[key] = 1;
             }
         }
-        else if (keys == null || typeof keys !== "object") {
-            vlib.Scheme.throw_invalid_type("keys", keys, ["string", "string[]", "object"], true);
-        } else {
-            keys_obj = keys;
+        else {
+            throw new Error("Define one of the following parameters: [key, keys].");
         }
 
         // Drop index.
@@ -875,11 +867,7 @@ class UIDCollection {
     constructor(
         name: string,
         collection: MongoCollection,
-        indexes: Array<{
-            keys: string[] | string | Record<string, any>,
-            options?: Record<string, any>,
-            commit_quorum?: any
-        }> = [],
+        indexes: IndexOptions[] = [],
         ttl: number | null = null,
     ) {
         this._col = new Collection(name, collection, ttl, indexes, true);
@@ -901,16 +889,8 @@ class UIDCollection {
             @name: commitQuorum
             @desc: The `commitQuorum` argument for the orignal mongodb `createIndex()` function.
      */
-    async create_index({
-        keys, 
-        options = null, 
-        commit_quorum = null
-    }: {
-        keys: string[] | string | Record<string, any>,
-        options?: Record<string, any> | null,
-        commit_quorum?: any
-    }): Promise<string> {
-        return this._col.create_index({keys, options, commit_quorum});
+    async create_index(args: IndexOptions): Promise<string> {
+        return this._col.create_index(args);
     }
 
     /*  @docs:
@@ -1353,34 +1333,26 @@ class Database {
         preview: {type: "boolean", default: true},
         preview_ip_whitelist: {type: "array", default: []},
         daemon: {type: ["object", "boolean"], default: {}},
-        _server: "object",
+        _server: {type: ["object", "undefined"]},
     }
 
     uri: string | null;
-    preview: boolean;
-    preview_ip_whitelist: string[];
+    // preview: boolean;
+    // preview_ip_whitelist: string[];
     client_opts: Record<string, any> | null;
     config: Record<string, any>;
-    source: any; // Using vlib.Path type
+    source: vlib.Path | undefined; // Using vlib.Path type
     start_args: string[];
-    _collections: Array<{
+    _collections: {
         name: string,
         ttl?: number | null,
-        indexes?: Array<{
-            keys: string[] | string | Record<string, any>,
-            options?: Record<string, any>,
-            commit_quorum?: any
-        }>,
-    }>;
-    _uid_collections: Array<{
+        indexes?: string[] | IndexOptions[],
+    }[];
+    _uid_collections: {
         name: string,
         ttl?: number | null,
-        indexes?: Array<{
-            keys: string[] | string | Record<string, any>,
-            options?: Record<string, any>,
-            commit_quorum?: any
-        }>,
-    }>;
+        indexes?: string[] | IndexOptions[],
+    }[];
     server: any;
     client: MongoClient | null;
     collections: Record<string, Collection | UIDCollection>;
@@ -1388,6 +1360,9 @@ class Database {
     daemon?: any;
     db?: any;
 
+    // System.
+    _listed_cols: any;
+
     constructor({
         uri = null,
         source = null,
@@ -1396,41 +1371,33 @@ class Database {
         client = null,
         collections = [],
         uid_collections = [],
-        preview = true,
-        preview_ip_whitelist = [],
+        // preview = true,
+        // preview_ip_whitelist = [],
         daemon = {},
-        _server,
+        _server = undefined,
     }: {
         uri?: string | null,
         source?: string | null,
         config?: Record<string, any> | null,
         start_args?: string[],
         client?: Record<string, any> | null,
-        collections?: Array<{
+        collections?: {
             name: string,
             ttl?: number | null,
-            indexes?: Array<{
-                keys: string[] | string | Record<string, any>,
-                options?: Record<string, any>,
-                commit_quorum?: any
-            }>,
-        }>,
-        uid_collections?: Array<{
+            indexes?: string[] | IndexOptions[],
+        }[],
+        uid_collections?: {
             name: string,
             ttl?: number | null,
-            indexes?: Array<{
-                keys: string[] | string | Record<string, any>,
-                options?: Record<string, any>,
-                commit_quorum?: any
-            }>,
-        }>,
+            indexes?: string[] | IndexOptions[],
+        }[],
         preview?: boolean,
         preview_ip_whitelist?: string[],
         daemon?: Record<string, any> | boolean,
         _server: any,
     }) {
         // Checks.
-        if (_server.is_primary && uri == null) {
+        if (!_server || (_server.is_primary && uri == null)) {
             ({uri, config, start_args, config, client} = vlib.Scheme.verify({
                 object: arguments[0], 
                 check_unknown: true, 
@@ -1439,14 +1406,14 @@ class Database {
         }
         // Arguments.
         this.uri = uri;
-        this.preview = preview;
-        this.preview_ip_whitelist = preview_ip_whitelist;
+        // this.preview = preview;
+        // this.preview_ip_whitelist = preview_ip_whitelist;
         this.client_opts = client;
         this.config = config || {};
-        this.source = source != null ? new vlib.Path(source) : _server.source.join(".db");
+        this.source = source != null ? new vlib.Path(source) : _server?.source.join(".db");
         this.start_args = start_args;
-        this._collections = collections;
-        this._uid_collections = uid_collections;
+        this._collections = collections ?? [];
+        this._uid_collections = uid_collections ?? [];
         this.server = _server;
 
         // Attributes.
@@ -1454,18 +1421,21 @@ class Database {
         this.collections = {};
 
         // Initialize the service daemon.
-        if (this.server.daemon && daemon !== false) {
+        if (this.server?.daemon && daemon !== false) {
             const log_source = this.server.source.join(".logs");
             if (!log_source.exists()) {
                 log_source.mkdir_sync();
             }
+            if (!this.server) {
+                throw new Error("Parameter 'Database._server' must be defined for this behaviour.");
+            }
             this.daemon = new vlib.Daemon({
                 name: this.server.daemon.name + ".mongodb",
                 user: (daemon as Record<string, any>).user || this.server.daemon.user,
                 group: (daemon as Record<string, any>).group || this.server.daemon.group,
                 command: "mongod",
                 cwd: this.server.daemon.cwd,
-                args: ["--config", this.source.join("mongod.json").str(), ...this.start_args],
+                args: ["--config", this.source!.join("mongod.json").str(), ...this.start_args],
                 env: (daemon as Record<string, any>).env || this.server.daemon.env,
                 description: (daemon as Record<string, any>).description || `Service daemon for the mongo database of website ${this.server.domain}.`,
                 auto_restart: true,
@@ -1475,9 +1445,9 @@ class Database {
         }
     }
 
-    // Database preview.
+    /* Database preview.
     _initialize_db_preview(): void {
-        /*
+        
         if (this.preview && this.server.production === false) {
             this.server.endpoint(
 
@@ -2001,9 +1971,8 @@ class Database {
                     }
                 },
             )
-        }
-        */
-    }
+        } 
+    }*/
 
     // Connect.
     async connect(): Promise<void> {
@@ -2023,7 +1992,7 @@ class Database {
         // Set default config.
         if (this.config.systemLog === undefined) { this.config.systemLog = {}; }
 
-        this.config.systemLog.path = this.source.join("mongod.log").str()
+        this.config.systemLog.path = this.source?.join("mongod.log").str()
 
         if (this.config.systemLog.destination === undefined) {
             this.config.systemLog.destination = "file";
@@ -2034,30 +2003,32 @@ class Database {
         if (this.config.systemLog.logRotate === undefined) {
             this.config.systemLog.logRotate = "reopen";
         }
-        if (this.config.systemLog.verbosity === undefined) {
+        if (this.config.systemLog.verbosity === undefined && this.server) {
             this.config.systemLog.verbosity = this.server.production ? 0 : 1;
         }
 
         if (this.config.storage === undefined) { this.config.storage = {}; }
 
-        const db_path = this.source.join("db");
-        this.config.storage.dbPath = db_path.str()
-        if (!db_path.exists()) {
-            db_path.mkdir_sync();
-        }
+        if (this.source) {
+            const db_path = this.source.join("db");
+            this.config.storage.dbPath = db_path.str()
+            if (!db_path.exists()) {
+                db_path.mkdir_sync();
+            }
 
-        if (this.config.processManagement === undefined) { this.config.processManagement = {}; }
-        this.config.processManagement.pidFilePath = this.source.join("mongod.pid").str()
+            if (this.config.processManagement === undefined) { this.config.processManagement = {}; }
+            this.config.processManagement.pidFilePath = this.source.join("mongod.pid").str()
+        }
 
         if (this.config.net === undefined) { this.config.net = {}; }
         if (this.config.net.port === undefined) { this.config.net.port = 27017; }
         if (this.config.net.bindIp === undefined) { this.config.net.bindIp = "127.0.0.1"; }
 
         // Mode 2: Start database.
-        if (this.server.is_primary && this.uri == null) {
+        if (this.server?.is_primary && this.uri == null) {
             // Create the database.
-            if (!this.source.exists()) {
-                this.source.mkdir_sync();
+            if (!this.source!.exists()) {
+                this.source!.mkdir_sync();
             }
 
             // Set the uri.
@@ -2066,7 +2037,7 @@ class Database {
             }
 
             // Save the config.
-            const config_path = this.source.join("mongod.json");
+            const config_path = this.source!.join("mongod.json");
             config_path.save_sync(JSON.stringify(this.config));
 
             // Start the database.
@@ -2092,45 +2063,74 @@ class Database {
         }
 
         // Assign URI.
-        else if (!this.server.is_primary && this.uri == null) {
+        else if (this.server && !this.server.is_primary && this.uri == null) {
             this.uri = `mongodb://${this.config.net.bindIp}:${this.config.net.port}/main`
         }
 
         // Initialize client.
         if (this.uri) {
-            this.client = new MongoClient(this.uri, this.client_opts || {});
+            const opts = this.client_opts ?? {};
+            opts.serverApi ??= {
+                version: ServerApiVersion.v1,
+                strict: true,
+                deprecationErrors: true,
+            };
+            this.client = new MongoClient(this.uri, opts);
         }
 
         // Connect.
         await this.connect();
 
+        // Batch.
+        const multiprocessing = true;
+        const promises: Promise<any>[] = [];
+
         // Create collections.
-        this._collections.forEach((info) => {
-            if (this[info.name as keyof this] !== undefined) {
-                throw Error(`Unable to initialize database collection "${info.name}", this attribute name is already used.`);
-            }
-            if (Array.isArray(info.indexes)) {
-                for (const index of info.indexes) {
-                    if (index != null && typeof index === "object") {
-                        (index as any).forced = true;
+        for (const info of this._collections) {
+            const promise = new Promise<void>(resolve => {
+                if (this[info.name as keyof this] !== undefined) {
+                    throw Error(`Unable to initialize database collection "${info.name}", this attribute name is reserved for a system value.`);
+                }
+                if (Array.isArray(info.indexes)) {
+                    for (let index = 0; index < info.indexes.length; index++) {
+                        let item = info.indexes[index];
+                        if (typeof item === "string") {
+                            info.indexes[index] = item = { keys: [item] } as IndexOptions
+                        }
                     }
                 }
-            }
-            this[info.name] = this.create_collection(info);
-        })
-        this._uid_collections.forEach((info) => {
-            if (this[info.name as keyof this] !== undefined) {
-                throw Error(`Unable to initialize database collection "${info.name}", this attribute name is already used.`);
-            }
-            if (Array.isArray(info.indexes)) {
-                for (const index of info.indexes) {
-                    if (index != null && typeof index === "object") {
-                        (index as any).forced = true;
+                this.create_collection(info as any).then(data => {
+                    this[info.name] = data;
+                    resolve();
+                })
+            });
+            if (multiprocessing) { promises.push(promise) }
+            else { await promise }
+        }
+        for (const info of this._uid_collections) {
+            const promise = new Promise<void>(resolve => {
+                if (this[info.name as keyof this] !== undefined) {
+                    throw Error(`Unable to initialize database collection "${info.name}", this attribute name is reserved for a system value.`);
+                }
+                if (Array.isArray(info.indexes)) {
+                    for (let index = 0; index < info.indexes.length; index++) {
+                        let item = info.indexes[index];
+                        if (typeof item === "string") {
+                            info.indexes[index] = item = {
+                                key: item,
+                            } as IndexOptions
+                        }
                     }
                 }
-            }
-            this[info.name] = this.create_uid_collection(info);
-        })
+                this.create_uid_collection(info as any).then(data => {
+                    this[info.name] = data;
+                    resolve();
+                })
+            });
+            if (multiprocessing) { promises.push(promise) }
+            else { await promise }
+        }
+        if (multiprocessing) { await Promise.all(promises); }
     }
 
     // Close.
@@ -2143,15 +2143,11 @@ class Database {
      *  @title: Create Collection
      *  @description: Create a database collection.
      */
-    create_collection(info: {
+    async create_collection(info: {
         name: string,
-        indexes?: Array<{
-            keys: string[] | string | Record<string, any>,
-            options?: Record<string, any>,
-            commit_quorum?: any
-        }>,
+        indexes?: IndexOptions[],
         ttl?: number | null
-    } | string): Collection {
+    } | string): Promise<Collection> {
         // Set name by single string argument.
         let name: string;
         let indexes: Array<any> = [];
@@ -2169,6 +2165,20 @@ class Database {
         if (name in this.collections) {
             throw Error(`Collection "${name}" is already initialized.`)
         }
+
+        // Logs.
+        logger.log(1, log_source, `Initializing collection "${name}".`);
+
+        // Check if the collection exists
+        if (this._listed_cols == null) {
+            this._listed_cols = await this.db.listCollections().toArray();
+        }
+        if (!this._listed_cols.find(x => x.name === name)) {
+            logger.log(0, log_source, `Creating collection "${name}".`);
+            await this.db.createCollection(name);
+        }
+        
+        // Create collection.
         const col = new Collection(
             name,
             this.db.collection(name),
@@ -2183,15 +2193,11 @@ class Database {
      *  @title: Create UID Based Collection
      *  @description: Create a UID based database collection.
      */
-    create_uid_collection(info: {
+    async create_uid_collection(info: {
         name: string,
-        indexes?: Array<{
-            keys: string[] | string | Record<string, any>,
-            options?: Record<string, any>,
-            commit_quorum?: any
-        }>,
+        indexes?: IndexOptions[],
         ttl?: number | null
-    } | string): UIDCollection {
+    } | string): Promise<UIDCollection> {
         // Set name by single string argument.
         let name: string;
         let indexes: Array<any> = [];
@@ -2205,10 +2211,22 @@ class Database {
             ttl = info.ttl || null;
         }
 
-        // Check collection.
+        // Check duplicate collection.
         if (name in this.collections) {
             throw Error(`Collection "${name}" is already initialized.`)
+        }  
+
+        // Logs.
+        logger.log(1, log_source, `Initializing uid collection "${name}".`);
+
+        // Check if the collection exists
+        const collections = await this.db.listCollections({ name: name }).toArray();
+        if (collections.length === 0) {
+            logger.log(0, log_source, `Creating uid collection "${name}".`);
+            await this.db.createCollection(name);
         }
+
+        // Collection.
         const col = new UIDCollection(
             name,
             this.db.collection(name),