@vandenberghinc/volt 1.1.5 → 1.1.7

package/backend/dist/esm-dev/database.js

@@ -8,7 +8,7 @@ import { spawn } from "child_process";
 import { deserialize, serialize } from "bson";
 import { MongoClient, ServerApiVersion } from 'mongodb';
 import { logger, LogSource } from "./logger.js";
-import { vlib } from "/Users/administrator/persistance/private/dev/vinc/volt/backend/./src/vinc.dev.js";
+import { vlib } from "@vinc";
 const log_source = new LogSource("Database");
 // ---------------------------------------------------------
 // Collection.
@@ -31,7 +31,7 @@ class Collection {
     static constructor_scheme = {
         name: "string",
         uid_based: "boolean",
-        ttl: { type: "number", default: null },
+        ttl: { type: "number", default: null }, // ttl in msec
         indexes: {
             type: "array",
             default: [],
@@ -40,7 +40,7 @@ class Collection {
                 scheme: {
                     key: { type: "string", required: (data) => data.key == null && data.keys == null },
                     keys: {
-                        type: ["string", "array"],
+                        type: ["array"],
                         required: (data) => data.key == null && data.keys == null, value_scheme: "string",
                         postprocess: (keys) => typeof keys === "string" ? [keys] : keys,
                     },
@@ -51,11 +51,6 @@ class Collection {
                 postprocess: (info) => {
                     if (typeof info === "string")
                         return { keys: [info] };
-                    if (typeof info === "object" && info.key) {
-                        info.keys = info.key;
-                        delete info.key;
-                        return info;
-                    }
                     return info;
                 },
             },
@@ -65,9 +60,10 @@ class Collection {
     col;
     name;
     uid_based;
-    ttl;
+    ttl; // ttl in msec
     ttl_enabled;
-    constructor(name, collection, ttl = null, indexes = [], uid_based = false) {
+    constructor(name, collection, ttl = null, // ttl in msec
+    indexes = [], uid_based = false) {
         // Verify scheme.
         ({ indexes, ttl } = vlib.Scheme.verify({
             object: {
@@ -107,13 +103,8 @@ class Collection {
         }
         // Create indexes.
         if (Array.isArray(indexes) && indexes.length > 0) {
-            for (const index of indexes) {
-                this.create_index({
-                    keys: index.keys,
-                    options: index.options,
-                    commit_quorum: index.commit_quorum,
-                    forced: index.forced,
-                });
+            for (let item of indexes) {
+                this.create_index(item);
             }
         }
     }
@@ -228,23 +219,22 @@ class Collection {
             @name: commitQuorum
             @desc: The `commitQuorum` argument for the orignal mongodb `createIndex()` function.
      */
-    async create_index({ keys, options = null, commit_quorum = null, forced = false }) {
+    async create_index({ key = undefined, keys = undefined, // following the IndexOptions "key" or "keys" must be defined in typescript but not in raw js.
+    options = undefined, commit_quorum = undefined, forced = false }) {
+        // Create keys objs per input type.
         let keys_obj = {};
-        if (typeof keys === "string") {
+        if (key) {
             keys_obj = {};
-            keys_obj[keys] = 1;
+            keys_obj[key] = 1;
         }
-        else if (Array.isArray(keys)) {
+        else if (Array.isArray(keys) && keys.length > 0) {
             keys_obj = {};
             for (const key of keys) {
                 keys_obj[key] = 1;
             }
         }
-        else if (keys == null || typeof keys !== "object") {
-            vlib.Scheme.throw_invalid_type("keys", keys, ["string", "string[]", "object"], true);
-        }
         else {
-            keys_obj = keys;
+            throw new Error("Define one of the following parameters: [key, keys].");
         }
         // Drop index.
         if (forced) {
@@ -776,8 +766,8 @@ class UIDCollection {
             @name: commitQuorum
             @desc: The `commitQuorum` argument for the orignal mongodb `createIndex()` function.
      */
-    async create_index({ keys, options = null, commit_quorum = null }) {
-        return this._col.create_index({ keys, options, commit_quorum });
+    async create_index(args) {
+        return this._col.create_index(args);
     }
     /*  @docs:
      *  @title: Find
@@ -1165,11 +1155,11 @@ class Database {
         preview: { type: "boolean", default: true },
         preview_ip_whitelist: { type: "array", default: [] },
         daemon: { type: ["object", "boolean"], default: {} },
-        _server: "object",
+        _server: { type: ["object", "undefined"] },
     };
     uri;
-    preview;
-    preview_ip_whitelist;
+    // preview: boolean;
+    // preview_ip_whitelist: string[];
     client_opts;
     config;
     source; // Using vlib.Path type
@@ -1182,9 +1172,14 @@ class Database {
     proc;
     daemon;
     db;
-    constructor({ uri = null, source = null, config = null, start_args = [], client = null, collections = [], uid_collections = [], preview = true, preview_ip_whitelist = [], daemon = {}, _server, }) {
+    // System.
+    _listed_cols;
+    constructor({ uri = null, source = null, config = null, start_args = [], client = null, collections = [], uid_collections = [], 
+    // preview = true,
+    // preview_ip_whitelist = [],
+    daemon = {}, _server = undefined, }) {
         // Checks.
-        if (_server.is_primary && uri == null) {
+        if (!_server || (_server.is_primary && uri == null)) {
             ({ uri, config, start_args, config, client } = vlib.Scheme.verify({
                 object: arguments[0],
                 check_unknown: true,
@@ -1193,24 +1188,27 @@ class Database {
         }
         // Arguments.
         this.uri = uri;
-        this.preview = preview;
-        this.preview_ip_whitelist = preview_ip_whitelist;
+        // this.preview = preview;
+        // this.preview_ip_whitelist = preview_ip_whitelist;
         this.client_opts = client;
         this.config = config || {};
-        this.source = source != null ? new vlib.Path(source) : _server.source.join(".db");
+        this.source = source != null ? new vlib.Path(source) : _server?.source.join(".db");
         this.start_args = start_args;
-        this._collections = collections;
-        this._uid_collections = uid_collections;
+        this._collections = collections ?? [];
+        this._uid_collections = uid_collections ?? [];
         this.server = _server;
         // Attributes.
         this.client = null;
         this.collections = {};
         // Initialize the service daemon.
-        if (this.server.daemon && daemon !== false) {
+        if (this.server?.daemon && daemon !== false) {
             const log_source = this.server.source.join(".logs");
             if (!log_source.exists()) {
                 log_source.mkdir_sync();
             }
+            if (!this.server) {
+                throw new Error("Parameter 'Database._server' must be defined for this behaviour.");
+            }
             this.daemon = new vlib.Daemon({
                 name: this.server.daemon.name + ".mongodb",
                 user: daemon.user || this.server.daemon.user,
@@ -1226,9 +1224,9 @@ class Database {
             });
         }
     }
-    // Database preview.
-    _initialize_db_preview() {
-        /*
+    /* Database preview.
+    _initialize_db_preview(): void {
+        
         if (this.preview && this.server.production === false) {
             this.server.endpoint(
 
@@ -1753,8 +1751,7 @@ class Database {
                 },
             )
         }
-        */
-    }
+    }*/
     // Connect.
     async connect() {
         try {
@@ -1774,7 +1771,7 @@ class Database {
         if (this.config.systemLog === undefined) {
             this.config.systemLog = {};
         }
-        this.config.systemLog.path = this.source.join("mongod.log").str();
+        this.config.systemLog.path = this.source?.join("mongod.log").str();
         if (this.config.systemLog.destination === undefined) {
             this.config.systemLog.destination = "file";
         }
@@ -1784,21 +1781,23 @@ class Database {
         if (this.config.systemLog.logRotate === undefined) {
             this.config.systemLog.logRotate = "reopen";
         }
-        if (this.config.systemLog.verbosity === undefined) {
+        if (this.config.systemLog.verbosity === undefined && this.server) {
             this.config.systemLog.verbosity = this.server.production ? 0 : 1;
         }
         if (this.config.storage === undefined) {
             this.config.storage = {};
         }
-        const db_path = this.source.join("db");
-        this.config.storage.dbPath = db_path.str();
-        if (!db_path.exists()) {
-            db_path.mkdir_sync();
-        }
-        if (this.config.processManagement === undefined) {
-            this.config.processManagement = {};
+        if (this.source) {
+            const db_path = this.source.join("db");
+            this.config.storage.dbPath = db_path.str();
+            if (!db_path.exists()) {
+                db_path.mkdir_sync();
+            }
+            if (this.config.processManagement === undefined) {
+                this.config.processManagement = {};
+            }
+            this.config.processManagement.pidFilePath = this.source.join("mongod.pid").str();
         }
-        this.config.processManagement.pidFilePath = this.source.join("mongod.pid").str();
         if (this.config.net === undefined) {
             this.config.net = {};
         }
@@ -1809,7 +1808,7 @@ class Database {
             this.config.net.bindIp = "127.0.0.1";
         }
         // Mode 2: Start database.
-        if (this.server.is_primary && this.uri == null) {
+        if (this.server?.is_primary && this.uri == null) {
             // Create the database.
             if (!this.source.exists()) {
                 this.source.mkdir_sync();
@@ -1839,7 +1838,7 @@ class Database {
             });
         }
         // Assign URI.
-        else if (!this.server.is_primary && this.uri == null) {
+        else if (this.server && !this.server.is_primary && this.uri == null) {
             this.uri = `mongodb://${this.config.net.bindIp}:${this.config.net.port}/main`;
         }
         // Initialize client.
@@ -1854,33 +1853,65 @@ class Database {
         }
         // Connect.
         await this.connect();
+        // Batch.
+        const multiprocessing = true;
+        const promises = [];
         // Create collections.
-        this._collections.forEach((info) => {
-            if (this[info.name] !== undefined) {
-                throw Error(`Unable to initialize database collection "${info.name}", this attribute name is already used.`);
-            }
-            if (Array.isArray(info.indexes)) {
-                for (const index of info.indexes) {
-                    if (index != null && typeof index === "object") {
-                        index.forced = true;
+        for (const info of this._collections) {
+            const promise = new Promise(resolve => {
+                if (this[info.name] !== undefined) {
+                    throw Error(`Unable to initialize database collection "${info.name}", this attribute name is reserved for a system value.`);
+                }
+                if (Array.isArray(info.indexes)) {
+                    for (let index = 0; index < info.indexes.length; index++) {
+                        let item = info.indexes[index];
+                        if (typeof item === "string") {
+                            info.indexes[index] = item = { keys: [item] };
+                        }
                     }
                 }
+                this.create_collection(info).then(data => {
+                    this[info.name] = data;
+                    resolve();
+                });
+            });
+            if (multiprocessing) {
+                promises.push(promise);
             }
-            this[info.name] = this.create_collection(info);
-        });
-        this._uid_collections.forEach((info) => {
-            if (this[info.name] !== undefined) {
-                throw Error(`Unable to initialize database collection "${info.name}", this attribute name is already used.`);
+            else {
+                await promise;
             }
-            if (Array.isArray(info.indexes)) {
-                for (const index of info.indexes) {
-                    if (index != null && typeof index === "object") {
-                        index.forced = true;
+        }
+        for (const info of this._uid_collections) {
+            const promise = new Promise(resolve => {
+                if (this[info.name] !== undefined) {
+                    throw Error(`Unable to initialize database collection "${info.name}", this attribute name is reserved for a system value.`);
+                }
+                if (Array.isArray(info.indexes)) {
+                    for (let index = 0; index < info.indexes.length; index++) {
+                        let item = info.indexes[index];
+                        if (typeof item === "string") {
+                            info.indexes[index] = item = {
+                                key: item,
+                            };
+                        }
                     }
                 }
+                this.create_uid_collection(info).then(data => {
+                    this[info.name] = data;
+                    resolve();
+                });
+            });
+            if (multiprocessing) {
+                promises.push(promise);
             }
-            this[info.name] = this.create_uid_collection(info);
-        });
+            else {
+                await promise;
+            }
+        }
+        if (multiprocessing) {
+            await Promise.all(promises);
+        }
     }
     // Close.
     async close() {
@@ -1891,7 +1922,7 @@ class Database {
      *  @title: Create Collection
      *  @description: Create a database collection.
      */
-    create_collection(info) {
+    async create_collection(info) {
         // Set name by single string argument.
         let name;
         let indexes = [];
@@ -1908,6 +1939,17 @@ class Database {
         if (name in this.collections) {
             throw Error(`Collection "${name}" is already initialized.`);
         }
+        // Logs.
+        logger.log(1, log_source, `Initializing collection "${name}".`);
+        // Check if the collection exists
+        if (this._listed_cols == null) {
+            this._listed_cols = await this.db.listCollections().toArray();
+        }
+        if (!this._listed_cols.find(x => x.name === name)) {
+            logger.log(0, log_source, `Creating collection "${name}".`);
+            await this.db.createCollection(name);
+        }
+        // Create collection.
         const col = new Collection(name, this.db.collection(name), ttl, indexes);
         this.collections[name] = col;
         return col;
@@ -1916,7 +1958,7 @@ class Database {
      *  @title: Create UID Based Collection
      *  @description: Create a UID based database collection.
      */
-    create_uid_collection(info) {
+    async create_uid_collection(info) {
         // Set name by single string argument.
         let name;
         let indexes = [];
@@ -1929,10 +1971,19 @@ class Database {
             indexes = info.indexes || [];
             ttl = info.ttl || null;
         }
-        // Check collection.
+        // Check duplicate collection.
         if (name in this.collections) {
             throw Error(`Collection "${name}" is already initialized.`);
         }
+        // Logs.
+        logger.log(1, log_source, `Initializing uid collection "${name}".`);
+        // Check if the collection exists
+        const collections = await this.db.listCollections({ name: name }).toArray();
+        if (collections.length === 0) {
+            logger.log(0, log_source, `Creating uid collection "${name}".`);
+            await this.db.createCollection(name);
+        }
+        // Collection.
         const col = new UIDCollection(name, this.db.collection(name), indexes, ttl);
         this.collections[name] = col;
         return col;

package/backend/dist/esm-dev/endpoint.d.ts

@@ -1,18 +1,17 @@
 import { View } from './view.js';
 import { RateLimitGroup } from "./rate_limit.js";
 import { Stream, AuthStream, Params } from "./stream.js";
+import type { Server } from "./server.js";
 type EndpointCallback = ((stream: Stream, params: Params) => any) | ((stream: AuthStream, params: Params) => any);
-export interface EndpointOptions {
+interface BaseEndpointOptions {
     method?: string;
     endpoint: string | RegExp;
     authenticated?: boolean;
     rate_limit?: string | RateLimitGroup | RateLimitGroup[] | null;
     params?: Record<string, any> | null;
     callback?: EndpointCallback;
-    view?: View | Record<string, any> | null;
     data?: any;
-    content_type?: string;
-    compress?: boolean;
+    compress?: "auto" | boolean;
     cache?: boolean | number;
     ip_whitelist?: string[] | null;
     sitemap?: boolean | null;
@@ -21,15 +20,25 @@ export interface EndpointOptions {
     _path?: string | null;
     _is_static?: boolean;
 }
+interface EndpointOptionsWithView extends Omit<BaseEndpointOptions, "content_type" | "view"> {
+    view: View | Record<string, any>;
+    content_type?: string;
+}
+interface EndpointOptionsWithoutView extends BaseEndpointOptions {
+    view?: null | undefined;
+    content_type: string;
+}
+export type EndpointOptions = EndpointOptionsWithView | EndpointOptionsWithoutView;
 declare class Endpoint {
     static rate_limits: Map<string, any>;
+    static compressed_content_types: string[];
     method: string;
     endpoint: string | RegExp;
     authenticated: boolean;
     params: Record<string, any> | null;
     callback?: EndpointCallback;
     data: any;
-    content_type: string;
+    content_type?: string;
     compress: boolean;
     cache: boolean | number;
     sitemap: boolean | null;
@@ -46,12 +55,17 @@ declare class Endpoint {
     raw_data?: any;
     content_length?: number;
     is_image_endpoint?: boolean;
-    constructor({ method, endpoint, authenticated, rate_limit, params, callback, view, data, content_type, compress, cache, ip_whitelist, sitemap, robots, _templates, // only used in loading static files.
-    _path, _is_static, }: EndpointOptions);
+    _initialized: boolean;
+    _server: any;
+    constructor({ method, endpoint, authenticated, rate_limit, params, callback, view, data, content_type, // = "text/plain",
+    compress, cache, ip_whitelist, sitemap, robots, _templates, // only used in loading static files.
+    _path, _is_static, _server, }: EndpointOptions & {
+        _server: Server;
+    });
     _load_data_by_path(server: any): this;
     _set_headers(stream: any): void;
     _refresh(server: any): Promise<void>;
-    _initialize(server: any): Promise<void>;
-    _serve(stream: any, status_code?: number): Promise<void>;
+    _dynamic_initialize(): Promise<void>;
+    _serve(stream: Stream, status_code?: number): Promise<void>;
 }
 export { Endpoint };

package/backend/dist/esm-dev/endpoint.js

@@ -7,8 +7,8 @@
 import CleanCSS from 'clean-css';
 import zlib from 'zlib';
 import { View } from './view.js';
-import { vlib } from "/Users/administrator/persistance/private/dev/vinc/volt/backend/./src/vinc.dev.js";
-import { Utils, FrontendError } from "./utils.js";
+import { vlib } from "@vinc";
+import { Utils, APIError } from "./utils.js";
 import { Status } from "./status.js";
 import { logger, LogSource } from "./logger.js";
 import { RateLimits } from "./rate_limit.js";
@@ -117,6 +117,57 @@ const log_source = new LogSource("Endpoint");
 class Endpoint {
     // Static attributes.
     static rate_limits = new Map();
+    static compressed_content_types = [
+        // Image formats (often already compressed)
+        "image/jpeg",
+        "image/png",
+        "image/gif",
+        "image/webp",
+        "image/bmp",
+        "image/tiff",
+        "image/vnd.microsoft.icon", // ICO
+        // Audio formats (usually compressed)
+        "audio/mpeg", // MP3
+        "audio/mp3",
+        "audio/ogg",
+        "audio/wav",
+        "audio/x-wav",
+        "audio/flac",
+        "audio/aac",
+        "audio/midi",
+        // Video formats (typically compressed)
+        "video/mp4",
+        "video/mpeg",
+        "video/ogg",
+        "video/webm",
+        "video/x-msvideo", // AVI
+        "video/quicktime", // MOV
+        // Archive / Compressed file formats
+        "application/zip",
+        "application/x-7z-compressed",
+        "application/x-rar-compressed",
+        "application/x-tar",
+        "application/gzip",
+        "application/x-gzip",
+        "application/x-bzip",
+        "application/x-bzip2",
+        "application/x-xz",
+        // Documents that are usually compressed internally
+        "application/pdf",
+        "application/vnd.ms-powerpoint",
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        "application/vnd.ms-excel",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "application/msword",
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        // Font files
+        "font/woff",
+        "font/woff2",
+        "application/font-sfnt",
+        "application/vnd.ms-fontobject",
+        // Other binary data
+        "application/octet-stream",
+    ];
     // Instance attributes
     method;
     endpoint;
@@ -141,8 +192,11 @@ class Endpoint {
     raw_data;
     content_length;
     is_image_endpoint;
-    constructor({ method = "GET", endpoint = "/", authenticated = false, rate_limit = null, params = null, callback = undefined, view = null, data = null, content_type = "text/plain", compress = true, cache = true, ip_whitelist = null, sitemap = null, robots = null, _templates = {}, // only used in loading static files.
-    _path = null, _is_static = false, }) {
+    _initialized = false;
+    _server;
+    constructor({ method = "GET", endpoint = "/", authenticated = false, rate_limit = null, params = null, callback = undefined, view = null, data = null, content_type, // = "text/plain",
+    compress = "auto", cache = true, ip_whitelist = null, sitemap = null, robots = null, _templates = {}, // only used in loading static files.
+    _path = null, _is_static = false, _server, }) {
         // Attributes.
         this.method = method;
         this.endpoint = endpoint;
@@ -153,7 +207,6 @@ class Endpoint {
         }
         this.data = data;
         this.content_type = content_type;
-        this.compress = compress !== false;
         this.cache = cache;
         this.sitemap = sitemap;
         this.robots = robots;
@@ -174,6 +227,19 @@ class Endpoint {
         if (typeof this.endpoint === "string") {
             this.endpoint = Utils.clean_endpoint(this.endpoint);
         }
+        // Set compress.
+        if (compress === "auto" || typeof compress !== "boolean") {
+            compress = Endpoint.compressed_content_types.includes(this.content_type ?? "");
+        }
+        else if (compress === true && this.content_type != null && Endpoint.compressed_content_types.includes(this.content_type)) {
+            logger.warn(1, log_source, `Overriding parameter "compress", disabling compression of endpoint ${this.endpoint}.`);
+            compress = false;
+        }
+        this.compress = compress;
+        this.compress = false;
+        if (this.compress) {
+            console.log(">>> compress", { endpoint: this.endpoint, content_type: this.content_type });
+        }
         // Argument `view` may also be passed as an object instead of class View.
         if (view == null) {
             this.view = null;
@@ -215,6 +281,12 @@ class Endpoint {
         }
         // The endpoint parent for params verification.
         this._verify_params_parent = this.method + ":" + this.endpoint + ":";
+        // Initialize.
+        this._server = _server;
+        // Initialize html.
+        if (this.view != null) {
+            this.view._initialize(_server, this);
+        }
     }
     // Load data by path.
     _load_data_by_path(server) {
@@ -279,28 +351,33 @@ class Endpoint {
         }
     }
     // Initialize.
-    async _initialize(server) {
+    async _dynamic_initialize() {
+        if (!this._server) {
+            throw new Error(`Endpoint "${this.method}:${this.endpoint}" is not initialized by the server yet.`);
+        }
         // Build html code of view.
         if (this.view != null) {
-            this.view._initialize(server, this);
             await this.view._build_html();
         }
         // Compression enabled.
-        if (server.production && this.callback == null && this.compress) {
+        if (this._server.production && this.callback == null && this.compress) {
             this._is_compressed = true;
             if (this.data != null) {
                 this.raw_data = this.data;
                 this.data = zlib.gzipSync(this.data, { level: zlib.constants.Z_BEST_COMPRESSION });
+                this.content_length = this.data.length;
             }
             else if (this.view != null) {
                 this.view.raw_html = this.view.html;
                 this.view.html = zlib.gzipSync(this.view.html, { level: zlib.constants.Z_BEST_COMPRESSION });
+                this.content_length = this.view.html.length;
             }
+            console.log(">>> Compressed - content_length:", this.content_length);
         }
         // Set cache headers.
-        if (!server.production) {
-            this.cache = false;
-        }
+        // if (!this._server.production) {
+        this.cache = false; // @todo @tmp
+        // }
         if ((this.callback == null || this.is_image_endpoint) && (typeof this.cache === "number" || this.cache === true)) {
             if (this.cache === 1 || this.cache === true) {
                 this.headers.push(["Cache-Control", "max-age=86400"]);
@@ -322,9 +399,16 @@ class Endpoint {
         if (this.content_type != null) {
             this.headers.push(["Content-Type", this.content_type]);
         }
+        if (this._is_compressed) {
+            console.log("Compressed headers:", this.headers);
+        }
+        this._initialized = true;
     }
     // Serve a client.
     async _serve(stream, status_code = 200) {
+        if (!this._initialized) {
+            await this._dynamic_initialize();
+        }
         try {
             // Check IP whitelist.
             if (this.ip_whitelist && !this.ip_whitelist.includes(stream.ip)) {
@@ -371,22 +455,15 @@ class Endpoint {
                     }
                 }
                 catch (err) {
-                    if (err instanceof FrontendError) {
-                        let data = { error: err.message != null ? err.message : "Internal Server Error" };
-                        if (err.data != null && typeof err.data === "object") {
-                            Object.assign(data, err.data);
-                        }
-                        stream.send({
-                            status: err.status != null ? err.status : Status.internal_server_error,
-                            headers: { "Content-Type": "application/json" },
-                            data,
-                        });
+                    if (err instanceof APIError) {
+                        err.serve(stream);
                     }
                     else {
-                        stream.send({
+                        stream.error({
                             status: Status.internal_server_error,
                             headers: { "Content-Type": "application/json" },
-                            data: { error: "Internal Server Error" },
+                            message: "Internal Server Error",
+                            type: "InternalServerError",
                         });
                     }
                     logger.error(log_source, `${this.method}:${this.endpoint}: `, err); // after sending the response since this edits the error.