nesoi 3.0.0

package/lib/adapters/postgres/src/migrator/database.d.ts

@@ -0,0 +1,37 @@
+import postgres from 'postgres';
+export type TableColumn = {
+    column_name: string;
+    data_type: string;
+    nullable: boolean;
+    field_exists: boolean;
+};
+export declare class Database {
+    static connect(config?: postgres.Options<any>): postgres.Sql<{}>;
+    /**
+     * Check if the connection to PostgreSQL is working, by performing
+     * a SELECT on the pg_database table.
+     * @param config
+     */
+    static checkConnection(sql: postgres.Sql<any>): Promise<any>;
+    /**
+     * List all tables of the database.
+     * @param config
+     */
+    static listTables(sql: postgres.Sql<any>): Promise<any[]>;
+    /**
+     * Read schema of table.
+     * @param config
+     */
+    private getSchema;
+    /**
+     * Connect to PostgreSQL and create a database.
+     *
+     * The `if_exists` flag controls what happens if the database already exists
+     * - fail: Throw an exception
+     * - keep: Do nothing
+     * - delete: *DROP DATABASE*
+     */
+    static createDatabase(name: string, config?: postgres.Options<any>, $?: {
+        if_exists: 'fail' | 'keep' | 'delete';
+    }): Promise<void>;
+}

package/lib/adapters/postgres/src/migrator/database.js

@@ -0,0 +1,88 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Database = void 0;
+const postgres_1 = __importDefault(require("postgres"));
+const log_1 = require("../../../../engine/util/log");
+class Database {
+    static connect(config) {
+        return (0, postgres_1.default)(config);
+    }
+    /**
+     * Check if the connection to PostgreSQL is working, by performing
+     * a SELECT on the pg_database table.
+     * @param config
+     */
+    static async checkConnection(sql) {
+        try {
+            await sql `SELECT datname FROM pg_database`;
+        }
+        catch (e) {
+            return e;
+        }
+        return true;
+    }
+    /**
+     * List all tables of the database.
+     * @param config
+     */
+    static async listTables(sql) {
+        const columns = await sql `
+            SELECT table_name
+            FROM information_schema.tables
+            WHERE table_schema='public'
+            AND table_type='BASE TABLE'
+        `;
+        return columns.map(col => col.table_name);
+    }
+    /**
+     * Read schema of table.
+     * @param config
+     */
+    async getSchema(sql, tableName) {
+        const columns = await sql `
+            SELECT column_name, data_type, is_nullable 
+            FROM information_schema.columns 
+            WHERE table_name = ${tableName}`;
+        if (!columns.length) {
+            return;
+        }
+        return columns.map(col => ({
+            ...col,
+            nullable: col.is_nullable === 'YES',
+            field_exists: false
+        }));
+    }
+    /**
+     * Connect to PostgreSQL and create a database.
+     *
+     * The `if_exists` flag controls what happens if the database already exists
+     * - fail: Throw an exception
+     * - keep: Do nothing
+     * - delete: *DROP DATABASE*
+     */
+    static async createDatabase(name, config, $) {
+        const sql = (0, postgres_1.default)(Object.assign({}, config, {
+            db: 'postgres'
+        }));
+        const dbs = await sql `SELECT datname FROM pg_database`;
+        const alreadyExists = dbs.some(db => db.datname === name);
+        if (alreadyExists) {
+            if (!$ || $.if_exists === 'fail') {
+                throw new Error(`Database ${name} already exists`);
+            }
+            if ($.if_exists === 'keep') {
+                return;
+            }
+            if ($.if_exists === 'delete') {
+                log_1.Log.warn('migrator', 'create_db', `Database '${name}' is being dropped due to a if_exists:'delete' flag.`);
+                await sql `DROP DATABASE ${sql(name)}`;
+            }
+        }
+        log_1.Log.info('migrator', 'create_db', `Creating database '${name}'`);
+        await sql `CREATE DATABASE ${sql(name)}`;
+    }
+}
+exports.Database = Database;

package/lib/adapters/postgres/src/migrator/index.d.ts

@@ -0,0 +1 @@
+export { Migrator } from './migrator';

package/lib/adapters/postgres/src/migrator/index.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Migrator = void 0;
+var migrator_1 = require("./migrator");
+Object.defineProperty(exports, "Migrator", { enumerable: true, get: function () { return migrator_1.Migrator; } });

package/lib/adapters/postgres/src/migrator/migration.d.ts

@@ -0,0 +1,73 @@
+import { $Bucket, $Space } from "../../../../elements";
+import postgres from 'postgres';
+import { AnyDaemon } from "../../../../engine/daemon";
+export type MigrationFieldOperation = {
+    create: {
+        type: string;
+        nullable?: boolean;
+        pk?: boolean;
+        fk?: string;
+    };
+} | {
+    alter: {
+        name?: string;
+        type?: string;
+        nullable?: boolean;
+        fk?: string;
+    };
+} | {
+    drop: {
+        cascade?: boolean;
+    };
+};
+export declare class MigrationField {
+    oldColumn: string | undefined;
+    newColumn: string;
+    operation: MigrationFieldOperation;
+    constructor(oldColumn: string | undefined, newColumn: string, operation: MigrationFieldOperation);
+    describe(): string;
+    sql(): string | undefined;
+}
+export declare class Migration {
+    private type;
+    private tableName;
+    private options;
+    name: string;
+    private fields;
+    private needsReview;
+    constructor(type: 'create' | 'alter', tableName: string, options: Record<string, MigrationField[]>);
+    describe(): string;
+    sqlUp(): string;
+    sqlDown(): string;
+    save(dirpath?: string): void;
+    hash(): string;
+    /**
+     * Generates an encoded version of the migration, which is
+     * stored as a comment on the migration file. This can be
+     * used to recreate the `Migration` object in memory if needed.
+     * > **Attention** This encoded version is NOT used to run the migration.
+     * > The SQL is used to run the migration. This can be used
+     * > for analysis/display purposes only.
+     * > There's no guarantee that this will match the SQL if
+     * > the migration file is manually altered.
+     * @returns
+     */
+    private encode;
+    private decode;
+    private fnUp;
+    private fnDown;
+}
+export declare class BucketMigrator<S extends $Space, D extends AnyDaemon, ModuleName extends NoInfer<keyof S['modules']>> {
+    private daemon;
+    private sql;
+    private module;
+    private bucket;
+    private tableName;
+    protected schema: $Bucket;
+    constructor(daemon: D, sql: postgres.Sql<any>, module: ModuleName, bucket: NoInfer<keyof S['modules'][ModuleName]['buckets']>, tableName: string);
+    generate(): Promise<Migration | undefined>;
+    private getCurrentSchema;
+    private generateFieldOptions;
+    private generateField;
+    private fieldType;
+}

package/lib/adapters/postgres/src/migrator/migration.js

@@ -0,0 +1,351 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BucketMigrator = exports.Migration = exports.MigrationField = void 0;
+const string_1 = require("../../../../engine/util/string");
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+const crypto_1 = require("crypto");
+const datetime_1 = require("../../../../engine/data/datetime");
+const daemon_1 = require("../../../../engine/daemon");
+class MigrationField {
+    constructor(oldColumn, newColumn, operation) {
+        this.oldColumn = oldColumn;
+        this.newColumn = newColumn;
+        this.operation = operation;
+    }
+    describe() {
+        const col_str = (0, string_1.colored)(this.newColumn, 'lightblue');
+        if ('create' in this.operation) {
+            const type_str = (0, string_1.colored)(this.operation.create.type, 'purple');
+            return `Create column ${col_str} as ${type_str}`;
+        }
+        else if ('alter' in this.operation) {
+            const op = this.operation.alter;
+            const name_str = op.name ? (0, string_1.colored)(op.name, 'lightcyan') : undefined;
+            const type_str = op.type ? (0, string_1.colored)(op.type, 'purple') : undefined;
+            const null_str = op.nullable ? (0, string_1.colored)('NOT NULL', 'purple') : undefined;
+            const props = (type_str ? `Alter column ${col_str} type to ${type_str};` : '') +
+                (null_str ? `Alter column ${col_str} nullable to ${null_str};` : '');
+            if (name_str) {
+                return `Rename column ${col_str} to ${name_str}.${props}`;
+            }
+            return props;
+        }
+        else {
+            return (0, string_1.colored)('Unknown', 'lightred');
+        }
+    }
+    sql() {
+        if ('create' in this.operation) {
+            return `${this.newColumn} ${this.operation.create.type}`;
+        }
+    }
+}
+exports.MigrationField = MigrationField;
+class Migration {
+    constructor(type, tableName, options) {
+        this.type = type;
+        this.tableName = tableName;
+        this.options = options;
+        this.name = `${datetime_1.NesoiDatetime.now().epoch}_${this.tableName}`;
+        this.needsReview = Object.values(options).some(opts => opts.length > 1);
+        this.fields = Object.values(options).map(opts => opts[0]);
+    }
+    describe() {
+        let str = '';
+        str += '┌\n';
+        str += `│ ${(0, string_1.colored)(this.name, 'lightcyan')}\n`;
+        str += `│ ${this.needsReview
+            ? (0, string_1.colored)('⚠ More than one option for some fields. Requires review.', 'red')
+            : (0, string_1.colored)('✓ Only one option for each field.', 'lightgreen')}\n`;
+        str += '└\n\n';
+        if (this.type === 'create') {
+            str += `◆ Create table ${(0, string_1.colored)(this.tableName, 'lightblue')}\n`;
+        }
+        else if (this.type === 'alter') {
+            str += `◆ Alter table '${this.tableName}'\n`;
+        }
+        this.fields.forEach(field => {
+            str += `└ ${field.describe()}\n`;
+        });
+        str += '\n';
+        str += `${(0, string_1.colored)('▲ UP', 'lightgreen')}:\n`;
+        str += this.sqlUp();
+        str += '\n';
+        str += `${(0, string_1.colored)('▼ DOWN', 'yellow')}:\n`;
+        str += this.sqlDown();
+        return str;
+    }
+    sqlUp() {
+        if (this.type === 'create') {
+            return `CREATE TABLE ${this.tableName} (\n` +
+                this.fields.map(field => '\t' + field.sql()).join(',\n')
+                + '\n)';
+        }
+        return '';
+    }
+    sqlDown() {
+        if (this.type === 'create') {
+            return `DROP TABLE ${this.tableName}`;
+        }
+        return '';
+    }
+    save(dirpath = './migrations') {
+        const filepath = path.join(dirpath, this.name + '.ts');
+        const { encoded, hash } = this.encode();
+        let str = '';
+        str += 'import { Migrator } from \'nesoi/lib/adapters/postgres/src/migrator\';\n';
+        str += '\n';
+        str += '/**\n';
+        str += ` * ${this.name}\n`;
+        str += ` * $hash[${hash}]\n`;
+        str += ' *\n';
+        str += ` * $type[${this.type}]\n`;
+        str += ` * $table[${this.tableName}]\n`;
+        str += ` * $fields[${encoded}]\n`;
+        str += ' */\n';
+        str += '\n';
+        str += 'export default Migrator.migration({\n';
+        str += '\t' + this.fnUp().replace(/\n/g, '\n\t') + ',\n';
+        str += '\t' + this.fnDown().replace(/\n/g, '\n\t') + '\n';
+        str += '})';
+        fs.writeFileSync(filepath, str);
+    }
+    hash() {
+        const { hash } = this.encode();
+        return hash;
+    }
+    /**
+     * Generates an encoded version of the migration, which is
+     * stored as a comment on the migration file. This can be
+     * used to recreate the `Migration` object in memory if needed.
+     * > **Attention** This encoded version is NOT used to run the migration.
+     * > The SQL is used to run the migration. This can be used
+     * > for analysis/display purposes only.
+     * > There's no guarantee that this will match the SQL if
+     * > the migration file is manually altered.
+     * @returns
+     */
+    encode() {
+        const v0 = () => {
+            const f_str = this.fields.map(field => {
+                const [op_key, op] = Object.entries(field.operation)[0];
+                const op_str = `[${op_key};${op.type || ''};${op.nullable ? 1 : 0};${op.pk ? 1 : 0};${op.fk || ''};${op.cascade || ''}]`;
+                return `${field.oldColumn || ''}:${field.newColumn}:${op_str}`;
+            }).join('\n');
+            const encoded = btoa(f_str);
+            const hash = (0, crypto_1.createHash)('md5').update(encoded).digest('hex');
+            return {
+                encoded,
+                hash: 'v0.' + hash
+            };
+        };
+        return v0();
+    }
+    decode() {
+        // TODO
+    }
+    fnUp() {
+        let str = '';
+        str += 'up: async (daemon, sql) => {\n';
+        str += '\tawait sql`\n';
+        str += '\t\t' + this.sqlUp().replace(/\n/g, '\n\t\t') + '\n';
+        str += '\t`\n';
+        str += '}';
+        return str;
+    }
+    fnDown() {
+        let str = '';
+        str += 'down: async (daemon, sql) => {\n';
+        str += '\tawait sql`\n';
+        str += '\t\t' + this.sqlDown().replace(/\n/g, '\n\t\t') + '\n';
+        str += '\t`\n';
+        str += '}';
+        return str;
+    }
+}
+exports.Migration = Migration;
+class BucketMigrator {
+    constructor(daemon, sql, module, bucket, tableName) {
+        this.daemon = daemon;
+        this.sql = sql;
+        this.module = module;
+        this.bucket = bucket;
+        this.tableName = tableName;
+        this.schema = daemon_1.Daemon.getModule(daemon, module)
+            .buckets[bucket]
+            .schema;
+    }
+    async generate() {
+        const current = await this.getCurrentSchema();
+        const options = this.generateFieldOptions(current);
+        if (!Object.keys(options).length) {
+            return;
+        }
+        const type = current ? 'alter' : 'create';
+        return new Migration(type, this.tableName, options);
+    }
+    async getCurrentSchema() {
+        const columns = await this.sql `
+            SELECT column_name, data_type, is_nullable 
+            FROM information_schema.columns 
+            WHERE table_name = ${this.tableName}`;
+        if (!columns.length) {
+            return;
+        }
+        return columns.map(col => ({
+            ...col,
+            nullable: col.is_nullable === 'YES',
+            field_exists: false
+        }));
+    }
+    generateFieldOptions(current) {
+        let mappedCurrent;
+        if (current) {
+            mappedCurrent = {};
+            current.forEach(col => {
+                mappedCurrent[col.column_name] = col;
+            });
+            Object.keys(this.schema.model.fields)
+                .forEach(name => {
+                if (mappedCurrent[name]) {
+                    mappedCurrent[name].field_exists = true;
+                }
+            });
+        }
+        const fields = {};
+        Object.entries(this.schema.model.fields)
+            .forEach(([name, field]) => {
+            const options = this.generateField(name, field, mappedCurrent);
+            if (options.length) {
+                fields[name] = options;
+            }
+        });
+        return fields;
+    }
+    generateField(name, $, columns) {
+        const type = this.fieldType($);
+        const pk = $.name === 'id';
+        const nullable = !$.required;
+        // Table doesn't exist yet, only option is to create the field
+        if (!columns) {
+            return [new MigrationField(undefined, $.name, {
+                    create: { type, pk, nullable }
+                })];
+        }
+        // Table exists, evaluate options
+        else {
+            // Field exists, alter only what changed
+            const column = columns[$.name];
+            if (column) {
+                if ($.name === 'id') {
+                    // Id can't be modified for now.
+                    return [];
+                }
+                // TODO: check details such as
+                // - changes in decimal precision
+                // - changes in maxLength
+                // - changes in fk (!!!)
+                const _type = type.startsWith(column.data_type) ? undefined : column.data_type;
+                const _nullable = column.nullable !== nullable ? column.nullable : undefined;
+                if (_type !== undefined || _nullable !== undefined) {
+                    return [new MigrationField(undefined, $.name, {
+                            alter: { type: _type, nullable: _nullable }
+                        })];
+                }
+                return [];
+            }
+            // If field doesn't exists in columns, it might:
+            //  - be a new field
+            //  - be a field of the same type being renamed
+            else {
+                const options = [];
+                options.push(new MigrationField(undefined, $.name, {
+                    create: { type, pk, nullable }
+                }));
+                const deletedColumnsOfSameType = Object.values(columns)
+                    .filter(col => !col.field_exists)
+                    .filter(col => type.startsWith(col.data_type));
+                if (deletedColumnsOfSameType.length) {
+                    // TODO: check details such as
+                    // - changes in decimal precision
+                    // - changes in maxLength
+                    // - changes in fk (!!!)
+                    deletedColumnsOfSameType.forEach(col => {
+                        const _type = type.startsWith(col.data_type) ? undefined : col.data_type;
+                        const _nullable = col.nullable !== nullable ? col.nullable : undefined;
+                        if (_type !== undefined || _nullable !== undefined) {
+                            options.push(new MigrationField(undefined, $.name, {
+                                alter: { name: col.column_name, type: _type, nullable: _nullable }
+                            }));
+                        }
+                    });
+                }
+                return options;
+            }
+        }
+    }
+    fieldType($) {
+        if ($.name === 'id') {
+            if ($.type === 'string') {
+                return 'character(64) PRIMARY KEY';
+            }
+            return 'serial4 PRIMARY KEY';
+        }
+        let type = {
+            'boolean': () => 'boolean',
+            'date': () => 'date',
+            'datetime': () => 'timestamp without time zone',
+            'decimal': () => 'numeric(18,9)', // TODO: read from schema (not stored yet)
+            'dict': () => 'jsonb',
+            'enum': () => 'character(64)', // TODO: read from schema maxLength
+            'file': () => { throw new Error('A file field shouldn\'t be stored on SQL'); },
+            'float': () => 'double precision',
+            'int': () => 'integer',
+            'obj': () => 'jsonb',
+            'string': () => 'character varying', // TODO: char() if maxLength
+            'unknown': () => { throw new Error('An unknown field shouldn\'t be stored on SQL'); },
+        }[$.type]();
+        if ($.array) {
+            type += '[]';
+        }
+        if ($.required) {
+            type += ' NOT NULL';
+        }
+        return type;
+    }
+}
+exports.BucketMigrator = BucketMigrator;

package/lib/adapters/postgres/src/migrator/migrator.d.ts

@@ -0,0 +1,21 @@
+import postgres from 'postgres';
+import { Migration } from './migration';
+import { $Space } from "../../../../elements";
+import { MigrationMethod, MigrationStatus } from './runner';
+import { AnyDaemon } from "../../../../engine/daemon";
+export type MigratorConfig = {
+    dirpath?: string;
+    postgres?: postgres.Options<any>;
+};
+export declare class Migrator<S extends $Space> {
+    protected daemon: AnyDaemon;
+    private sql;
+    dirpath: string;
+    static MIGRATION_TABLE_NAME: string;
+    status: MigrationStatus;
+    private constructor();
+    static prepare(daemon: AnyDaemon, sql: postgres.Sql<any>): Promise<Migrator<$Space>>;
+    generate(): Promise<Migration[]>;
+    generateForBucket<ModuleName extends keyof S['modules']>(module: ModuleName, bucket: keyof S['modules'][ModuleName]['buckets'], tableName: string): Promise<Migration | undefined>;
+    static migration<S extends $Space>(...$: ConstructorParameters<typeof MigrationMethod>): MigrationMethod;
+}

package/lib/adapters/postgres/src/migrator/migrator.js

@@ -0,0 +1,73 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Migrator = void 0;
+const log_1 = require("../../../../engine/util/log");
+const migration_1 = require("./migration");
+const runner_1 = require("./runner");
+const daemon_1 = require("../../../../engine/daemon");
+const bucket_1 = require("../../../../elements/entities/bucket/bucket");
+const string_1 = require("../../../../engine/util/string");
+class Migrator {
+    constructor(daemon, sql, dirpath = './migrations') {
+        this.daemon = daemon;
+        this.sql = sql;
+        this.dirpath = dirpath;
+    }
+    static async prepare(daemon, sql) {
+        const migrator = new Migrator(daemon, sql);
+        const oldTable = await migrator.sql `
+            SELECT * FROM pg_catalog.pg_tables WHERE tablename = ${Migrator.MIGRATION_TABLE_NAME};
+        `;
+        if (!oldTable.length) {
+            await migrator.sql `CREATE TABLE ${migrator.sql(Migrator.MIGRATION_TABLE_NAME)} (
+                id SERIAL PRIMARY KEY,
+                name VARCHAR NOT NULL,
+                description VARCHAR,
+                batch INT4 NOT NULL,
+                timestamp TIMESTAMP NOT NULL,
+                hash VARCHAR NOT NULL,
+                filehash VARCHAR NOT NULL
+            )`;
+        }
+        migrator.status = await runner_1.MigrationRunner.status(migrator.sql, migrator.dirpath);
+        return migrator;
+    }
+    async generate() {
+        const modules = daemon_1.Daemon.getModules(this.daemon);
+        const migrations = [];
+        for (const module of modules) {
+            const buckets = daemon_1.Daemon.getModule(this.daemon, module.name).buckets;
+            for (const bucket in buckets) {
+                const adapter = bucket_1.Bucket.getAdapter(buckets[bucket]);
+                if (!adapter?.tableName)
+                    continue;
+                const migration = await this.generateForBucket(module.name, bucket, adapter.tableName);
+                if (migration) {
+                    migrations.push(migration);
+                }
+            }
+        }
+        return migrations;
+    }
+    async generateForBucket(module, bucket, tableName) {
+        const migrator = new migration_1.BucketMigrator(this.daemon, this.sql, module, bucket, tableName);
+        const migration = await migrator.generate();
+        const tag = (0, string_1.colored)(`${module}::bucket:${bucket}`, 'lightcyan');
+        if (!migration) {
+            log_1.Log.info('migrator', 'bucket', `No changes detected on ${tag}.`);
+            return undefined;
+        }
+        const hash = migration.hash();
+        const alreadyExists = this.status.items.find(item => item.hash === hash);
+        if (alreadyExists && alreadyExists?.state === 'pending') {
+            log_1.Log.warn('migrator', 'bucket', `A similar migration for ${tag} was found pending, ignoring this one.`);
+            return undefined;
+        }
+        return migration;
+    }
+    static migration(...$) {
+        return new runner_1.MigrationMethod(...$);
+    }
+}
+exports.Migrator = Migrator;
+Migrator.MIGRATION_TABLE_NAME = '__nesoi_migrations';