nesoi 3.0.8 → 3.0.9

package/lib/adapters/postgres/src/migrator/database.d.ts

@@ -2,6 +2,7 @@ import postgres from 'postgres';
 export type TableColumn = {
     column_name: string;
     data_type: string;
+    udt_name: string;
     nullable: boolean;
     field_exists: boolean;
 };
@@ -18,20 +19,16 @@ export declare class Database {
      * @param config
      */
     static listTables(sql: postgres.Sql<any>): Promise<any[]>;
-    /**
-     * Read schema of table.
-     * @param config
-     */
-    private getSchema;
     /**
      * Connect to PostgreSQL and create a database.
      *
      * The `if_exists` flag controls what happens if the database already exists
      * - fail: Throw an exception
      * - keep: Do nothing
-     * - delete: *DROP DATABASE*
+     * - delete: **DROP DATABASE**
      */
     static createDatabase(name: string, config?: postgres.Options<any>, $?: {
+        default_db?: string;
         if_exists: 'fail' | 'keep' | 'delete';
     }): Promise<void>;
 }

package/lib/adapters/postgres/src/migrator/database.js

@@ -37,35 +37,35 @@ class Database {
         `;
         return columns.map(col => col.table_name);
     }
-    /**
-     * Read schema of table.
-     * @param config
-     */
-    async getSchema(sql, tableName) {
-        const columns = await sql `
-            SELECT column_name, data_type, is_nullable 
-            FROM information_schema.columns 
-            WHERE table_name = ${tableName}`;
-        if (!columns.length) {
-            return;
-        }
-        return columns.map(col => ({
-            ...col,
-            nullable: col.is_nullable === 'YES',
-            field_exists: false
-        }));
-    }
+    // /**
+    //  * Read schema of a table.
+    //  * @param config 
+    //  */
+    // private async getSchema(sql: postgres.Sql<any>, tableName: string): Promise<TableColumn[] | undefined> {
+    //     const columns = await sql`
+    //         SELECT column_name, data_type, is_nullable 
+    //         FROM information_schema.columns 
+    //         WHERE table_name = ${tableName}`;
+    //     if (!columns.length) {
+    //         return
+    //     }
+    //     return columns.map(col => ({
+    //         ...col,
+    //         nullable: col.is_nullable === 'YES',
+    //         field_exists: false
+    //     }) as TableColumn);
+    // }
     /**
      * Connect to PostgreSQL and create a database.
      *
      * The `if_exists` flag controls what happens if the database already exists
      * - fail: Throw an exception
      * - keep: Do nothing
-     * - delete: *DROP DATABASE*
+     * - delete: **DROP DATABASE**
      */
     static async createDatabase(name, config, $) {
         const sql = (0, postgres_1.default)(Object.assign({}, config, {
-            db: 'postgres'
+            db: $?.default_db || 'postgres'
         }));
         const dbs = await sql `SELECT datname FROM pg_database`;
         const alreadyExists = dbs.some(db => db.datname === name);

package/lib/adapters/postgres/src/migrator/generator/generator.d.ts

@@ -0,0 +1,22 @@
+import { $Bucket, $Space } from "../../../../../elements";
+import postgres from 'postgres';
+import { AnyDaemon } from "../../../../../engine/daemon";
+import { BucketAdapterConfig } from "../../../../../elements/entities/bucket/adapters/bucket_adapter";
+import { $Migration } from './migration';
+export declare class MigrationGenerator<S extends $Space, D extends AnyDaemon, ModuleName extends NoInfer<keyof S['modules']>> {
+    private sql;
+    private module;
+    private tableName;
+    protected schema: $Bucket;
+    protected config?: BucketAdapterConfig;
+    constructor(daemon: D, sql: postgres.Sql<any>, module: ModuleName, bucketName: NoInfer<keyof S['modules'][ModuleName]['buckets']>, tableName: string);
+    generate(): Promise<$Migration | undefined>;
+    private getCurrentSchema;
+    private generateDrops;
+    private generateSteps;
+    private generateFieldSteps;
+    private fieldUdt;
+    private fieldTypeFromUdt;
+    private fieldType;
+    private manualReview;
+}

package/lib/adapters/postgres/src/migrator/generator/generator.js

@@ -0,0 +1,326 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MigrationGenerator = void 0;
+const daemon_1 = require("../../../../../engine/daemon");
+const migration_1 = require("./migration");
+const string_1 = require("../../../../../engine/util/string");
+const ui_1 = __importDefault(require("../../../../../engine/cli/ui"));
+const log_1 = require("../../../../../engine/util/log");
+class MigrationOption {
+    constructor(schema) {
+        this.schema = schema;
+        this.selected = false;
+        this.excluded_by = [];
+    }
+}
+class MigrationGenerator {
+    constructor(daemon, sql, module, bucketName, tableName) {
+        this.sql = sql;
+        this.module = module;
+        this.tableName = tableName;
+        const bucket = daemon_1.Daemon.getModule(daemon, module).buckets[bucketName];
+        this.schema = bucket.schema;
+        this.config = bucket.adapter.config;
+    }
+    async generate() {
+        const current = await this.getCurrentSchema();
+        const drops = current ? this.generateDrops(current.columns) : {};
+        const steps = this.generateSteps(current?.mapped, drops);
+        steps.push(...Object.values(drops).map(d => ({
+            options: [d]
+        })));
+        if (!steps.length) {
+            return;
+        }
+        const type = current ? 'alter' : 'create';
+        const migration = await this.manualReview(type, steps);
+        return migration;
+    }
+    async getCurrentSchema() {
+        const rawColumns = await this.sql `
+            SELECT column_name, udt_name, is_nullable, numeric_precision, numeric_scale
+            FROM information_schema.columns 
+            WHERE table_name = ${this.tableName}`;
+        if (!rawColumns.length) {
+            return;
+        }
+        const columns = rawColumns.map(col => ({
+            column_name: col.column_name,
+            udt_name: col.udt_name,
+            data_type: this.fieldTypeFromUdt(col.udt_name, {
+                n0: col.numeric_precision,
+                n1: col.numeric_scale,
+            }),
+            nullable: col.is_nullable === 'YES',
+            field_exists: false
+        }));
+        // Map current columns by name, and flag if they already exist
+        const mapped = {};
+        columns.forEach(col => {
+            mapped[col.column_name] = col;
+            const created_by = this.config?.meta.created_by || 'created_by';
+            const created_at = this.config?.meta.created_at || 'created_at';
+            const updated_by = this.config?.meta.updated_by || 'updated_by';
+            const updated_at = this.config?.meta.updated_at || 'updated_at';
+            if (col.column_name === created_by
+                || col.column_name === created_at
+                || col.column_name === updated_by
+                || col.column_name === updated_at) {
+                mapped[col.column_name].field_exists = true;
+            }
+        });
+        Object.keys(this.schema.model.fields)
+            .forEach(name => {
+            if (mapped[name]) {
+                mapped[name].field_exists = true;
+            }
+        });
+        return { columns, mapped };
+    }
+    generateDrops(columns) {
+        const drops = {};
+        for (const col of columns) {
+            if (col.field_exists)
+                continue;
+            drops[col.column_name] = new MigrationOption(new migration_1.$MigrationField(col.column_name, {
+                drop: {
+                    type: col.data_type,
+                    nullable: col.nullable
+                }
+            }));
+        }
+        return drops;
+    }
+    generateSteps(current, drops = {}) {
+        const steps = [];
+        // Generate migration step for each field
+        Object.values(this.schema.model.fields)
+            .forEach(field => {
+            const fieldSteps = this.generateFieldSteps(field, current, drops);
+            if (fieldSteps.length) {
+                steps.push(...fieldSteps);
+            }
+        });
+        // Add meta fields when creating table
+        if (!current) {
+            const created_by = this.config?.meta.created_by || 'created_by';
+            const created_at = this.config?.meta.created_at || 'created_at';
+            const updated_by = this.config?.meta.updated_by || 'updated_by';
+            const updated_at = this.config?.meta.updated_at || 'updated_at';
+            steps.push({ options: [new MigrationOption(new migration_1.$MigrationField(created_by, {
+                        create: { type: 'character(64)', nullable: true }
+                    }))] });
+            steps.push({ options: [new MigrationOption(new migration_1.$MigrationField(created_at, {
+                        create: { type: 'timestamp without time zone' }
+                    }))] });
+            steps.push({ options: [new MigrationOption(new migration_1.$MigrationField(updated_by, {
+                        create: { type: 'character(64)', nullable: true }
+                    }))] });
+            steps.push({ options: [new MigrationOption(new migration_1.$MigrationField(updated_at, {
+                        create: { type: 'timestamp without time zone' }
+                    }))] });
+        }
+        return steps;
+    }
+    generateFieldSteps($, current, drops = {}) {
+        const type = this.fieldType($);
+        const pk = $.name === 'id';
+        const nullable = !$.required;
+        // Table doesn't exist yet, only option is to create the field
+        if (!current) {
+            const options = [
+                new MigrationOption(new migration_1.$MigrationField($.name, {
+                    create: { type, pk, nullable }
+                }))
+            ];
+            return [{ options }];
+        }
+        // Table exists, evaluate options
+        else {
+            const col = current[$.name];
+            // Field exists in columns, alter only what changed
+            if (col) {
+                if ($.name === 'id') {
+                    // Id can't be modified for now.
+                    return [];
+                }
+                // TODO: check details such as
+                // - changes in decimal precision
+                // - changes in maxLength
+                const typeChanged = !type.startsWith(col.data_type);
+                const nullableChanged = col.nullable !== nullable;
+                const steps = [];
+                if (typeChanged) {
+                    const options = [
+                        new MigrationOption(new migration_1.$MigrationField($.name, {
+                            alter_type: { from: col.data_type, to: type, using: {} }
+                        }))
+                    ];
+                    steps.push({ options });
+                }
+                if (nullableChanged) {
+                    const options = [
+                        new MigrationOption(new migration_1.$MigrationField($.name, {
+                            alter_null: { from: col.nullable, to: nullable }
+                        }))
+                    ];
+                    steps.push({ options });
+                }
+                return steps;
+            }
+            // Field doesn't exists in columns, it might:
+            //  - 1: be a new field
+            //  - 2: be a field of the same type being renamed
+            else {
+                const options = [];
+                // Option 1
+                const createOption = new MigrationOption(new migration_1.$MigrationField($.name, {
+                    create: { type, pk, nullable }
+                }));
+                options.push(createOption);
+                const deletedColumnsOfSameType = Object.values(current)
+                    .filter(col => !col.field_exists)
+                    .filter(col => type.startsWith(col.data_type));
+                if (deletedColumnsOfSameType.length) {
+                    // TODO: check details such as
+                    // - changes in decimal precision
+                    // - changes in maxLength
+                    deletedColumnsOfSameType.forEach(col => {
+                        // Option 2
+                        const renameOption = new MigrationOption(new migration_1.$MigrationField(col.column_name, {
+                            rename: { name: $.name }
+                        }));
+                        options.push(renameOption);
+                        // If this option is picked, the drop option for this column is no longer valid
+                        drops[col.column_name].excluded_by.push(renameOption);
+                    });
+                }
+                return [{ options }];
+            }
+        }
+    }
+    fieldUdt($) {
+        if ($.name === 'id') {
+            if ($.type === 'string') {
+                return 'bpchar';
+            }
+            return 'int4';
+        }
+        let type = {
+            'boolean': () => 'bool',
+            'date': () => 'date',
+            'datetime': () => 'timestamp',
+            'decimal': () => 'numeric',
+            'dict': () => 'jsonb',
+            'enum': () => 'bpchar', // TODO: read from schema maxLength
+            'file': () => 'jsonb',
+            'float': () => 'float8',
+            'int': () => 'int4',
+            'obj': () => 'jsonb',
+            'string': () => 'varchar', // TODO: char() if maxLength
+            'unknown': () => { throw new Error('An unknown field shouldn\'t be stored on SQL'); },
+        }[$.type]();
+        if ($.array) {
+            type = '_' + type;
+        }
+        return type;
+    }
+    fieldTypeFromUdt(udt, extra) {
+        const array = udt.startsWith('_');
+        if (array)
+            udt = udt.slice(1);
+        let type = {
+            'bool': () => 'boolean',
+            'date': () => 'date',
+            'timestamp': () => 'timestamp',
+            'numeric': () => `numeric(${extra.n0},${extra.n1})`,
+            'jsonb': () => 'jsonb',
+            'bpchar': () => 'character(64)', // TODO: read from schema maxLength
+            'float8': () => 'double precision',
+            'int4': () => 'integer',
+            'varchar': () => 'character varying', // TODO: char() if maxLength
+            'unknown': () => { throw new Error('An unknown field shouldn\'t be stored on SQL'); },
+        }[udt]();
+        if (array)
+            type += '[]';
+        return type;
+    }
+    fieldType($) {
+        const udt = this.fieldUdt($);
+        return this.fieldTypeFromUdt(udt, {
+            n0: ($.meta?.decimal?.left || 9) + ($.meta?.decimal?.right || 9),
+            n1: $.meta?.decimal?.right || 9
+        });
+    }
+    async manualReview(type, steps) {
+        let header_shown = false;
+        const header = () => {
+            if (header_shown)
+                return;
+            let str = '';
+            str += '┌\n';
+            str += `│ module: ${(0, string_1.colored)(this.module, 'cyan')}\n`;
+            str += `│ table: ${(0, string_1.colored)(this.tableName, 'lightcyan')}\n`;
+            str += `│ ${(0, string_1.colored)('⚠ Requires manual review.', 'red')}\n`;
+            str += '└\n\n';
+            console.clear();
+            console.log(str);
+            header_shown = true;
+        };
+        const fields = [];
+        for (const step of steps) {
+            const stepFields = step.options.filter(field => {
+                if (field.excluded_by.length === 0)
+                    return true;
+                return !field.excluded_by.some(field => field.selected);
+            });
+            let schema;
+            if (stepFields.length === 0) {
+                continue;
+            }
+            else if (stepFields.length === 1) {
+                schema = stepFields[0].schema;
+            }
+            else {
+                header();
+                const opt = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+                const selected = await ui_1.default.select('Pick one of the options below:\n', stepFields.map((s, i) => `${(0, string_1.colored)(opt[i], 'lightcyan')} ${stepFields[i].schema.describe()}`));
+                console.log();
+                stepFields[selected.i].selected = true;
+                schema = stepFields[selected.i].schema;
+            }
+            if ('drop' in schema.operation && !schema.operation.drop.nullable) {
+                header();
+                const defaul = await ui_1.default.question(`Column '${schema.column}' is NOT NULL and is being deleted. What should be the default value on rollback?\n`);
+                schema.operation.drop.default = defaul;
+            }
+            if ('alter_type' in schema.operation) {
+                header();
+                const from = (0, string_1.colored)(schema.operation.alter_type.from, 'lightcyan');
+                const to = (0, string_1.colored)(schema.operation.alter_type.to, 'lightcyan');
+                const up = (0, string_1.colored)('▲ UP', 'lightgreen');
+                const down = (0, string_1.colored)('▼ DOWN', 'yellow');
+                const defaultUp = `${schema.column}::${schema.operation.alter_type.to}`;
+                const defaultDown = `${schema.column}::${schema.operation.alter_type.from}`;
+                const usingUp = await ui_1.default.question(`Column '${schema.column}' is changing from ${from} to ${to}. Write a cast expression for the ${up} migration.\n`, defaultUp);
+                const usingDown = await ui_1.default.question(`Column '${schema.column}' is changing from ${from} to ${to}. Write a cast expression for the ${down} migration.\n`, defaultDown);
+                schema.operation.alter_type.using.up = usingUp;
+                schema.operation.alter_type.using.down = usingDown;
+            }
+            fields.push(schema);
+        }
+        const migration = new migration_1.$Migration(this.module, type, this.tableName, fields);
+        console.clear();
+        console.log(migration.describe());
+        const proceed = await ui_1.default.yesOrNo('Is everything OK with the migration above?');
+        if (!proceed) {
+            log_1.Log.warn('migrator', 'generator', 'Migration rejected by manual review.');
+            return;
+        }
+        return migration;
+    }
+}
+exports.MigrationGenerator = MigrationGenerator;

package/lib/adapters/postgres/src/migrator/generator/migration.d.ts

@@ -0,0 +1,66 @@
+export type $MigrationFieldOperation = {
+    create: {
+        type: string;
+        nullable?: boolean;
+        pk?: boolean;
+    };
+} | {
+    rename: {
+        name: string;
+    };
+} | {
+    alter_type: {
+        from: string;
+        to: string;
+        using: {
+            up: string;
+            down: string;
+        };
+    };
+} | {
+    alter_null: {
+        from: boolean;
+        to: boolean;
+    };
+} | {
+    drop: {
+        type: string;
+        nullable: boolean;
+        default?: string;
+    };
+} | {
+    create_fk: {
+        table: string;
+        field: string;
+    };
+} | {
+    drop_fk: {
+        table: string;
+        field: string;
+    };
+};
+export declare class $MigrationField {
+    column: string;
+    operation: $MigrationFieldOperation;
+    constructor(column: string, operation: $MigrationFieldOperation);
+    describe(): string;
+    sqlUp(table_op: 'create' | 'alter'): string;
+    sqlDown(): string;
+}
+export declare class $Migration {
+    module: string;
+    private type;
+    private tableName;
+    private fields;
+    description?: string | undefined;
+    name: string;
+    constructor(module: string, type: 'create' | 'alter' | 'custom', tableName: string, fields: $MigrationField[], description?: string | undefined);
+    describe(): string;
+    sqlUp(): string[];
+    sqlDown(): string[];
+    save(dirpath?: string): string;
+    private fnUp;
+    private fnDown;
+    hash(): string;
+    static empty(module: string, name: string): $Migration;
+}