nesoi 3.0.0 → 3.0.6

package/lib/adapters/postgres/src/migrator/runner.js

@@ -41,6 +41,8 @@ const crypto_1 = require("crypto");
 const string_1 = require("../../../../engine/util/string");
 const log_1 = require("../../../../engine/util/log");
 const datetime_1 = require("../../../../engine/data/datetime");
+const daemon_1 = require("../../../../engine/daemon");
+const trx_1 = require("../../../../engine/transaction/trx");
 class MigrationMethod {
     constructor($) {
         this.description = $.description;
@@ -62,11 +64,13 @@ class MigrationStatus {
             if (old) {
                 if (old.filehash === filehash) {
                     old.state = 'done';
+                    old.method = migration.method;
                 }
             }
             else {
                 this.items.push({
                     id: undefined,
+                    module: migration.module,
                     name: migration.name,
                     description: migration.method.description,
                     batch: undefined,
@@ -78,7 +82,7 @@ class MigrationStatus {
             }
         });
         const lastBatch = Math.max(...this.items.map(item => item.batch || 0), 0);
-        this.nextBatch = lastBatch + 1;
+        this.batch = lastBatch;
     }
     describe() {
         let str = '';
@@ -89,7 +93,8 @@ class MigrationStatus {
                 'pending': () => (0, string_1.colored)('pending', 'yellow'),
                 'lost': () => (0, string_1.colored)('lost', 'brown'),
             }[item.state]();
-            str += `└ ${item.id || '*'}\t${state}\t${item.name} @ ${item.batch || '...'}\n`;
+            const module = (0, string_1.colored)(item.module, 'lightcyan');
+            str += `└ ${item.id || '*'}\t${state}\t${module} ${item.name} @ ${item.batch || '...'}\n`;
         });
         return str;
     }
@@ -100,19 +105,26 @@ class MigrationRunner {
         this.dirpath = dirpath;
         this.migrations = [];
     }
-    static async scanFiles(dirpath) {
+    static async scanFiles(daemon, dirpath) {
+        const modules = daemon_1.Daemon.getModules(daemon);
         const files = [];
-        fs.readdirSync(dirpath, { withFileTypes: true })
-            .forEach(node => {
-            const nodePath = path.resolve(dirpath, node.name);
-            if (!nodePath.endsWith('.ts')) {
-                return;
-            }
-            files.push({
-                name: node.name,
-                path: nodePath
+        for (const module of modules) {
+            const modulepath = path.join('modules', module.name, dirpath);
+            if (!fs.existsSync(modulepath))
+                continue;
+            fs.readdirSync(modulepath, { withFileTypes: true })
+                .forEach(node => {
+                const nodePath = path.resolve(modulepath, node.name);
+                if (nodePath.endsWith('.d.ts')) {
+                    return;
+                }
+                files.push({
+                    module: module.name,
+                    name: node.name,
+                    path: nodePath
+                });
             });
-        });
+        }
         const migrations = [];
         for (const file of files) {
             const contents = fs.readFileSync(file.path).toString();
@@ -123,7 +135,8 @@ class MigrationRunner {
             }
             const { default: method } = await Promise.resolve(`${file.path}`).then(s => __importStar(require(s)));
             if (method instanceof MigrationMethod) {
-                migrations.push({ ...file, hash, method });
+                const name = file.name.replace('.ts', '').replace('.js', '');
+                migrations.push({ module: file.module, name, path: file.path, hash, method });
             }
         }
         return migrations;
@@ -131,36 +144,61 @@ class MigrationRunner {
     static async scanDb(sql) {
         const db = await sql `
             SELECT * FROM ${sql(migrator_1.Migrator.MIGRATION_TABLE_NAME)}
+            ORDER BY id
         `;
         return db;
     }
-    static async status(sql, dirpath) {
-        const fileMigrations = await MigrationRunner.scanFiles(dirpath);
+    static async status(daemon, sql, dirpath) {
+        const fileMigrations = await MigrationRunner.scanFiles(daemon, dirpath);
         const dbMigrations = await MigrationRunner.scanDb(sql);
         return new MigrationStatus(fileMigrations, dbMigrations);
     }
     static async up(daemon, sql, mode = 'one', dirpath = './migrations') {
-        let status = await MigrationRunner.status(sql, dirpath);
+        let status = await MigrationRunner.status(daemon, sql, dirpath);
         console.log(status.describe());
         const pending = status.items.filter(item => item.state === 'pending');
         if (!pending.length) {
             log_1.Log.info('migrator', 'up', 'No migrations to run.');
             return;
         }
-        if (mode === 'one') {
-            const migration = pending[0];
-            await this.migrateUp(daemon, sql, migration, status.nextBatch);
-        }
-        else {
-            for (const migration of pending) {
-                await this.migrateUp(daemon, sql, migration, status.nextBatch);
+        await sql.begin(async (sql) => {
+            if (mode === 'one') {
+                const migration = pending[0];
+                await this.migrateUp(daemon, sql, migration, status.batch + 1);
             }
+            else {
+                for (const migration of pending) {
+                    await this.migrateUp(daemon, sql, migration, status.batch + 1);
+                }
+            }
+        });
+        status = await MigrationRunner.status(daemon, sql, dirpath);
+        console.log(status.describe());
+    }
+    static async down(daemon, sql, mode = 'one', dirpath = './migrations') {
+        let status = await MigrationRunner.status(daemon, sql, dirpath);
+        console.log(status.describe());
+        const lastBatch = status.items.filter(item => item.batch === status.batch);
+        if (!lastBatch.length) {
+            log_1.Log.info('migrator', 'down', 'No migrations to rollback.');
+            return;
         }
-        status = await MigrationRunner.status(sql, dirpath);
+        await sql.begin(async (sql) => {
+            if (mode === 'one') {
+                const migration = lastBatch.at(-1);
+                await this.migrateDown(daemon, sql, migration);
+            }
+            else {
+                for (const migration of lastBatch) {
+                    await this.migrateDown(daemon, sql, migration);
+                }
+            }
+        });
+        status = await MigrationRunner.status(daemon, sql, dirpath);
         console.log(status.describe());
     }
-    static async oneUp(daemon, sql, migration, dirpath = './migrations') {
-        let status = await MigrationRunner.status(sql, dirpath);
+    static async injectUp(daemon, sql, migration, dirpath = './migrations') {
+        let status = await MigrationRunner.status(daemon, sql, dirpath);
         console.log(status.describe());
         const mig = {
             ...migration,
@@ -168,22 +206,35 @@ class MigrationRunner {
             hash: migration.hash(),
             filehash: '',
             method: {
-                up: async (_, sql) => {
-                    await sql.unsafe(migration.sqlUp());
+                up: async ($) => {
+                    await $.sql.unsafe(migration.sqlUp());
                 },
-                down: async (_, sql) => {
-                    await sql.unsafe(migration.sqlDown());
+                down: async ($) => {
+                    await $.sql.unsafe(migration.sqlDown());
                 }
             }
         };
-        await this.migrateUp(daemon, sql, mig, status.nextBatch);
-        status = await MigrationRunner.status(sql, dirpath);
+        await sql.begin(async (sql) => {
+            await this.migrateUp(daemon, sql, mig, status.batch + 1);
+        });
+        status = await MigrationRunner.status(daemon, sql, dirpath);
         console.log(status.describe());
     }
     static async migrateUp(daemon, sql, migration, batch) {
         log_1.Log.info('migrator', 'up', `Running migration ${(0, string_1.colored)('▲ UP', 'lightgreen')} ${(0, string_1.colored)(migration.name, 'lightblue')}`);
-        await migration.method.up(daemon, sql);
+        const status = await daemon.trx(migration.module)
+            .run(async (trx) => {
+            trx_1.Trx.set(trx, 'sql', sql);
+            await migration.method.up({
+                sql,
+                trx
+            });
+        });
+        if (status.state !== 'ok') {
+            throw new Error('Migration failed. Rolling back all batch changes.');
+        }
         const row = {
+            module: migration.module,
             name: migration.name,
             batch,
             timestamp: datetime_1.NesoiDatetime.now(),
@@ -198,5 +249,23 @@ class MigrationRunner {
             ${sql(row)}
         `;
     }
+    static async migrateDown(daemon, sql, migration) {
+        log_1.Log.info('migrator', 'up', `Running migration ${(0, string_1.colored)('▼ DOWN', 'yellow')} ${(0, string_1.colored)(migration.name, 'lightblue')}`);
+        const status = await daemon.trx(migration.module)
+            .run(async (trx) => {
+            trx_1.Trx.set(trx, 'sql', sql);
+            await migration.method.down({
+                sql,
+                trx
+            });
+        });
+        if (status.state !== 'ok') {
+            throw new Error('Migration failed. Rolling back all batch changes.');
+        }
+        await sql `
+            DELETE FROM ${sql(migrator_1.Migrator.MIGRATION_TABLE_NAME)}
+            WHERE id = ${migration.id}
+        `;
+    }
 }
 exports.MigrationRunner = MigrationRunner;

package/lib/adapters/postgres/src/postgres.bucket_adapter.d.ts

@@ -1,26 +1,8 @@
 import { $Bucket } from "../../../elements";
-import { BucketAdapter, BucketAdapterConfig } from "../../../elements/entities/bucket/adapters/bucket_adapter";
+import { BucketAdapter } from "../../../elements/entities/bucket/adapters/bucket_adapter";
 import { AnyTrxNode } from "../../../engine/transaction/trx_node";
-import postgres from 'postgres';
-import { PostgresNQLRunner } from './postgres.nql';
-import { AnyTrx } from "../../../engine/transaction/trx";
-import { TrxEngineWrapFn } from "../../../engine/transaction/trx_engine.config";
 import { NQL_QueryMeta } from "../../../elements/entities/bucket/query/nql.schema";
-export type PostgresConfig = BucketAdapterConfig & {
-    connection?: postgres.Options<any>;
-};
-export declare class PostgresProvider {
-    config?: PostgresConfig | undefined;
-    static make<Name extends string>(name: Name, config?: PostgresConfig): {
-        name: Name;
-        up: () => PostgresProvider;
-        down: () => void;
-    };
-    sql: postgres.Sql<any>;
-    nql: PostgresNQLRunner;
-    private constructor();
-    static wrap(provider: string): (trx: AnyTrx, fn: TrxEngineWrapFn<any, any>, providers: Record<string, any>) => Promise<import("../../../engine/transaction/trx_node").TrxNodeStatus>;
-}
+import { PostgresProvider } from './postgres.provider';
 export declare class PostgresBucketAdapter<$ extends $Bucket, Obj extends $['#data']> extends BucketAdapter<$['#data']> {
     schema: $;
     provider: PostgresProvider;
@@ -34,12 +16,27 @@ export declare class PostgresBucketAdapter<$ extends $Bucket, Obj extends $['#da
     protected deleteEverything(trx: AnyTrxNode): Promise<void>;
     index(trx: AnyTrxNode): Promise<Obj[]>;
     get(trx: AnyTrxNode, id: Obj['id']): Promise<Obj>;
+    private precleanup;
+    create(trx: AnyTrxNode, obj: Record<string, any>): Promise<Obj>;
+    createMany(trx: AnyTrxNode, objs: Record<string, any>[]): Promise<Obj[]>;
+    patch(trx: AnyTrxNode, obj: Record<string, any>): Promise<Obj>;
+    patchMany(trx: AnyTrxNode, objs: Record<string, any>[]): Promise<$["#data"][]>;
+    replace(trx: AnyTrxNode, obj: Record<string, any>): Promise<Obj>;
+    replaceMany(trx: AnyTrxNode, objs: Record<string, any>[]): Promise<$["#data"][]>;
     put(trx: AnyTrxNode, obj: Record<string, any>): Promise<Obj>;
-    putMany(trx: AnyTrxNode, objs: Record<string, any>[]): Promise<Obj[]>;
+    putMany(trx: AnyTrxNode, objs: Record<string, any>[]): Promise<$["#data"][]>;
     delete(trx: AnyTrxNode, id: Obj['id']): Promise<void>;
     deleteMany(trx: AnyTrxNode, ids: Obj['id'][]): Promise<void>;
     syncOne(trx: AnyTrxNode, id: Obj['id'], lastObjUpdateEpoch: number): Promise<any>;
     syncOneAndPast(trx: AnyTrxNode, id: Obj['id'], lastUpdateEpoch: number): Promise<any>;
     syncAll(trx: AnyTrxNode, lastHash?: string, lastUpdateEpoch?: number): Promise<any>;
-    static getTableName(trx: AnyTrxNode, meta: NQL_QueryMeta): string;
+    static getTableMeta(trx: AnyTrxNode, meta: NQL_QueryMeta): {
+        tableName: string;
+        meta: {
+            created_at: string;
+            created_by: string;
+            updated_at: string;
+            updated_by: string;
+        };
+    };
 }

package/lib/adapters/postgres/src/postgres.bucket_adapter.js

@@ -1,69 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.PostgresBucketAdapter = exports.PostgresProvider = void 0;
+exports.PostgresBucketAdapter = void 0;
 const bucket_adapter_1 = require("../../../elements/entities/bucket/adapters/bucket_adapter");
 const log_1 = require("../../../engine/util/log");
 const trx_node_1 = require("../../../engine/transaction/trx_node");
-const date_1 = require("../../../engine/data/date");
-const datetime_1 = require("../../../engine/data/datetime");
-const decimal_1 = require("../../../engine/data/decimal");
-const postgres_nql_1 = require("./postgres.nql");
 const trx_1 = require("../../../engine/transaction/trx");
-const database_1 = require("./migrator/database");
-class PostgresProvider {
-    static make(name, config) {
-        return {
-            name: name,
-            up: () => new PostgresProvider(config),
-            down: () => { }
-        };
-    }
-    constructor(config) {
-        this.config = config;
-        log_1.Log.info('postgres', 'provider', 'Connecting to Postgres database');
-        this.sql = database_1.Database.connect({
-            ...(config?.connection || {}),
-            debug: true,
-            types: {
-                char: {
-                    to: 1042,
-                    from: [1042],
-                    serialize: (val) => val?.trim?.(),
-                    parse: (val) => val?.trim?.()
-                },
-                date: {
-                    to: 1082,
-                    from: [1082],
-                    serialize: (val) => val?.toISO(),
-                    parse: (val) => val ? date_1.NesoiDate.fromISO(val) : undefined
-                },
-                datetime: {
-                    to: 1114,
-                    from: [1114],
-                    serialize: (val) => val?.toISO(),
-                    parse: (val) => datetime_1.NesoiDatetime.fromISO(val?.replace(' ', 'T') + 'Z')
-                },
-                decimal: {
-                    to: 1700,
-                    from: [1700],
-                    serialize: (val) => val?.toString(),
-                    parse: (val) => val ? new decimal_1.Decimal(val) : undefined
-                }
-            }
-        });
-        this.nql = new postgres_nql_1.PostgresNQLRunner();
-    }
-    static wrap(provider) {
-        return (trx, fn, providers) => {
-            const postgres = providers[provider].sql;
-            return postgres.begin(sql => {
-                trx_1.Trx.set(trx.root, 'sql', sql);
-                return fn(trx.root);
-            });
-        };
-    }
-}
-exports.PostgresProvider = PostgresProvider;
 class PostgresBucketAdapter extends bucket_adapter_1.BucketAdapter {
     constructor(schema, provider, tableName) {
         super(schema, provider.nql, provider.config);
@@ -96,6 +37,7 @@ class PostgresBucketAdapter extends bucket_adapter_1.BucketAdapter {
         const objs = await this.guard(sql) `
             SELECT *
             FROM ${sql(this.tableName)}
+            ORDER BY ${this.config.meta.updated_at} DESC
         `;
         return objs;
     }
@@ -109,54 +51,123 @@ class PostgresBucketAdapter extends bucket_adapter_1.BucketAdapter {
         return objs[0];
     }
     /* Write Operations */
-    async put(trx, obj) {
+    precleanup(obj) {
+        var _a, _b;
+        obj[_a = this.config.meta.created_by] ?? (obj[_a] = null);
+        obj[_b = this.config.meta.updated_by] ?? (obj[_b] = null);
+        for (const key in obj) {
+            if (obj[key] === undefined) {
+                delete obj[key];
+            }
+        }
+    }
+    async create(trx, obj) {
         const sql = trx_1.Trx.get(trx, 'sql');
+        // Use schema fields excluding id
         const keys = Object.keys(this.schema.model.fields)
             .filter(key => obj[key] !== undefined);
+        // Add meta (created_*/updated_*)
+        keys.push(...Object.values(this.config.meta));
+        this.precleanup(obj);
         // Create
-        if (!obj.id) {
-            const objs = await this.guard(sql) `
-                INSERT INTO ${sql(this.tableName)}
-                ${sql(obj, keys)}
-                RETURNING *`;
-            return objs[0];
-        }
-        // Update (Replace)
-        else {
-            const objs = await this.guard(sql) `
-                UPDATE ${sql(this.tableName)} SET
-                ${sql(obj, keys)}
-                WHERE id = ${obj.id}
-                RETURNING *
-            `;
-            return objs[0];
+        const objs = await this.guard(sql) `
+            INSERT INTO ${sql(this.tableName)}
+            ${sql(obj, keys)}
+            RETURNING *`;
+        return objs[0];
+    }
+    async createMany(trx, objs) {
+        const sql = trx_1.Trx.get(trx, 'sql');
+        // Use schema fields excluding id
+        const keys = Object.keys(this.schema.model.fields)
+            .filter(key => key !== 'id');
+        // Add meta (created_*/updated_*)
+        keys.push(...Object.values(this.config.meta));
+        // Pre-cleanup
+        for (const obj of objs) {
+            this.precleanup(obj);
         }
+        // Create
+        const inserted = await this.guard(sql) `
+            INSERT INTO ${sql(this.tableName)}
+            ${sql(objs, keys)}
+            RETURNING *
+        `;
+        return inserted;
     }
-    async putMany(trx, objs) {
+    async patch(trx, obj) {
         const sql = trx_1.Trx.get(trx, 'sql');
-        const keys = Object.keys(this.schema.model.fields);
+        // Use schema keys that exist on object
+        const keys = Object.keys(this.schema.model.fields)
+            .filter(key => obj[key] !== undefined)
+            .filter(key => key in obj);
+        // Add meta
+        keys.push(this.config.meta.updated_by, this.config.meta.updated_at);
+        // Pre-cleanup
+        this.precleanup(obj);
+        // Update
+        const objs = await this.guard(sql) `
+            UPDATE ${sql(this.tableName)} SET
+            ${sql(obj, keys)}
+            WHERE id = ${obj.id}
+            RETURNING *
+        `;
+        return objs[0];
+    }
+    async patchMany(trx, objs) {
+        const _objs = [];
         for (const obj of objs) {
-            for (const key in obj) {
-                if (obj[key] === undefined) {
-                    delete obj[key];
-                }
-            }
+            _objs.push(await this.patch(trx, obj));
         }
-        const insert = objs.filter(obj => !obj.id);
-        const update = objs.filter(obj => !!obj.id);
-        let inserted = [];
-        if (insert.length) {
-            const ikeys = keys.filter(key => key !== 'id');
-            inserted = await this.guard(sql) `
-                INSERT INTO ${sql(this.tableName)}
-                ${sql(objs, ikeys)}
-            `;
+        return _objs;
+    }
+    async replace(trx, obj) {
+        const sql = trx_1.Trx.get(trx, 'sql');
+        // Use all schema keys
+        const keys = Object.keys(this.schema.model.fields)
+            .filter(key => obj[key] !== undefined);
+        keys.push(this.config.meta.updated_by, this.config.meta.updated_at);
+        this.precleanup(obj);
+        const objs = await this.guard(sql) `
+            UPDATE ${sql(this.tableName)} SET
+            ${sql(obj, keys)}
+            WHERE id = ${obj.id}
+            RETURNING *
+        `;
+        return objs[0];
+    }
+    async replaceMany(trx, objs) {
+        const _objs = [];
+        for (const obj of objs) {
+            _objs.push(await this.replace(trx, obj));
         }
-        const updated = [];
-        if (update.length) {
-            throw new Error('Not implemented yet.');
+        return _objs;
+    }
+    async put(trx, obj) {
+        const sql = trx_1.Trx.get(trx, 'sql');
+        // Use all schema keys
+        const keys = Object.keys(this.schema.model.fields)
+            .filter(key => obj[key] !== undefined);
+        // Add meta (created_*/updated_*)
+        const ikeys = keys.concat(...Object.values(this.config.meta));
+        const ukeys = keys.concat(this.config.meta.updated_by, this.config.meta.updated_at);
+        this.precleanup(obj);
+        const objs = await this.guard(sql) `
+            INSERT INTO ${sql(this.tableName)}
+            ${sql(obj, ikeys)}
+            ON CONFLICT(id)
+            DO UPDATE SET
+            ${sql(obj, ukeys)}
+            RETURNING *
+        `;
+        return objs[0];
+    }
+    async putMany(trx, objs) {
+        const _objs = [];
+        for (const obj of objs) {
+            _objs.push(await this.put(trx, obj));
         }
-        return [...inserted, ...updated];
+        return _objs;
     }
     async delete(trx, id) {
         const sql = trx_1.Trx.get(trx, 'sql');
@@ -268,11 +279,16 @@ class PostgresBucketAdapter extends bucket_adapter_1.BucketAdapter {
         //     reset: false
         // };
     }
-    static getTableName(trx, meta) {
+    static getTableMeta(trx, meta) {
+        const trxModule = trx_node_1.TrxNode.getModule(trx);
         const bucketName = meta.bucket.name;
-        const bucket = trx_node_1.TrxNode.getModule(trx).buckets[bucketName];
+        const refName = (trxModule.name === meta.bucket.module ? '' : `${meta.bucket.module}::`) + bucketName;
+        const bucket = trxModule.buckets[refName];
         const adapter = bucket.adapter;
-        return adapter.tableName;
+        return {
+            tableName: adapter.tableName,
+            meta: adapter.config.meta
+        };
     }
 }
 exports.PostgresBucketAdapter = PostgresBucketAdapter;

package/lib/adapters/postgres/src/postgres.cli.d.ts

@@ -1,10 +1,10 @@
 import { CLIAdapter, CLICommand } from "../../../engine/cli/cli_adapter";
-import { PostgresProvider } from './postgres.bucket_adapter';
+import { PostgresProvider } from './postgres.provider';
 import { AnyDaemon } from "../../../engine/daemon";
 export declare class cmd_check extends CLICommand {
     provider: PostgresProvider;
     constructor(provider: PostgresProvider);
-    run(): Promise<void>;
+    run(daemon: AnyDaemon): Promise<void>;
 }
 export declare class cmd_tables extends CLICommand {
     provider: PostgresProvider;
@@ -14,7 +14,7 @@ export declare class cmd_tables extends CLICommand {
 export declare class cmd_create_db extends CLICommand {
     provider: PostgresProvider;
     constructor(provider: PostgresProvider);
-    run(_: AnyDaemon, $: {
+    run(daemon: AnyDaemon, $: {
         name: string;
     }): Promise<void>;
 }
@@ -40,6 +40,26 @@ export declare class cmd_migrate_one_up extends CLICommand {
     constructor(provider: PostgresProvider);
     run(daemon: AnyDaemon): Promise<void>;
 }
+export declare class cmd_migrate_down extends CLICommand {
+    provider: PostgresProvider;
+    constructor(provider: PostgresProvider);
+    run(daemon: AnyDaemon): Promise<void>;
+}
+export declare class cmd_migrate_one_down extends CLICommand {
+    provider: PostgresProvider;
+    constructor(provider: PostgresProvider);
+    run(daemon: AnyDaemon): Promise<void>;
+}
+export declare class cmd_query extends CLICommand {
+    provider: PostgresProvider;
+    constructor(provider: PostgresProvider);
+    run(): Promise<void>;
+}
+export declare class cmd_import_csv extends CLICommand {
+    provider: PostgresProvider;
+    constructor(provider: PostgresProvider);
+    run(daemon: AnyDaemon, input: Record<string, any>): Promise<void>;
+}
 export declare class PostgresCLI extends CLIAdapter {
     provider: PostgresProvider;
     constructor(provider: PostgresProvider);