nesoi 3.0.0

package/lib/adapters/postgres/src/migrator/runner.d.ts

@@ -0,0 +1,61 @@
+import postgres from 'postgres';
+import { Migration } from './migration';
+import { AnyDaemon, Daemon } from "../../../../engine/daemon";
+type MigrationFn = (daemon: Daemon<any, any>, sql: postgres.Sql<any>) => Promise<void>;
+export type MigrationRow = {
+    id: number;
+    name: string;
+    description?: string;
+    batch: number;
+    timestamp: string;
+    hash: string;
+    filehash: string;
+};
+export declare class MigrationMethod {
+    description?: string;
+    up: MigrationFn;
+    down: MigrationFn;
+    constructor($: {
+        description?: string;
+        up: MigrationFn;
+        down: MigrationFn;
+    });
+}
+export declare class MigrationStatus {
+    items: {
+        state: 'done' | 'pending' | 'lost';
+        id?: number;
+        name: string;
+        description?: string;
+        batch?: number;
+        timestamp?: string;
+        hash: string;
+        filehash?: string;
+        method?: MigrationMethod;
+    }[];
+    nextBatch: number;
+    constructor(fileMigrations: {
+        name: string;
+        hash: string;
+        path: string;
+        method: MigrationMethod;
+    }[], dbMigrations: MigrationRow[]);
+    describe(): string;
+}
+export declare class MigrationRunner {
+    dirpath: string;
+    private migrations;
+    private constructor();
+    static scanFiles(dirpath: string): Promise<{
+        name: string;
+        path: string;
+        hash: string;
+        method: MigrationMethod;
+    }[]>;
+    static scanDb(sql: postgres.Sql<any>): Promise<postgres.RowList<MigrationRow[]>>;
+    static status(sql: postgres.Sql<any>, dirpath: string): Promise<MigrationStatus>;
+    static up(daemon: AnyDaemon, sql: postgres.Sql<any>, mode?: 'one' | 'batch', dirpath?: string): Promise<void>;
+    static oneUp(daemon: AnyDaemon, sql: postgres.Sql<any>, migration: Migration, dirpath?: string): Promise<void>;
+    private static migrateUp;
+}
+export {};

package/lib/adapters/postgres/src/migrator/runner.js

@@ -0,0 +1,202 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MigrationRunner = exports.MigrationStatus = exports.MigrationMethod = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const migrator_1 = require("./migrator");
+const crypto_1 = require("crypto");
+const string_1 = require("../../../../engine/util/string");
+const log_1 = require("../../../../engine/util/log");
+const datetime_1 = require("../../../../engine/data/datetime");
+class MigrationMethod {
+    constructor($) {
+        this.description = $.description;
+        this.up = $.up;
+        this.down = $.down;
+    }
+}
+exports.MigrationMethod = MigrationMethod;
+class MigrationStatus {
+    constructor(fileMigrations, dbMigrations) {
+        this.items = dbMigrations.map(migration => ({
+            ...migration,
+            state: 'lost'
+        }));
+        fileMigrations.forEach(migration => {
+            const file = fs.readFileSync(migration.path).toString();
+            const filehash = (0, crypto_1.createHash)('md5').update(file).digest('hex');
+            const old = this.items.find(item => item.name === migration.name);
+            if (old) {
+                if (old.filehash === filehash) {
+                    old.state = 'done';
+                }
+            }
+            else {
+                this.items.push({
+                    id: undefined,
+                    name: migration.name,
+                    description: migration.method.description,
+                    batch: undefined,
+                    hash: migration.hash,
+                    filehash,
+                    state: 'pending',
+                    method: migration.method
+                });
+            }
+        });
+        const lastBatch = Math.max(...this.items.map(item => item.batch || 0), 0);
+        this.nextBatch = lastBatch + 1;
+    }
+    describe() {
+        let str = '';
+        str += `◆ ${(0, string_1.colored)('Migration Status', 'lightblue')}\n`;
+        this.items.forEach(item => {
+            const state = {
+                'done': () => (0, string_1.colored)('done', 'green'),
+                'pending': () => (0, string_1.colored)('pending', 'yellow'),
+                'lost': () => (0, string_1.colored)('lost', 'brown'),
+            }[item.state]();
+            str += `└ ${item.id || '*'}\t${state}\t${item.name} @ ${item.batch || '...'}\n`;
+        });
+        return str;
+    }
+}
+exports.MigrationStatus = MigrationStatus;
+class MigrationRunner {
+    constructor(dirpath = './migrations') {
+        this.dirpath = dirpath;
+        this.migrations = [];
+    }
+    static async scanFiles(dirpath) {
+        const files = [];
+        fs.readdirSync(dirpath, { withFileTypes: true })
+            .forEach(node => {
+            const nodePath = path.resolve(dirpath, node.name);
+            if (!nodePath.endsWith('.ts')) {
+                return;
+            }
+            files.push({
+                name: node.name,
+                path: nodePath
+            });
+        });
+        const migrations = [];
+        for (const file of files) {
+            const contents = fs.readFileSync(file.path).toString();
+            const hash = contents.match(/\$hash\[(.*)\]/)?.[1];
+            if (!hash) {
+                log_1.Log.error('migrator', 'prepare', `Unable to read hash of migration at ${file.path}`);
+                continue;
+            }
+            const { default: method } = await Promise.resolve(`${file.path}`).then(s => __importStar(require(s)));
+            if (method instanceof MigrationMethod) {
+                migrations.push({ ...file, hash, method });
+            }
+        }
+        return migrations;
+    }
+    static async scanDb(sql) {
+        const db = await sql `
+            SELECT * FROM ${sql(migrator_1.Migrator.MIGRATION_TABLE_NAME)}
+        `;
+        return db;
+    }
+    static async status(sql, dirpath) {
+        const fileMigrations = await MigrationRunner.scanFiles(dirpath);
+        const dbMigrations = await MigrationRunner.scanDb(sql);
+        return new MigrationStatus(fileMigrations, dbMigrations);
+    }
+    static async up(daemon, sql, mode = 'one', dirpath = './migrations') {
+        let status = await MigrationRunner.status(sql, dirpath);
+        console.log(status.describe());
+        const pending = status.items.filter(item => item.state === 'pending');
+        if (!pending.length) {
+            log_1.Log.info('migrator', 'up', 'No migrations to run.');
+            return;
+        }
+        if (mode === 'one') {
+            const migration = pending[0];
+            await this.migrateUp(daemon, sql, migration, status.nextBatch);
+        }
+        else {
+            for (const migration of pending) {
+                await this.migrateUp(daemon, sql, migration, status.nextBatch);
+            }
+        }
+        status = await MigrationRunner.status(sql, dirpath);
+        console.log(status.describe());
+    }
+    static async oneUp(daemon, sql, migration, dirpath = './migrations') {
+        let status = await MigrationRunner.status(sql, dirpath);
+        console.log(status.describe());
+        const mig = {
+            ...migration,
+            state: 'pending',
+            hash: migration.hash(),
+            filehash: '',
+            method: {
+                up: async (_, sql) => {
+                    await sql.unsafe(migration.sqlUp());
+                },
+                down: async (_, sql) => {
+                    await sql.unsafe(migration.sqlDown());
+                }
+            }
+        };
+        await this.migrateUp(daemon, sql, mig, status.nextBatch);
+        status = await MigrationRunner.status(sql, dirpath);
+        console.log(status.describe());
+    }
+    static async migrateUp(daemon, sql, migration, batch) {
+        log_1.Log.info('migrator', 'up', `Running migration ${(0, string_1.colored)('▲ UP', 'lightgreen')} ${(0, string_1.colored)(migration.name, 'lightblue')}`);
+        await migration.method.up(daemon, sql);
+        const row = {
+            name: migration.name,
+            batch,
+            timestamp: datetime_1.NesoiDatetime.now(),
+            hash: migration.hash,
+            filehash: migration.filehash
+        };
+        if (migration.description) {
+            row.description = migration.description;
+        }
+        await sql `
+            INSERT INTO ${sql(migrator_1.Migrator.MIGRATION_TABLE_NAME)}
+            ${sql(row)}
+        `;
+    }
+}
+exports.MigrationRunner = MigrationRunner;

package/lib/adapters/postgres/src/postgres.bucket_adapter.d.ts

@@ -0,0 +1,45 @@
+import { $Bucket } from "../../../elements";
+import { BucketAdapter, BucketAdapterConfig } from "../../../elements/entities/bucket/adapters/bucket_adapter";
+import { AnyTrxNode } from "../../../engine/transaction/trx_node";
+import postgres from 'postgres';
+import { PostgresNQLRunner } from './postgres.nql';
+import { AnyTrx } from "../../../engine/transaction/trx";
+import { TrxEngineWrapFn } from "../../../engine/transaction/trx_engine.config";
+import { NQL_QueryMeta } from "../../../elements/entities/bucket/query/nql.schema";
+export type PostgresConfig = BucketAdapterConfig & {
+    connection?: postgres.Options<any>;
+};
+export declare class PostgresProvider {
+    config?: PostgresConfig | undefined;
+    static make<Name extends string>(name: Name, config?: PostgresConfig): {
+        name: Name;
+        up: () => PostgresProvider;
+        down: () => void;
+    };
+    sql: postgres.Sql<any>;
+    nql: PostgresNQLRunner;
+    private constructor();
+    static wrap(provider: string): (trx: AnyTrx, fn: TrxEngineWrapFn<any, any>, providers: Record<string, any>) => Promise<import("../../../engine/transaction/trx_node").TrxNodeStatus>;
+}
+export declare class PostgresBucketAdapter<$ extends $Bucket, Obj extends $['#data']> extends BucketAdapter<$['#data']> {
+    schema: $;
+    provider: PostgresProvider;
+    tableName: string;
+    constructor(schema: $, provider: PostgresProvider, tableName: string);
+    private guard;
+    getQueryMeta(): {
+        scope: string;
+        avgTime: number;
+    };
+    protected deleteEverything(trx: AnyTrxNode): Promise<void>;
+    index(trx: AnyTrxNode): Promise<Obj[]>;
+    get(trx: AnyTrxNode, id: Obj['id']): Promise<Obj>;
+    put(trx: AnyTrxNode, obj: Record<string, any>): Promise<Obj>;
+    putMany(trx: AnyTrxNode, objs: Record<string, any>[]): Promise<Obj[]>;
+    delete(trx: AnyTrxNode, id: Obj['id']): Promise<void>;
+    deleteMany(trx: AnyTrxNode, ids: Obj['id'][]): Promise<void>;
+    syncOne(trx: AnyTrxNode, id: Obj['id'], lastObjUpdateEpoch: number): Promise<any>;
+    syncOneAndPast(trx: AnyTrxNode, id: Obj['id'], lastUpdateEpoch: number): Promise<any>;
+    syncAll(trx: AnyTrxNode, lastHash?: string, lastUpdateEpoch?: number): Promise<any>;
+    static getTableName(trx: AnyTrxNode, meta: NQL_QueryMeta): string;
+}

package/lib/adapters/postgres/src/postgres.bucket_adapter.js

@@ -0,0 +1,278 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PostgresBucketAdapter = exports.PostgresProvider = void 0;
+const bucket_adapter_1 = require("../../../elements/entities/bucket/adapters/bucket_adapter");
+const log_1 = require("../../../engine/util/log");
+const trx_node_1 = require("../../../engine/transaction/trx_node");
+const date_1 = require("../../../engine/data/date");
+const datetime_1 = require("../../../engine/data/datetime");
+const decimal_1 = require("../../../engine/data/decimal");
+const postgres_nql_1 = require("./postgres.nql");
+const trx_1 = require("../../../engine/transaction/trx");
+const database_1 = require("./migrator/database");
+class PostgresProvider {
+    static make(name, config) {
+        return {
+            name: name,
+            up: () => new PostgresProvider(config),
+            down: () => { }
+        };
+    }
+    constructor(config) {
+        this.config = config;
+        log_1.Log.info('postgres', 'provider', 'Connecting to Postgres database');
+        this.sql = database_1.Database.connect({
+            ...(config?.connection || {}),
+            debug: true,
+            types: {
+                char: {
+                    to: 1042,
+                    from: [1042],
+                    serialize: (val) => val?.trim?.(),
+                    parse: (val) => val?.trim?.()
+                },
+                date: {
+                    to: 1082,
+                    from: [1082],
+                    serialize: (val) => val?.toISO(),
+                    parse: (val) => val ? date_1.NesoiDate.fromISO(val) : undefined
+                },
+                datetime: {
+                    to: 1114,
+                    from: [1114],
+                    serialize: (val) => val?.toISO(),
+                    parse: (val) => datetime_1.NesoiDatetime.fromISO(val?.replace(' ', 'T') + 'Z')
+                },
+                decimal: {
+                    to: 1700,
+                    from: [1700],
+                    serialize: (val) => val?.toString(),
+                    parse: (val) => val ? new decimal_1.Decimal(val) : undefined
+                }
+            }
+        });
+        this.nql = new postgres_nql_1.PostgresNQLRunner();
+    }
+    static wrap(provider) {
+        return (trx, fn, providers) => {
+            const postgres = providers[provider].sql;
+            return postgres.begin(sql => {
+                trx_1.Trx.set(trx.root, 'sql', sql);
+                return fn(trx.root);
+            });
+        };
+    }
+}
+exports.PostgresProvider = PostgresProvider;
+class PostgresBucketAdapter extends bucket_adapter_1.BucketAdapter {
+    constructor(schema, provider, tableName) {
+        super(schema, provider.nql, provider.config);
+        this.schema = schema;
+        this.provider = provider;
+        this.tableName = tableName;
+    }
+    guard(sql) {
+        return (template, ...params) => {
+            return sql.call(sql, template, ...params).catch(e => {
+                log_1.Log.error('bucket', 'postgres', e.toString(), e);
+                throw new Error('Database error.');
+            });
+        };
+    }
+    getQueryMeta() {
+        return {
+            scope: 'PG',
+            avgTime: 50
+        };
+    }
+    /* Dangerous, not implemented. */
+    deleteEverything(trx) {
+        throw new Error('Unsafe operation.');
+        return Promise.resolve();
+    }
+    /* Read operations */
+    async index(trx) {
+        const sql = trx_1.Trx.get(trx, 'sql');
+        const objs = await this.guard(sql) `
+            SELECT *
+            FROM ${sql(this.tableName)}
+        `;
+        return objs;
+    }
+    async get(trx, id) {
+        const sql = trx_1.Trx.get(trx, 'sql');
+        const objs = await this.guard(sql) `
+            SELECT *
+            FROM ${sql(this.tableName)}
+            WHERE id = ${id}
+        `;
+        return objs[0];
+    }
+    /* Write Operations */
+    async put(trx, obj) {
+        const sql = trx_1.Trx.get(trx, 'sql');
+        const keys = Object.keys(this.schema.model.fields)
+            .filter(key => obj[key] !== undefined);
+        // Create
+        if (!obj.id) {
+            const objs = await this.guard(sql) `
+                INSERT INTO ${sql(this.tableName)}
+                ${sql(obj, keys)}
+                RETURNING *`;
+            return objs[0];
+        }
+        // Update (Replace)
+        else {
+            const objs = await this.guard(sql) `
+                UPDATE ${sql(this.tableName)} SET
+                ${sql(obj, keys)}
+                WHERE id = ${obj.id}
+                RETURNING *
+            `;
+            return objs[0];
+        }
+    }
+    async putMany(trx, objs) {
+        const sql = trx_1.Trx.get(trx, 'sql');
+        const keys = Object.keys(this.schema.model.fields);
+        for (const obj of objs) {
+            for (const key in obj) {
+                if (obj[key] === undefined) {
+                    delete obj[key];
+                }
+            }
+        }
+        const insert = objs.filter(obj => !obj.id);
+        const update = objs.filter(obj => !!obj.id);
+        let inserted = [];
+        if (insert.length) {
+            const ikeys = keys.filter(key => key !== 'id');
+            inserted = await this.guard(sql) `
+                INSERT INTO ${sql(this.tableName)}
+                ${sql(objs, ikeys)}
+            `;
+        }
+        const updated = [];
+        if (update.length) {
+            throw new Error('Not implemented yet.');
+        }
+        return [...inserted, ...updated];
+    }
+    async delete(trx, id) {
+        const sql = trx_1.Trx.get(trx, 'sql');
+        await this.guard(sql) `
+            DELETE FROM ${sql(this.tableName)}
+            WHERE id = ${id}
+        `;
+    }
+    async deleteMany(trx, ids) {
+        const sql = trx_1.Trx.get(trx, 'sql');
+        await this.guard(sql) `
+            DELETE FROM ${sql(this.tableName)}
+            WHERE id IN ${ids}
+        `;
+    }
+    /* Cache Operations */
+    async syncOne(trx, id, lastObjUpdateEpoch) {
+        throw new Error('Not implemented yet.');
+        return {};
+        // // 1. Check if object was deleted
+        // const obj = await this.get(trx, id);
+        // if (!obj) {
+        //     return 'deleted' as const;
+        // }
+        // // 2. Check if object was updated
+        // const updateEpoch = this.getUpdateEpoch(obj);
+        // const hasObjUpdated = updateEpoch > lastObjUpdateEpoch;
+        // if (!hasObjUpdated) {
+        //     return null;
+        // }
+        // // 3. Return updated object and epoch
+        // return {
+        //     obj,
+        //     updateEpoch
+        // };
+    }
+    async syncOneAndPast(trx, id, lastUpdateEpoch) {
+        throw new Error('Not implemented yet.');
+        return {};
+        // // 1. Check if object was deleted
+        // const obj = await this.get(trx, id);
+        // if (!obj) {
+        //     return 'deleted' as const;
+        // }
+        // // 2. Check if object was updated
+        // const objUpdateEpoch = this.getUpdateEpoch(obj);
+        // const hasObjUpdated = objUpdateEpoch > lastUpdateEpoch;       
+        // if (!hasObjUpdated) {
+        //     return null;
+        // }
+        // // 3. Return all objects updated and the max epoch
+        // let updateEpoch = 0;
+        // const changed = (Object.values(this.data) as Obj[])
+        //     .map(obj => {
+        //         const epoch = this.getUpdateEpoch(obj);
+        //         if (epoch > updateEpoch) {
+        //             updateEpoch = epoch;
+        //         }
+        //         return { obj, updateEpoch: epoch };
+        //     })
+        //     .filter(obj => obj.updateEpoch > lastUpdateEpoch);
+        // if (!changed.length) {
+        //     return null;
+        // }
+        // return changed;
+    }
+    async syncAll(trx, lastHash, lastUpdateEpoch = 0) {
+        throw new Error('Not implemented yet.');
+        return {};
+        // // 1. Hash the current ids
+        // const idStr = Object.keys(this.data).sort().join('');
+        // const hash = createHash('md5').update(idStr).digest('hex');
+        // // 2. If hash changed, return a reset sync with all objects
+        // if (hash !== lastHash) {
+        //     let updateEpoch = 0;
+        //     const sync = (await this.index(trx) as Obj[])
+        //         .map(obj => {
+        //             const epoch = this.getUpdateEpoch(obj);
+        //             if (epoch > updateEpoch) {
+        //                 updateEpoch = epoch;
+        //             }
+        //             return { obj, updateEpoch: epoch };
+        //         });
+        //     return {
+        //         sync,
+        //         hash,
+        //         updateEpoch,
+        //         reset: true
+        //     };
+        // }
+        // // 3. Find the data that changed and return it
+        // let updateEpoch = 0;
+        // const sync = (Object.values(this.data) as Obj[])
+        //     .map(obj => {
+        //         const epoch = this.getUpdateEpoch(obj);
+        //         if (epoch > updateEpoch) {
+        //             updateEpoch = epoch;
+        //         }
+        //         return { obj, updateEpoch: epoch };
+        //     })
+        //     .filter(obj => obj.updateEpoch > lastUpdateEpoch);
+        // if (!sync.length) {
+        //     return null;
+        // }
+        // return {
+        //     sync,
+        //     hash,
+        //     updateEpoch,
+        //     reset: false
+        // };
+    }
+    static getTableName(trx, meta) {
+        const bucketName = meta.bucket.name;
+        const bucket = trx_node_1.TrxNode.getModule(trx).buckets[bucketName];
+        const adapter = bucket.adapter;
+        return adapter.tableName;
+    }
+}
+exports.PostgresBucketAdapter = PostgresBucketAdapter;

package/lib/adapters/postgres/src/postgres.cli.d.ts

@@ -0,0 +1,46 @@
+import { CLIAdapter, CLICommand } from "../../../engine/cli/cli_adapter";
+import { PostgresProvider } from './postgres.bucket_adapter';
+import { AnyDaemon } from "../../../engine/daemon";
+export declare class cmd_check extends CLICommand {
+    provider: PostgresProvider;
+    constructor(provider: PostgresProvider);
+    run(): Promise<void>;
+}
+export declare class cmd_tables extends CLICommand {
+    provider: PostgresProvider;
+    constructor(provider: PostgresProvider);
+    run(): Promise<void>;
+}
+export declare class cmd_create_db extends CLICommand {
+    provider: PostgresProvider;
+    constructor(provider: PostgresProvider);
+    run(_: AnyDaemon, $: {
+        name: string;
+    }): Promise<void>;
+}
+export declare class cmd_status extends CLICommand {
+    provider: PostgresProvider;
+    constructor(provider: PostgresProvider);
+    run(daemon: AnyDaemon): Promise<void>;
+}
+export declare class cmd_make_migrations extends CLICommand {
+    provider: PostgresProvider;
+    constructor(provider: PostgresProvider);
+    run(daemon: AnyDaemon, $: {
+        tag: string;
+    }): Promise<void>;
+}
+export declare class cmd_migrate_up extends CLICommand {
+    provider: PostgresProvider;
+    constructor(provider: PostgresProvider);
+    run(daemon: AnyDaemon): Promise<void>;
+}
+export declare class cmd_migrate_one_up extends CLICommand {
+    provider: PostgresProvider;
+    constructor(provider: PostgresProvider);
+    run(daemon: AnyDaemon): Promise<void>;
+}
+export declare class PostgresCLI extends CLIAdapter {
+    provider: PostgresProvider;
+    constructor(provider: PostgresProvider);
+}