@foul11/awesome-db 1.1.0

package/src/index.ts

@@ -0,0 +1,227 @@
+export * from './alter';
+export * from './defaults';
+export * from './Error';
+export * from './indexer';
+export * from './ORM';
+export * from './SQLParser';
+export * from './utils';
+export * from './WebpackFileProvider';
+export * from './log';
+
+export type * from './types';
+
+import { WebpackFileProvider } from './WebpackFileProvider';
+import { Kysely, SqliteDialect } from 'kysely';
+import { Migrator } from 'kysely/migration';
+import { format as format_sql } from '@foul11/sql-formatter';
+import { isWebpack } from '@foul11/awesome';
+import { v4 } from 'uuid';
+
+import Sqlite from 'better-sqlite3';
+import bindings from 'bindings';
+import path from 'path';
+
+import type { logger_db } from './log';
+import type { Database } from 'better-sqlite3';
+import type { CompiledQuery } from 'kysely';
+
+function create_sqlite(db_path: string): Database {
+    const db = new Sqlite(db_path, {
+        nativeBinding: isWebpack()
+            ? bindings({
+                module_root: process.cwd(),
+                bindings: 'better_sqlite3.node',
+                try: [
+                    [ 'module_root', 'dist', 'bindings' ],
+                    [ 'module_root', 'bindings' ],
+                    [ 'dist', 'bindings' ],
+                    [ 'bindings' ],
+                ],
+                path: true,
+            } as any)
+            : null,
+    });
+    
+    db.function('new_uuid', () => {
+        return v4();
+    });
+    
+    return db;
+}
+
+type KyselyDB<DB> = Kysely<DB> & {
+    in_migration: boolean
+    
+    migration_log: string[]
+    transaction_logs: string[]
+};
+
+function format_sqlite_no_throw(log_kysely_db: typeof logger_db, query: CompiledQuery<unknown> | string) {
+    const sql = typeof query === 'string'
+        ? query
+        : query.sql;
+    
+    try {
+        return format_sql(sql, {
+            language: 'sqlite',
+            tabWidth: 4,
+            indentStyle: 'tabularRight',
+            ...(typeof query !== 'string' && {
+                params: query.parameters as any,
+           }),
+        });
+    } catch (e) {
+        log_kysely_db.error(e as Error);
+        return sql;
+    }
+}
+
+function create_kysely<DB>(db: unknown, log_kysely_db: typeof logger_db) {
+    const db_kysely = new Kysely<DB>({
+        dialect: new SqliteDialect({
+            database: db as Database,
+        }),
+        log: (event) => {
+            db_kysely.transaction_logs.push(event.query.sql);
+            
+            if (log_kysely_db.isSillyEnabled()) {
+                if (!db_kysely.in_migration) {
+                    log_kysely_db.silly(`\n${format_sqlite_no_throw(log_kysely_db, event.query)}\n`, {
+                        runtime: event.queryDurationMillis,
+                    });
+                } else {
+                    db_kysely.migration_log.push(format_sqlite_no_throw(log_kysely_db, event.query));
+                }
+            }
+            
+            if (event.level === 'error') {
+                log_kysely_db.error(event.error, {
+                    query: db_kysely.transaction_logs.length
+                        ? db_kysely.transaction_logs.map(sql => format_sqlite_no_throw(log_kysely_db, sql)).join('\n')
+                        : format_sqlite_no_throw(log_kysely_db, event.query),
+                    runtime: event.queryDurationMillis,
+                });
+            }
+            
+            if (/commit|begin|rollback/.test(event.query.sql)) {
+                db_kysely.transaction_logs = [];
+            }
+        },
+    }) as KyselyDB<DB>;
+    
+    /**
+     * Nested transaction hack
+     * Catch execute method,
+     * replace arg in func to proxy transaction,
+     * method transaction() return this transaction
+     */
+    function proxy_transaction(trx: any) {
+        return new Proxy(trx, {
+            get(target: any, prop) {
+                if (prop === 'transaction')
+                    return () => ({
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+                        execute: (fn: Function) => (
+                            fn.apply(target, [ target ])
+                        ),
+                    });
+                
+                if (typeof target[prop] == 'function') {
+                    return target[prop].bind(target);
+                } else {
+                    return target[prop];
+                }
+            },
+        });
+    }
+
+    const old_transaction = db_kysely.transaction;
+    db_kysely.transaction = function (...trx_args) {
+        const transaction = old_transaction.call(this, ...trx_args);
+        
+        return new Proxy(transaction, {
+            get(target: any, prop) {
+                if (prop === 'execute')
+                    // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+                    return (fn: Function) => (
+                        target.execute((trx: any, ...args: any[]) => (
+                            fn.apply(target, [ proxy_transaction(trx), ...args ])
+                        ))
+                    );
+                
+                if (typeof target[prop] == 'function') {
+                    return target[prop].bind(target);
+                } else {
+                    return target[prop];
+                }
+            },
+        });
+    };
+    
+    return db_kysely;
+}
+
+async function create_migrator<DB>(db_name: string, db: KyselyDB<DB>, log_kysely_db: typeof logger_db) {
+    db.in_migration = true;
+    db.migration_log = [];
+    db.transaction_logs = [];
+    
+    const migrator = new Migrator({
+        db, provider: new WebpackFileProvider(db_name),
+    });
+    
+    async function migrateToLatest() {
+        const { error, results } = await migrator.migrateToLatest();
+        
+        for (const result of results ?? []) {
+            if (result.status === 'Success') {
+                log_kysely_db.info(`migration "${result.migrationName}" was executed successfully`);
+            } else if (result.status === 'Error') {
+                log_kysely_db.error(`failed to execute migration "${result.migrationName}"`);
+            }
+        }
+        
+        if (error) {
+            if (!(error instanceof Error))
+                throw error as any;
+            
+            log_kysely_db.error('failed to migrate');
+            log_kysely_db.error(error.stack);
+            
+            if (log_kysely_db.isDebugEnabled()) {
+                log_kysely_db.debug(`\n${db.migration_log.join('\n-------------- NEXT MIGRATION SQL-request --------------\n')}\n`, {});
+                
+                // eslint-disable-next-line no-debugger
+                debugger;
+            }
+            
+            throw error;
+        }
+    }
+    
+    log_kysely_db.debug('starting migration to latest');
+    await migrateToLatest();
+    log_kysely_db.debug('migration to latest completed');
+    
+    db.in_migration = false;
+    db.migration_log = [];
+    
+    return migrator;
+}
+
+export async function create_db<DB>(db_path: string, log_kysely: typeof logger_db) {
+    const db_name = path.parse(db_path).name;
+    const db_sqlite = create_sqlite(db_path) as unknown;
+    const db_log = log_kysely.child({ label5: db_name });
+    const db = create_kysely<DB>(db_sqlite, db_log);
+    const db_migrator = await create_migrator(db_name, db, db_log);
+    
+    return {
+        db: db as Kysely<DB>,
+        db_migrator,
+        db_sqlite,
+        db_log,
+    };
+}
+
+export const SqliteError = Sqlite.SqliteError;

package/src/indexer.ts

@@ -0,0 +1,75 @@
+import { table_info, master_table } from './utils';
+
+import type { ValueOf } from 'type-fest';
+import type { Kysely, AnyColumn } from 'kysely';
+
+export async function indexer<TDB>(
+    db: Kysely<TDB>,
+    options: {
+        exclude?: ValueOf<{ [TB in keyof TDB]: TB extends string ? [ TB, (true | AnyColumn<TDB, TB>[]) ] : never }>[]
+        include?: ValueOf<{ [TB in keyof TDB]: TB extends string ? [ TB, (true | AnyColumn<TDB, TB>[]) ] : never }>[]
+        cleanupOthers?: boolean
+    },
+) {
+    const master = await master_table(db);
+    
+    const idx_exclude = new Map<string, true | string[]>(options.exclude || []);
+    const idx_include = new Map<string, true | string[]>(options.include || []);
+    const cleanupOthers = options.cleanupOthers || false;
+    
+    const tables = master
+        .filter(r => r.type == 'table')
+        .filter(r => !options.include || idx_include.has(r.name))
+        .map(r => r.name);
+    
+    const indexes = new Set(master
+        .filter(r => r.type == 'index')
+        .map(r => r.name),
+    );
+    
+    const isExcluded = (table: string, column: { name: string }) => {
+        const exclude = idx_exclude.get(table);
+        
+        if (exclude === true || exclude?.includes(column.name))
+            return true;
+        
+        return false;
+    };
+    
+    const isIncluded = (table: string, column: { name: string }) => {
+        const include = idx_include.get(table);
+
+        if (options.include && !(include == true || include?.includes(column.name)))
+            return false;
+        
+        return true;
+    };
+    
+    for (const table of tables) {
+        const info = await table_info(db, table as any);
+        
+        for (const column of info) {
+            const idx_name = `autoIdx_${table}_${column.name}`;
+            const skip = !isIncluded(table, column) || isExcluded(table, column) || column.pk;
+            
+            if (indexes.has(idx_name)) {
+                if (cleanupOthers && skip) {
+                    await db.schema
+                        .dropIndex(idx_name)
+                        .execute();
+                }
+                
+                continue;
+            }
+            
+            if (skip)
+                continue;
+            
+            await db.schema
+                .createIndex(idx_name)
+                .on(table)
+                .column(column.name)
+                .execute();
+        }
+    }
+}

package/src/log/access_log.ts

@@ -0,0 +1,29 @@
+/* eslint-disable @stylistic/indent-binary-ops */
+
+import { f_printf, color, loggers, pretty_message, pretty_ms } from '@foul11/awesome-log';
+import type { AccessLogStoreData } from '../tables/AccessLog';
+
+const logger_access_log = loggers.add<{
+    data?: AccessLogStoreData | undefined
+    runtime?: number | bigint | undefined
+}>('access_log', {
+    format: f_printf(info => (``
+        + `${pretty_message(info, { disable_empty: true })}`
+        + `${info.runtime
+                ? color('ms', ` took ${pretty_ms(info.runtime).padStart(11, ' ')}`)
+                : ' '.repeat(6 + 11)
+            }`
+        + `${info.data
+                ? ` (${color('string', `'${info.data.action}'`)}, total: ${color('ms', pretty_ms(info.data.timings.total / 1e6).padStart(11, ' '))})`
+                : ``
+            }`
+    ).trim(), {
+        meta: { label: 'access_log' },
+        sanitize: [
+            'data',
+            'runtime',
+        ],
+    }),
+});
+
+export default logger_access_log;

package/src/log/db.ts

@@ -0,0 +1,28 @@
+/* eslint-disable @stylistic/indent-binary-ops */
+
+import { f_printf, color, loggers, pretty_message, pretty_ms } from '@foul11/awesome-log';
+
+const logger_db = loggers.add<{
+    query?: string | undefined
+    runtime?: number | bigint | undefined
+}>('db', {
+    format: f_printf(info => (``
+        + `${pretty_message(info, { disable_empty: true })}`
+        + `${info.query
+                ? `\n${info.query}\n`
+                : ``
+            }`
+        + `${info.runtime
+                ? color('string', `(runtime: ${color('ms', pretty_ms(info.runtime))})`)
+                : ``
+            }`
+    ).trim(), {
+        meta: { label: 'db' },
+        sanitize: [
+            'query',
+            'runtime',
+        ],
+    }),
+});
+
+export default logger_db;

package/src/log/index.ts

@@ -0,0 +1,2 @@
+export { default as logger_access_log } from './access_log';
+export { default as logger_db } from './db';

package/src/tables/AccessLog/index.ts

@@ -0,0 +1,252 @@
+export * from './schema';
+
+import deepmerge from 'deepmerge';
+
+import { SetString } from '../SetString';
+import { BitFields } from '../../BitFields';
+import { master_table } from '../../utils';
+import { ORM } from '../../ORM';
+
+import type { logger_access_log } from '../../log';
+
+import type { Kysely } from 'kysely';
+import type { BitFieldsUnpack } from '../../BitFields';
+import type { DBTablesByType } from '../../types';
+import type { TableSetString } from '../SetString';
+import type { TableAccessLog, TableAccessLogTiming } from './schema';
+
+type DefaultKeyAccessLog = 'access_log';
+interface DefaultKyselyAccessLog {
+    access_log: TableAccessLog
+    access_log_set_ip: TableSetString
+    access_log_set_ua: TableSetString
+    access_log_set_action: TableSetString
+    access_log_timing: TableAccessLogTiming
+    access_log_set_timing: TableSetString
+}
+
+export interface AccessLogStoreData {
+    action: string
+    flag: number
+    ip?: string | undefined
+    meta?: string | undefined
+    user_agent?: string | undefined
+    timings: Record<string, number> & {
+        total: number
+        exclude: number
+    }
+    extended: Record<string, any>
+}
+
+export interface AccessLogStore {
+    store(data: AccessLogStoreData): Promise<void>
+}
+
+export class AccessLogStoreDB implements AccessLogStore {
+    protected readonly db: Kysely<DefaultKyselyAccessLog>;
+    protected readonly domain: DefaultKeyAccessLog;
+    protected readonly logger: typeof logger_access_log | undefined;
+    
+    private constructor(db: any, domain: any, logger?: typeof logger_access_log) {
+        this.db = db;
+        this.domain = domain;
+        this.logger = logger;
+    }
+    
+    static async create<
+        DB,
+        Domain extends DBTablesByType<DB, TableAccessLog>,
+    >(
+        db: Kysely<DB>,
+        domain: Domain,
+        logger?: typeof logger_access_log,
+    ) {
+        const master = await master_table(db);
+        const tables = [
+            `${domain}`,
+            `${domain}_set_ip`,
+            `${domain}_set_ua`,
+            `${domain}_set_action`,
+            `${domain}_timing`,
+            `${domain}_set_timing`,
+        ];
+        
+        for (const table of tables) {
+            const info = master.find(r => r.name == table && r.type == 'table');
+            
+            if (!info)
+                throw new Error(`Table ${table} not found, can't create access log store`);
+        }
+        
+        return new AccessLogStoreDB(db, domain, logger);
+    }
+    
+    async store(data: AccessLogStoreData) {
+        const perf_db_start = process.hrtime.bigint();
+        await this.db
+            .transaction()
+            .execute(async (trx) => {
+                const access_log_set_ip_id = await SetString.try_store_value_or_throw<DefaultKyselyAccessLog>(data.ip, `${this.domain}_set_ip`, trx);
+                const access_log_set_ua_id = await SetString.try_store_value_or_throw<DefaultKyselyAccessLog>(data.user_agent, `${this.domain}_set_ua`, trx);
+                const access_log_set_action_id = await SetString.store_value_or_throw<DefaultKyselyAccessLog>(data.action, `${this.domain}_set_action`, trx);
+                
+                const access_log_id = (
+                    await trx
+                        .insertInto(`${this.domain}`)
+                        .returning('id')
+                        .values({
+                            flag: data.flag,
+                            meta: data.meta,
+                            access_log_set_ip_id,
+                            access_log_set_ua_id,
+                            access_log_set_action_id,
+                            ...data.extended,
+                        })
+                        .executeTakeFirstOrThrow()
+                ).id;
+                
+                for (const [ key, value ] of Object.entries(data.timings)) {
+                    const access_log_set_timing_id = await SetString.store_value_or_throw<DefaultKyselyAccessLog>(key, `${this.domain}_set_timing`, trx);
+                    
+                    await trx
+                        .insertInto(`${this.domain}_timing`)
+                        .values({
+                            access_log_id,
+                            access_log_set_timing_id,
+                            timing_ns: value,
+                        })
+                        .executeTakeFirstOrThrow();
+                }
+            });
+        const perf_db_end = process.hrtime.bigint();
+        
+        this.logger?.silly(`AccessLogStoreDB.store`, {
+            runtime: Number(perf_db_end - perf_db_start) / 1e6,
+            data,
+        });
+    }
+}
+
+export class AccessLogStoreDBDummy implements AccessLogStore {
+    static async create(..._: any[]) {
+        return new AccessLogStoreDBDummy();
+    }
+    
+    async store(..._: any[]) { /* NOOP */ }
+}
+
+class AccessLogTransaction<
+    T extends AccessLogInfo,
+> extends ORM {
+    protected readonly access_log: AccessLog<T>;
+    protected committed = false;
+    
+    protected readonly action: string;
+    protected info: AccessLogInfo & T;
+    
+    protected readonly timings_start:    Record<string, bigint> = {};
+    protected readonly timings_duration: Record<string, bigint> = {};
+    
+    protected readonly perf_total_start: bigint;
+    
+    constructor(access_log: AccessLog<T>, action: string, info: AccessLogInfo & T) {
+        super();
+        
+        this.perf_total_start = process.hrtime.bigint();
+        this.access_log = access_log;
+        this.action = action;
+        this.info = info;
+    }
+    
+    perf_start(name: string): this {
+        if (this.committed)
+            return this;
+        
+        this.timings_start[name] = process.hrtime.bigint();
+        return this;
+    }
+    
+    perf_stop(name: string): this {
+        if (this.committed)
+            return this;
+        
+        const start = this.timings_start[name];
+        const end = process.hrtime.bigint();
+        
+        if (!start)
+            return this;
+        
+        this.timings_start[name] = end;
+        this.timings_duration[name] = (this.timings_duration[name] || 0n) + (end - start);
+        return this;
+    }
+    
+    perf(name: string) {
+        return this.timings_duration[name];
+    }
+    
+    update_info(info: AccessLogInfo & T) {
+        this.info = deepmerge<AccessLogInfo & T>(this.info, info);
+    }
+    
+    async commit() {
+        if (this.committed)
+            return;
+        
+        this.committed = true;
+        const { flag, ip, user_agent, ...extended } = this.info;
+        
+        const perf_total = process.hrtime.bigint() - this.perf_total_start;
+        const perf_exclude = perf_total - Object.values(this.timings_duration).reduce((a, b) => a + b, 0n);
+        
+        return this.access_log.store.store({
+            action: this.action,
+            flag: this.access_log.flags.pack(flag ?? {}),
+            timings: {
+                total: Number(perf_total),
+                exclude: Number(perf_exclude),
+                ...this.timings_duration,
+            },
+            ip,
+            user_agent,
+            extended,
+        });
+    }
+}
+
+const accessLogDefaultFlags = {
+    rejected: 0,
+    internal: 1,
+} as const;
+
+type AccessLogInfo = {
+    flag?:       BitFieldsUnpack<typeof accessLogDefaultFlags>
+    ip?:         string | undefined
+    user_agent?: string | undefined
+};
+
+export class AccessLog<
+    T extends AccessLogInfo,
+//     Flags extends FieldFlagsInObj<IntRange<16, 32>>>
+> extends ORM {
+    public readonly flags: BitFields<typeof accessLogDefaultFlags>;
+    public readonly store: AccessLogStore;
+    
+    protected constructor(store: AccessLogStore) {
+        super();
+        
+        this.store = store;
+        this.flags = new BitFields(accessLogDefaultFlags);
+    }
+    
+    static new<T extends Record<string, any>>(options: {
+        store: AccessLogStore
+        // flags: Flags
+    }) {
+        return new AccessLog<T & AccessLogInfo>(options.store);
+    }
+    
+    begin(action: string, info: T) {
+        return new AccessLogTransaction(this, action, info);
+    }
+}

package/src/tables/AccessLog/schema.ts

@@ -0,0 +1,20 @@
+import type { Generated, GeneratedAlways } from 'kysely';
+
+export type TableAccessLog<
+    AdditionalFields extends Record<string, unknown> = Record<string, unknown>,
+> = AdditionalFields & {
+    id:                         Generated<number>
+    flag:                       number          // BitField
+    access_log_set_ip_id:       number | null   // fk, TableSetString, IP
+    access_log_set_ua_id:       number | null   // fk, TableSetString, UserAgent
+    access_log_set_action_id:   number          // fk, TableSetString, Совершенное действие, название endpoint или любой другой индификатор
+    meta:                       string | null   // JSON, дополнительная информация
+    created_at:                 GeneratedAlways<string>
+};
+
+export interface TableAccessLogTiming {
+    id:                         Generated<number>
+    access_log_id:              number // fk, TableAccessLog
+    access_log_set_timing_id:   number // fk, TableSetString
+    timing_ns:                  number // ns, время затраченное на этот этап
+}