@render-foundation/utils 0.0.59 → 0.0.61

package/lib/esm/src/client/pg/v2.js

@@ -0,0 +1,653 @@
+import { Pool } from 'pg';
+import { Kysely, NoResultError, PostgresDialect, sql, } from 'kysely';
+import { consoleLogger } from '../../logger';
+const transferTypToChan = (t) => {
+    if (t == 'open_compute_rewards') {
+        return 'open_compute';
+    }
+    if (t == 'node_operator_rewards') {
+        return 'node_operator';
+    }
+    return 'upgrade';
+};
+const transferFromDBV2 = (o) => {
+    return {
+        id: o.id,
+        solKey: o.sol_key,
+        solTx: o.sig,
+        sqTx: o.sq_tx,
+        createdAt: o.created_at,
+        payedAt: o.executed_at,
+        amountDue: BigInt(o.amount_due),
+        amountPayed: o.amount_payed ? BigInt(o.amount_payed) : undefined,
+        operatorRewards: o.channel == 'node_operator'
+            ? {
+                epochId: o.ui,
+                startTime: o.start_time,
+                endTime: o.end_time,
+            }
+            : undefined,
+        computeClientRewards: o.transfer_type == 'open_compute'
+            ? {
+                epochId: o.ui,
+                deviceId: o.device_id,
+            }
+            : undefined,
+        upgradeRewards: o.transfer_type == 'upgrade'
+            ? {
+                epochId: o.ui,
+                epochStartTime: o.start_time,
+                points: BigInt(o.points),
+            }
+            : undefined,
+    };
+};
+export const SOL_TX_TABLE = 'sol_tx';
+export const EPOCH_TABLE = 'epoch';
+export const MANUAL_BURN_TABLE = 'manual_burn';
+export const ENTITY_TABLE = 'entity';
+export const SOL_TRANSFER_TABLE = 'sol_transfer';
+export const SOL_TRANSFER_INFO_TABLE = 'sol_transfer_info';
+export const JOB_ID_TABLE = 'job_id';
+export const JOB_TABLE = 'job';
+export const BURN_TABLE = 'burn';
+export const pgClient = (config) => {
+    const dialect = new PostgresDialect({
+        pool: new Pool({
+            ...config,
+            port: config.port ?? 5432,
+            max: config.max ?? 10,
+        }),
+    });
+    // Database interface is passed to Kysely's constructor, and from now on, Kysely
+    // knows your database structure.
+    // Dialect is passed to Kysely's constructor, and from now on, Kysely knows how
+    // to communicate with your database.
+    const db = new Kysely({
+        dialect,
+    });
+    const getOrInsertEntity = async (db, p, log = consoleLogger) => {
+        const res = (await db
+            .selectFrom(ENTITY_TABLE)
+            .select('id')
+            .where('sol_key', '=', p.solKey)
+            .executeTakeFirst());
+        if (res) {
+            log.debug(`already found entity ${res.id}`);
+            return res.id;
+        }
+        const { id } = (await db
+            .insertInto(ENTITY_TABLE)
+            .values({
+            sol_key: p.solKey,
+        })
+            .onConflict((oc) => oc.column('sol_key').doNothing())
+            .returning((eb) => ['id as id'])
+            .executeTakeFirst());
+        log.debug(`inserted entity ${id}`);
+        return id;
+    };
+    const getOrInsertTx = async (db, p, log = consoleLogger) => {
+        const res = await db
+            .selectFrom(SOL_TX_TABLE)
+            .select('id')
+            .where('sig', '=', p.sig)
+            .executeTakeFirst();
+        if (res) {
+            log.debug(`already found tx ${res.id}`);
+            return res.id;
+        }
+        const { id } = (await db
+            .insertInto(SOL_TX_TABLE)
+            .values({
+            sig: p.sig,
+            executed_at: p.executedAt,
+        })
+            .onConflict((oc) => oc.column('sig').doNothing())
+            .returning((eb) => ['id as id'])
+            .executeTakeFirst());
+        log.debug(`inserted tx ${id}`);
+        return id;
+    };
+    const getOrInsertEpoch = async (db, p, log = consoleLogger) => {
+        const res = await db
+            .selectFrom(EPOCH_TABLE)
+            .select('id')
+            .where('ui', '=', String(p.uiEpochId))
+            .where('channel', '=', p.channel)
+            .executeTakeFirst();
+        if (res) {
+            log.debug(`already found epoch ${res.id}`);
+            return res.id;
+        }
+        const { id } = (await db
+            .insertInto(EPOCH_TABLE)
+            .values({
+            ui: p.uiEpochId,
+            channel: p.channel,
+            start_time: p.startTime,
+            end_time: p.endTime,
+            total_rndr: p.totalRndr,
+        })
+            .onConflict((oc) => oc.columns(['ui', 'channel']).doNothing())
+            .returning((eb) => ['id as id'])
+            .executeTakeFirst());
+        log.debug(`inserted epoch ${id}`);
+        return id;
+    };
+    const createTransfers = async (db, transfers, log = consoleLogger, totalRndr = BigInt(0)) => {
+        const epochIds = [];
+        for (const t of transfers) {
+            if (!t.computeClientRewards && !t.upgradeRewards && !t.operatorRewards) {
+                log.warn(`malformed transfer doesnt have type ${JSON.stringify(t, (key, value) => typeof value === 'bigint' ? value.toString() : value)}`);
+                continue;
+            }
+            let solTxId;
+            if (t.solTx) {
+                solTxId = await getOrInsertTx(db, {
+                    sig: t.solTx,
+                    executedAt: t.payedAt,
+                }, log);
+            }
+            let entityId;
+            if (t.solKey) {
+                entityId = await getOrInsertEntity(db, { solKey: t.solKey }, log);
+            }
+            log.info(`inserting transfer ${JSON.stringify(t, (key, value) => typeof value === 'bigint' ? value.toString() : value)}`);
+            const { id: transferId } = (await db
+                .insertInto(SOL_TRANSFER_TABLE)
+                .values({
+                sol_tx_id: solTxId,
+                entity_id: entityId,
+                channel: transferTypToChan((() => {
+                    if (t.operatorRewards) {
+                        return 'node_operator_rewards';
+                    }
+                    else if (t.upgradeRewards) {
+                        return 'upgrade_rewards';
+                    }
+                    return 'open_compute_rewards';
+                })()),
+                created_at: t.createdAt,
+                amount_due: BigInt(t.amountDue),
+                amount_payed: t.amountPayed ? BigInt(t.amountPayed) : 0,
+            })
+                .returning((eb) => ['id as id'])
+                .executeTakeFirst());
+            let epochId;
+            let deviceId;
+            let points;
+            let startTime;
+            let endTime;
+            let channel;
+            if (t.operatorRewards) {
+                log.info(`inserting operator rewards ${transferId}`);
+                epochId = t.operatorRewards.epochId;
+                startTime = t.operatorRewards.startTime;
+                endTime = t.operatorRewards.endTime;
+                channel = 'node_operator';
+            }
+            else if (t.computeClientRewards) {
+                log.info(`inserting compute client rewards ${transferId}`);
+                epochId = t.computeClientRewards.epochId;
+                deviceId = t.computeClientRewards.deviceId;
+                channel = 'open_compute';
+            }
+            else if (t.upgradeRewards) {
+                log.info(`inserting upgrade rewards ${transferId}`);
+                channel = 'upgrade';
+                epochId = t.upgradeRewards.epochId;
+                points = t.upgradeRewards.points;
+            }
+            let epochIdDb;
+            if (epochId != null) {
+                epochIdDb = await getOrInsertEpoch(db, {
+                    channel,
+                    startTime,
+                    endTime,
+                    uiEpochId: epochId,
+                    totalRndr,
+                }, log);
+            }
+            await db
+                .insertInto(SOL_TRANSFER_INFO_TABLE)
+                .values({
+                transfer_id: transferId,
+                device_id: deviceId,
+                points,
+                epoch_id: epochIdDb ? BigInt(epochIdDb) : null,
+            })
+                .executeTakeFirst();
+            epochIds.push({ epochDbId: epochIdDb });
+        }
+        return epochIds;
+    };
+    return {
+        db,
+        async saveNodeOperatorEpoch(epochId, transfers, jobIds, totalRndr, log = consoleLogger) {
+            await db
+                .transaction()
+                .setIsolationLevel('repeatable read')
+                .execute(async (trx) => {
+                const epochDbIds = await createTransfers(trx, transfers, log, totalRndr);
+                log.info(`inserted transfers`);
+                for (const id of jobIds) {
+                    await trx
+                        .insertInto(JOB_ID_TABLE)
+                        .values({
+                        id: BigInt(id),
+                        epoch_id: epochDbIds[0].epochDbId,
+                    })
+                        .executeTakeFirstOrThrow();
+                }
+                log.debug(`saved epoch`);
+                /*const now = new Date(Date.now())
+                const res = await trx
+                  .updateTable(EPOCH_TABLE)
+                  .set({
+                    saved_at: now,
+                    total_rndr: totalRndr,
+                  })
+                  .where('channel')
+                  .where('ui', '=', epochId.toString())
+                  .where('', '=')
+                  .executeTakeFirstOrThrow()*/
+            });
+            return;
+        },
+        async getLatestNodeOperatorEpoch(log = consoleLogger) {
+            const now = new Date(Date.now());
+            try {
+                const res = await db
+                    .selectFrom(EPOCH_TABLE)
+                    .selectAll()
+                    .where('channel', '=', 'node_operator')
+                    .where('end_time', '<=', now)
+                    .where('saved_at', 'is', null)
+                    .orderBy('end_time asc')
+                    .limit(1)
+                    .executeTakeFirstOrThrow();
+                return {
+                    epochId: BigInt(res.ui),
+                    startedAt: res.start_time,
+                    endedAt: res.end_time,
+                    savedAt: res.saved_at ?? undefined,
+                };
+            }
+            catch (e) {
+                if (e instanceof NoResultError) {
+                    log.info(`no node operator epoch found`);
+                    return undefined;
+                }
+                throw e;
+            }
+        },
+        async getOutstanding(f, log = consoleLogger) {
+            const { typ, nullKey, due } = f;
+            //select sol_key, sum(amount_due)-sum(amount_payed) from transfer group by sol_key;
+            let q = db
+                .selectFrom(SOL_TRANSFER_TABLE)
+                .innerJoin(ENTITY_TABLE, 'sol_transfer.entity_id', 'entity.id')
+                .select(({ fn, val, ref }) => [
+                'entity.sol_key',
+                sql `${fn.sum('sol_transfer.amount_due')} -
+          ${fn.sum('sol_transfer.amount_payed')}`.as('due'),
+            ])
+                .where('sol_transfer.channel', '=', transferTypToChan(typ))
+                .groupBy(['entity.sol_key']);
+            if (!nullKey) {
+                q = q.where('entity.sol_key', 'is not', null);
+            }
+            if (due) {
+                q = q.having(({ fn, ref }) => sql `${fn.sum('sol_transfer.amount_due')} -
+            ${fn.sum('sol_transfer.amount_payed')}
+            >
+            0`);
+            }
+            const res = await q.execute();
+            log.info(`outstanding query ${q.compile().sql} params ${JSON.stringify(f)} ${res.length} rows`);
+            const out = {};
+            for (const r of res) {
+                out[r.sol_key ?? ''] = BigInt(r.due);
+            }
+            return out;
+        },
+        async fetchTransfers(f, log = consoleLogger) {
+            let base = db
+                .selectFrom(SOL_TRANSFER_TABLE)
+                .selectAll(SOL_TRANSFER_TABLE)
+                .innerJoin(SOL_TRANSFER_INFO_TABLE, 'sol_transfer_info.transfer_id', 'sol_transfer.id')
+                .select(['device_id', 'points'])
+                .leftJoin(SOL_TX_TABLE, 'sol_tx.id', 'sol_transfer.sol_tx_id')
+                .select(['sig', 'executed_at'])
+                .innerJoin(ENTITY_TABLE, 'entity.id', 'sol_transfer.entity_id')
+                .select(['sol_key'])
+                .innerJoin(EPOCH_TABLE, 'epoch.id', 'sol_transfer_info.epoch_id')
+                .select(['ui', 'start_time', 'end_time'])
+                .where('sol_transfer.channel', '=', transferTypToChan(f.typ));
+            if (f.solKey) {
+                base = base.where('entity.sol_key', '=', f.solKey);
+            }
+            if (f.payed != undefined) {
+                base = base.where('sol_transfer.sol_tx_id', f.payed ? 'is not' : 'is', null);
+            }
+            if (f.createdBefore != undefined) {
+                base = base.where('sol_transfer.created_at', '<=', f.createdBefore);
+            }
+            if (f.epochId) {
+                base = base.where('epoch.ui', '=', String(f.epochId));
+            }
+            if (f.sort) {
+                base = base.orderBy('entity.sol_key asc');
+            }
+            const res = await base.execute();
+            log.info(`fetched ${base.compile().sql} params ${JSON.stringify(f)} ${res.length} transfers`);
+            return res.map((r) => transferFromDBV2(r));
+        },
+        async payoutTransfers(idToAmt, payedAt, solTx, log = consoleLogger) {
+            await db
+                .transaction()
+                .setIsolationLevel('repeatable read')
+                .execute(async (tx) => {
+                const alreadyUpdatedIds = (await tx
+                    .selectFrom(SOL_TRANSFER_TABLE)
+                    .innerJoin(SOL_TX_TABLE, 'sol_transfer.sol_tx_id', 'sol_tx.id')
+                    .select(['sol_transfer.id'])
+                    .where('sol_transfer.id', 'in', Object.keys(idToAmt))
+                    .where('sol_tx.sig', 'is not', null)
+                    .execute()).map(({ id }) => id);
+                for (const [id, amt] of Object.entries(idToAmt)) {
+                    if (alreadyUpdatedIds.includes(id)) {
+                        log.info(`transfer ${id} already marked payed out. skipping`);
+                        continue;
+                    }
+                    const solTxId = await getOrInsertTx(tx, { sig: solTx, executedAt: payedAt }, log);
+                    const res = await tx
+                        .updateTable(SOL_TRANSFER_TABLE)
+                        .set({
+                        sol_tx_id: solTxId,
+                        amount_payed: amt,
+                    })
+                        .where('id', '=', id.toString())
+                        .executeTakeFirstOrThrow();
+                    log.info(`marked ${res.numUpdatedRows} transfer payed out`);
+                }
+            });
+        },
+        async payoutTransfer(id, payedAt, solTx, amtPayed, log = consoleLogger) {
+            await db
+                .transaction()
+                .setIsolationLevel('repeatable read')
+                .execute(async (tx) => {
+                const solTxId = await getOrInsertTx(tx, { sig: solTx, executedAt: payedAt }, log);
+                const res = await tx
+                    .updateTable(SOL_TRANSFER_TABLE)
+                    .set({
+                    sol_tx_id: solTxId,
+                    amount_payed: amtPayed,
+                })
+                    .where('id', '=', id.toString())
+                    .executeTakeFirst();
+                log.info(`marked ${res.numUpdatedRows} transfer payed out`);
+            });
+        },
+        async createTransfers(transfers, log = consoleLogger) {
+            await db
+                .transaction()
+                .setIsolationLevel('repeatable read')
+                .execute(async (trx) => {
+                await createTransfers(trx, transfers, log);
+            });
+        },
+        async fetchInfoForEpoch(epochId, log = consoleLogger) {
+            const q = db
+                .selectFrom(JOB_ID_TABLE)
+                .innerJoin(EPOCH_TABLE, 'epoch.id', 'job_id.epoch_id')
+                .leftJoin(MANUAL_BURN_TABLE, 'manual_burn.epoch_id', 'epoch.id')
+                .where('epoch.ui', '=', String(epochId))
+                .where('epoch.channel', '=', 'node_operator')
+                .select([
+                'job_id.id',
+                'epoch.total_rndr',
+                'epoch.start_time',
+                'epoch.end_time',
+                'manual_burn.burned',
+                'manual_burn.tx_sig',
+            ])
+                .orderBy('job_id.id asc');
+            log.debug(`fetch epoch info q ${q.compile().sql} epochId ${epochId}`);
+            const res = await q.execute();
+            if (res.length == 0) {
+                throw new Error(`epoch ${epochId} not found`);
+            }
+            return {
+                jobIds: res.map((r) => Number(r.id)),
+                totalRndr: BigInt(res[0].total_rndr),
+                startTime: res[0].start_time,
+                endTime: res[0].end_time,
+                burned: BigInt(res[0].burned ?? 0),
+                burnTx: res[0].tx_sig,
+            };
+        },
+        async updateComputeClientSolKey(deviceId, solKey, log = consoleLogger) {
+            const q = db
+                .updateTable(`${ENTITY_TABLE} as e`)
+                .set({
+                sol_key: solKey,
+            })
+                .from(`${SOL_TRANSFER_TABLE} as t`)
+                .innerJoin(SOL_TRANSFER_INFO_TABLE, 'sol_transfer_info.transfer_id', 't.id')
+                //.whereRef('t.id', '=', 'o.transfer_id')
+                .where('e.sol_key', 'is', null)
+                .where('sol_transfer_info.device_id', '=', deviceId);
+            const sql = await q.compile();
+            log.info(`sql ${sql.sql} params ${sql.parameters}`);
+            const res = await q.executeTakeFirstOrThrow();
+            log.info(`updated ${res.numUpdatedRows} device-id sol_keys`);
+        },
+        async markJobsBurnt(trx, p, log = consoleLogger) {
+            const solTxId = await getOrInsertTx(trx, { sig: p.solTx, executedAt: p.executedAt }, log);
+            const { id } = await trx
+                .insertInto(BURN_TABLE)
+                .values({
+                sol_tx_id: solTxId,
+                priced_at: p.pricedAt,
+                usdc_spent: p.usdcSpent,
+                quote_amt: p.quoteAmt,
+                burned: p.burned,
+            })
+                //.onConflict((oc) => oc.column('sol_tx').doNothing())
+                .returning((eb) => ['id as id'])
+                .executeTakeFirstOrThrow();
+            const jobIdsStr = p.jobsIds.map(String);
+            const res = await trx
+                .updateTable(JOB_TABLE)
+                .where('id', 'in', jobIdsStr)
+                .set('burn_id', id)
+                .executeTakeFirstOrThrow();
+            log.debug(`set burn_id on ${res.numUpdatedRows} jobs`);
+            return;
+        },
+        async produceBurnedJobs(jobs, log = consoleLogger) {
+            await db.transaction().execute(async (trx) => {
+                let num = BigInt(0);
+                for (const { amount, completedAt, jobId } of jobs) {
+                    const res = await trx
+                        .insertInto(JOB_TABLE)
+                        .values({
+                        id: jobId,
+                        render_amt: amount,
+                        completed_at: completedAt,
+                        inserted_at: new Date(Date.now()),
+                    })
+                        .onConflict((oc) => oc.column('id').doNothing())
+                        .executeTakeFirstOrThrow();
+                    num += res.numInsertedOrUpdatedRows ?? BigInt(0);
+                }
+                log.debug(`inserted ${num} of ${jobs.length} burn jobs`);
+            });
+        },
+        async markJobsBurned(p, log = consoleLogger) {
+            let { markedAt, solTx } = p;
+            let res = await db
+                .updateTable(BURN_TABLE)
+                .set({ marked_burned_at: markedAt })
+                .from(SOL_TX_TABLE)
+                .whereRef('burn.sol_tx_id', '=', 'sol_tx.id')
+                .where('sol_tx.sig', '=', solTx)
+                .executeTakeFirstOrThrow();
+            log.debug(`set ${res.numUpdatedRows} ${solTx} -> marked_burned_at ${markedAt.toISOString()}`);
+        },
+        async consumeAndLockBurnJobs(limit, lowerRndrThreshold, upperRndrThreshold, log = consoleLogger) {
+            const trx = await begin(db);
+            const q = trx.trx
+                .selectFrom(trx.trx
+                .selectFrom(JOB_TABLE)
+                .select(({ fn, val, ref }) => [
+                'id',
+                fn
+                    .sum('render_amt')
+                    .over((ob) => ob.orderBy(`completed_at`, `asc`))
+                    .as('total'),
+            ])
+                .where('burn_id', 'is', null)
+                .as('t'))
+                .select('id')
+                .where('total', '>', lowerRndrThreshold)
+                .where('total', '<', upperRndrThreshold);
+            log.debug(`outstanding jobs q: ${q.compile().sql} ${JSON.stringify({
+                limit,
+                lowerRndrThreshold,
+                upperRndrThreshold,
+            })}`);
+            const idRs = await q.execute();
+            const ids = idRs.map((x) => x.id);
+            log.debug(`${ids.length} candidate jobs for burn. locking up to ${limit} jobs`);
+            if (ids.length == 0) {
+                return { trx, jobs: [] };
+            }
+            const rows = await trx.trx
+                .selectFrom(JOB_TABLE)
+                .selectAll()
+                .where('id', 'in', ids)
+                .limit(limit)
+                .forUpdate()
+                .skipLocked()
+                .execute();
+            log.debug(`locked ${rows.length} jobs`);
+            const jobs = [];
+            for (const r of rows) {
+                jobs.push({
+                    id: Number(r.id),
+                    completedAt: r.completed_at,
+                    rndrUsed: BigInt(r.render_amt),
+                });
+            }
+            return { trx, jobs };
+        },
+        async incrBuyBurnMetrics(log = consoleLogger) {
+            const res = await db
+                .selectFrom(JOB_TABLE)
+                .select('completed_at')
+                .orderBy('completed_at asc')
+                .where('burn_id', 'is', null)
+                .execute();
+            if (res.length == 0) {
+                return { outstandingJobs: 0 };
+            }
+            return {
+                outstandingJobs: res.length,
+                oldestJobFinishedAt: res[0].completed_at ?? undefined,
+            };
+        },
+        async deleteAll() {
+            if (config.database.includes('render')) {
+                throw new Error('cannot nuke from deployed database. call from tests only');
+            }
+            await Promise.all([
+                db
+                    .deleteFrom(SOL_TX_TABLE)
+                    .where('id', 'is not', null)
+                    .executeTakeFirst(),
+                db
+                    .deleteFrom(SOL_TRANSFER_TABLE)
+                    .where('id', 'is not', null)
+                    .executeTakeFirst(),
+                db
+                    .deleteFrom(SOL_TRANSFER_INFO_TABLE)
+                    .where('transfer_id', 'is not', null)
+                    .executeTakeFirst(),
+                db
+                    .deleteFrom(EPOCH_TABLE)
+                    .where('id', 'is not', null)
+                    .executeTakeFirst(),
+                db
+                    .deleteFrom(ENTITY_TABLE)
+                    .where('id', 'is not', null)
+                    .executeTakeFirst(),
+                db
+                    .deleteFrom(MANUAL_BURN_TABLE)
+                    .where('epoch_id', 'is not', null)
+                    .executeTakeFirst(),
+                db
+                    .deleteFrom(JOB_ID_TABLE)
+                    .where('id', 'is not', null)
+                    .executeTakeFirst(),
+                db.deleteFrom(JOB_TABLE).where('id', 'is not', null).executeTakeFirst(),
+                db
+                    .deleteFrom(BURN_TABLE)
+                    .where('id', 'is not', null)
+                    .executeTakeFirst(),
+            ]);
+        },
+    };
+};
+async function begin(db) {
+    const connection = new Deferred();
+    const result = new Deferred();
+    // Do NOT await this line.
+    db.transaction()
+        .setIsolationLevel('repeatable read')
+        .execute((trx) => {
+        connection.resolve(trx);
+        return result.promise;
+    })
+        .catch((err) => {
+        // Don't do anything here. Just swallow the exception.
+    });
+    const trx = await connection.promise;
+    return {
+        trx,
+        commit() {
+            result.resolve(null);
+        },
+        rollback() {
+            result.reject(new Error('rollback'));
+        },
+    };
+}
+export class Deferred {
+    #promise;
+    #resolve;
+    #reject;
+    constructor() {
+        this.#promise = new Promise((resolve, reject) => {
+            this.#reject = reject;
+            this.#resolve = resolve;
+        });
+    }
+    get promise() {
+        return this.#promise;
+    }
+    resolve = (value) => {
+        if (this.#resolve) {
+            this.#resolve(value);
+        }
+    };
+    reject = (reason) => {
+        if (this.#reject) {
+            this.#reject(reason);
+        }
+    };
+}
+//# sourceMappingURL=v2.js.map