@render-foundation/utils 0.0.200 → 0.0.231

package/lib/esm/src/client/pg/v2Client.js

@@ -9,19 +9,21 @@ export const MANUAL_BURN_TABLE = 'manual_burn';
 export const ENTITY_TABLE = 'entity';
 export const SOL_TRANSFER_TABLE = 'sol_transfer1';
 export const LIABILITY_TABLE = 'liability';
+export const LIABILITY_ADJUSTMENT_TABLE = 'liability_adjustment';
+export const LIABILITY_ADJUSTMENT_BATCH_TABLE = 'liability_adjustment_batch';
 export const JOB_ID_TABLE = 'job_id';
 export const JOB_TABLE = 'job';
 export const BURN_TABLE = 'burn';
 export const BRIDGE_TRANSFER_TABLE = 'bridge_transfer';
 export const NETWORK_REVENUE_TABLE = 'network_revenue';
-export const BURN_ADJUSTMENT_TABLE = 'burn_adjustment';
+export const JOB_BURN_ADJUSTMENT_TABLE = 'job_burn_adjustment';
+export const TX_BURN_ADJUSTMENT_TABLE = 'tx_burn_adjustment';
 export const ENTITY_EPOCH_INFO_TABLE = 'entity_epoch_info';
 export const POLYGON_UPGRADE_TABLE = 'polygon_upgrade';
 export const CURRENT_USER_GRANT_SPEND_TABLE = 'current_user_grant_spend';
 export const USER_GRANT_SPEND_TABLE = 'user_grant_spend';
 export const GRANT_BURN_ID_TABLE = 'grant_burn_id';
 export const GRANT_BURN_TABLE = 'grant_burn';
-export const DISPERSED_SETTLEMENT_TABLE = 'dispersed_settlement';
 BigInt.prototype.toJSON = function () {
     return this.toString();
 };
@@ -313,7 +315,11 @@ export const pgClient = (config) => {
                     sig: solTx,
                     executedAt,
                 }, log);
-                const { id } = await db
+                // sol_transfer1 already has UNIQUE(sol_tx_id, from_entity_id,
+                // to_entity_id, symbol). Without ON CONFLICT a retry of the record step
+                // throws a unique violation and wedges the BullMQ job; make it an
+                // idempotent no-op and re-fetch the existing id instead.
+                let transferRow = await db
                     .insertInto(SOL_TRANSFER_TABLE)
                     .values({
                     id: dbId,
@@ -324,9 +330,23 @@ export const pgClient = (config) => {
                     symbol,
                     amount_payed: amountPayed,
                 })
+                    .onConflict((oc) => oc
+                    .columns(['sol_tx_id', 'from_entity_id', 'to_entity_id', 'symbol'])
+                    .doNothing())
                     .returning((eb) => ['id as id'])
-                    .executeTakeFirstOrThrow();
-                transferId = BigInt(id);
+                    .executeTakeFirst();
+                if (!transferRow) {
+                    log.warn(`sol_transfer for sol_tx_id ${solTxId} ${fromEntityId}->${toEntityId} ${symbol} already exists; reusing existing id (idempotent retry)`);
+                    transferRow = await db
+                        .selectFrom(SOL_TRANSFER_TABLE)
+                        .select((eb) => ['id as id'])
+                        .where('sol_tx_id', '=', solTxId)
+                        .where('from_entity_id', '=', fromEntityId)
+                        .where('to_entity_id', '=', toEntityId)
+                        .where('symbol', '=', symbol)
+                        .executeTakeFirstOrThrow();
+                }
+                transferId = BigInt(transferRow.id);
             }
             ids.push({ transferId, liabilityId });
         }
@@ -633,9 +653,36 @@ export const pgClient = (config) => {
                 }
                 return b.as('x');
             }, (join) => join.onRef('y.sol_key', '=', 'x.sol_key'))
+                // Standing per-wallet/channel corrections (migration 36). NO epoch filter — an adjustment is
+                // a permanent delta on outstanding, not tied to a payout window; payments net it via `payed`.
+                .fullJoin((eb) => {
+                let b = eb
+                    .selectFrom(LIABILITY_ADJUSTMENT_TABLE)
+                    .select(({ fn }) => fn.sum(`amount_adjustment`).as('adj'))
+                    .innerJoin(ENTITY_TABLE, `${LIABILITY_ADJUSTMENT_TABLE}.entity_id`, 'entity.id')
+                    .select(['entity.sol_key'])
+                    .groupBy('entity.sol_key');
+                if (f.banned) {
+                    b = b.where('entity.banned', '=', true);
+                }
+                else {
+                    b = b.where('entity.banned', '=', false);
+                }
+                if (f.channels && f.channels.length > 0) {
+                    b = b.where((eb) => eb.or(f.channels.map((c) => eb('liability_adjustment.channel', '=', c))));
+                }
+                else {
+                    b = b.where((eb) => eb.or([
+                        eb('liability_adjustment.channel', '=', 'node_operator'),
+                        eb('liability_adjustment.channel', '=', 'availability'),
+                    ]));
+                }
+                return b.as('a');
+            }, (join) => join.on(sql `"a"."sol_key" = coalesce("y"."sol_key", "x"."sol_key")`))
                 .select(({ fn }) => [
-                fn.coalesce('y.sol_key', 'x.sol_key').as('sol_key'),
-                sql `${fn.coalesce(`x.due`, sql `0`)} -
+                fn.coalesce('y.sol_key', 'x.sol_key', 'a.sol_key').as('sol_key'),
+                sql `${fn.coalesce(`x.due`, sql `0`)} +
+              ${fn.coalesce(`a.adj`, sql `0`)} -
               ${fn.coalesce(`y.payed`, sql `0`)}`.as('out'),
             ])
                 .as('z'))
@@ -658,6 +705,56 @@ export const pgClient = (config) => {
             }
             return out;
         },
+        async addLiabilityAdjustments(batch, rows, log = consoleLogger) {
+            return await db.transaction().execute(async (trx) => {
+                // Upsert the batch by its label (idempotent re-runs; label is the run identity).
+                const b = await trx
+                    .insertInto(LIABILITY_ADJUSTMENT_BATCH_TABLE)
+                    .values({
+                    label: batch.label,
+                    description: batch.description,
+                    created_by: batch.createdBy,
+                })
+                    .onConflict((oc) => oc.column('label').doUpdateSet({
+                    description: batch.description,
+                    created_by: batch.createdBy,
+                }))
+                    .returning('id')
+                    .executeTakeFirstOrThrow();
+                const batchId = BigInt(b.id);
+                let inserted = 0;
+                let skipped = 0;
+                let net = BigInt(0);
+                let positive = BigInt(0);
+                let negative = BigInt(0);
+                for (const r of rows) {
+                    const entityId = await getOrInsertEntity(trx, { solKey: r.solKey }, log);
+                    const res = await trx
+                        .insertInto(LIABILITY_ADJUSTMENT_TABLE)
+                        .values({
+                        entity_id: entityId,
+                        channel: r.channel,
+                        amount_adjustment: r.amountAdjustment,
+                        batch_id: batchId,
+                    })
+                        // unique (entity_id, channel, batch_id) → a re-run of the same batch is a no-op
+                        .onConflict((oc) => oc.columns(['entity_id', 'channel', 'batch_id']).doNothing())
+                        .executeTakeFirst();
+                    if ((res.numInsertedOrUpdatedRows ?? BigInt(0)) > BigInt(0))
+                        inserted++;
+                    else
+                        skipped++;
+                    net += r.amountAdjustment;
+                    if (r.amountAdjustment > BigInt(0))
+                        positive += r.amountAdjustment;
+                    else
+                        negative += r.amountAdjustment;
+                }
+                log.info(`liability_adjustment batch '${batch.label}' (id ${batchId}): ` +
+                    `inserted ${inserted}, skipped ${skipped}, net ${net} (+${positive} / ${negative})`);
+                return { batchId, inserted, skipped, net, positive, negative };
+            });
+        },
         async fetchNodeOperatorEpoch(epochId, log = consoleLogger) {
             const q = db
                 .selectFrom(JOB_ID_TABLE)
@@ -942,7 +1039,7 @@ export const pgClient = (config) => {
             let buyBurnId;
             let grantBurnId;
             if (p.buyBurn) {
-                const { burned, usdcSpent, eurToUsdc, renderToUsdc, quoteAmt, tags, burnAdjustmentId, toBurn, } = p.buyBurn;
+                const { burned, usdcSpent, eurToUsdc, renderToUsdc, quoteAmt, tags, jobBurnAdjustmentId, txBurnAdjustmentId, toBurn, fromEscrowCredit, } = p.buyBurn;
                 let pricedAt = p.buyBurn.pricedAt;
                 if (!pricedAt) {
                     const jobs = await fetchJobs1(trx.trx, { ids: p.jobs.map((j) => String(j.id)) }, log);
@@ -961,17 +1058,30 @@ export const pgClient = (config) => {
                     quote_amt: quoteAmt,
                     burned: burned,
                     tags: tags.join(','),
-                    burn_adjustment_id: burnAdjustmentId,
+                    job_burn_adjustment_id: jobBurnAdjustmentId,
+                    tx_burn_adjustment_id: txBurnAdjustmentId,
                     to_burn: toBurn,
+                    from_escrow_credit: fromEscrowCredit ?? 0,
                 };
                 log.debug(`inserting burn ${JSON.stringify(v)}`);
-                const { id } = await trx.trx
+                // Idempotent on sol_tx_id (uniq_burn_sol_tx_id, migration 27): a retry of
+                // the record step for the same on-chain burn is a safe no-op. On conflict
+                // executeTakeFirst() returns undefined, so re-fetch the existing id.
+                let burnRow = await trx.trx
                     .insertInto(BURN_TABLE)
                     .values(v)
-                    //.onConflict((oc) => oc.column('sol_tx').doNothing())
+                    .onConflict((oc) => oc.column('sol_tx_id').doNothing())
                     .returning((eb) => ['id as id'])
-                    .executeTakeFirstOrThrow();
-                buyBurnId = Number(id);
+                    .executeTakeFirst();
+                if (!burnRow) {
+                    log.warn(`burn for sol_tx_id ${solTxId} already exists; reusing existing id (idempotent retry)`);
+                    burnRow = await trx.trx
+                        .selectFrom(BURN_TABLE)
+                        .select((eb) => ['id as id'])
+                        .where('sol_tx_id', '=', solTxId)
+                        .executeTakeFirstOrThrow();
+                }
+                buyBurnId = Number(burnRow.id);
             }
             if (p.grantBurn) {
                 const { burned, tags } = p.grantBurn;
@@ -981,12 +1091,22 @@ export const pgClient = (config) => {
                     tags: tags?.join(','),
                 };
                 log.debug(`inserting grant burn ${JSON.stringify(v)}`);
-                const { id } = await trx.trx
+                // Idempotent on sol_tx_id (uniq_grant_burn_sol_tx_id, migration 27).
+                let grantBurnRow = await trx.trx
                     .insertInto(GRANT_BURN_TABLE)
                     .values(v)
+                    .onConflict((oc) => oc.column('sol_tx_id').doNothing())
                     .returning((eb) => ['id as id'])
-                    .executeTakeFirstOrThrow();
-                grantBurnId = Number(id);
+                    .executeTakeFirst();
+                if (!grantBurnRow) {
+                    log.warn(`grant_burn for sol_tx_id ${solTxId} already exists; reusing existing id (idempotent retry)`);
+                    grantBurnRow = await trx.trx
+                        .selectFrom(GRANT_BURN_TABLE)
+                        .select((eb) => ['id as id'])
+                        .where('sol_tx_id', '=', solTxId)
+                        .executeTakeFirstOrThrow();
+                }
+                grantBurnId = Number(grantBurnRow.id);
             }
             const burnId = buyBurnId;
             const grantId = grantBurnId ?? buyBurnId;
@@ -1477,47 +1597,6 @@ export const pgClient = (config) => {
                 log.debug(`burn id ${id} inserted ${num} of ${p.jobs.length} burn jobs`);
             });
         },
-        async insertDispersedOutbox(p, log = consoleLogger) {
-            const res = await db
-                .insertInto(DISPERSED_SETTLEMENT_TABLE)
-                .values({
-                tx_signature: p.txSignature,
-                claim_uuid: p.claimUuid,
-                obligation_uuids: p.obligationUuids,
-                settled_amount: p.settledAmount,
-                burned_amount: p.burnedAmount,
-                executed_at: p.executedAt,
-                eur_render: p.eurRender ?? null,
-            })
-                .onConflict((oc) => oc.column('tx_signature').doNothing())
-                .executeTakeFirst();
-            log.debug(`dispersed outbox tx ${p.txSignature}: ${res.numInsertedOrUpdatedRows} row(s)`);
-        },
-        async markDispersedSettled(txSignature, p, log = consoleLogger) {
-            await db
-                .updateTable(DISPERSED_SETTLEMENT_TABLE)
-                .set({ settled_at: new Date(), idempotent_replay: p.idempotentReplay })
-                .where('tx_signature', '=', txSignature)
-                .execute();
-            log.debug(`dispersed settled tx ${txSignature}`);
-        },
-        async getUnsettledDispersed(log = consoleLogger) {
-            const rows = await db
-                .selectFrom(DISPERSED_SETTLEMENT_TABLE)
-                .selectAll()
-                .where('settled_at', 'is', null)
-                .execute();
-            log.debug(`dispersed outbox: ${rows.length} unsettled`);
-            return rows.map((r) => ({
-                txSignature: r.tx_signature,
-                claimUuid: r.claim_uuid,
-                obligationUuids: r.obligation_uuids,
-                settledAmount: r.settled_amount,
-                burnedAmount: r.burned_amount,
-                executedAt: r.executed_at,
-                eurRender: r.eur_render ?? undefined,
-            }));
-        },
         async fetchJobs(db, f, log = consoleLogger) {
             return await fetchJobs1(db.trx, f, log);
         },
@@ -1581,45 +1660,85 @@ export const pgClient = (config) => {
                     : {},
             }));
         },
-        async getBurnAdjustments(trx, ps, log = consoleLogger) {
-            if (ps.toFill) {
-                const q = trx.trx
-                    .selectFrom(BURN_ADJUSTMENT_TABLE)
-                    .selectAll(BURN_ADJUSTMENT_TABLE)
-                    .innerJoin(JOB_TABLE, `${JOB_TABLE}.id`, `${BURN_ADJUSTMENT_TABLE}.job_id`)
-                    .select([`${JOB_TABLE}.completed_at`, `${JOB_TABLE}.render_amt`])
-                    .leftJoin(BURN_TABLE, `${BURN_TABLE}.burn_adjustment_id`, `${BURN_ADJUSTMENT_TABLE}.id`)
-                    .select(({ fn, lit }) => [
-                    fn
-                        .sum(fn.coalesce(`${BURN_TABLE}.to_burn`, lit(0)))
-                        .as('tot_to_burn'),
-                    fn
-                        .sum(fn.coalesce(`${BURN_TABLE}.burned`, lit(0)))
-                        .as('tot_burned'),
-                ])
-                    .groupBy([
-                    `${BURN_ADJUSTMENT_TABLE}.id`,
-                    `${JOB_TABLE}.completed_at`,
-                    `${JOB_TABLE}.render_amt`,
-                ])
-                    .havingRef(sql `SUM(COALESCE("burn"."to_burn", 0)) - SUM(COALESCE("burn"."burned", 0))`, '<', `${BURN_ADJUSTMENT_TABLE}.down_adj_to_burn`);
-                log.info(`getBurnAdjustments ${q.compile().sql}`);
-                const res = await q.execute();
-                return res.map((r) => ({
-                    id: Number(r.id),
-                    createdAt: r.created_at,
-                    jobId: Number(r.job_id),
-                    downAdjRndrUsed: Number(r.down_adj_rndr_used),
-                    downAdjToBurn: Number(r.down_adj_to_burn),
-                    adjusted: Number(r.tot_to_burn) - Number(r.tot_burned),
-                    job: {
-                        id: Number(r.job_id),
-                        completedAt: r.completed_at,
-                        rndrUsed: BigInt(r.render_amt),
-                    },
-                }));
+        async createTxBurnAdjustment(trx, p, log = consoleLogger) {
+            // Idempotent per sol_tx_id (unique) — a re-run for the same bad tx is a no-op.
+            const row = await trx.trx
+                .insertInto(TX_BURN_ADJUSTMENT_TABLE)
+                .values({
+                sol_tx_id: p.solTxId,
+                surplus_render: p.surplusRender,
+                description: p.description,
+            })
+                .onConflict((oc) => oc.column('sol_tx_id').doNothing())
+                .returning('id')
+                .executeTakeFirst();
+            if (!row) {
+                log.info(`tx_burn_adjustment for sol_tx ${p.solTxId} already exists — skipped`);
+                return undefined;
             }
-            return [];
+            log.info(`created tx_burn_adjustment ${row.id}: ${p.surplusRender} RENDER surplus (sol_tx ${p.solTxId})`);
+            return Number(row.id);
+        },
+        async getJobBurnAdjustments(trx, ps, log = consoleLogger) {
+            if (!ps.toFill)
+                return [];
+            const q = trx.trx
+                .selectFrom(JOB_BURN_ADJUSTMENT_TABLE)
+                .selectAll(JOB_BURN_ADJUSTMENT_TABLE)
+                .innerJoin(JOB_TABLE, `${JOB_TABLE}.id`, `${JOB_BURN_ADJUSTMENT_TABLE}.job_id`)
+                .select([`${JOB_TABLE}.completed_at`, `${JOB_TABLE}.render_amt`])
+                .leftJoin(BURN_TABLE, `${BURN_TABLE}.job_burn_adjustment_id`, `${JOB_BURN_ADJUSTMENT_TABLE}.id`)
+                .select(({ fn, lit }) => [
+                fn.sum(fn.coalesce(`${BURN_TABLE}.to_burn`, lit(0))).as('tot_to_burn'),
+                fn.sum(fn.coalesce(`${BURN_TABLE}.burned`, lit(0))).as('tot_burned'),
+            ])
+                .groupBy([
+                `${JOB_BURN_ADJUSTMENT_TABLE}.id`,
+                `${JOB_TABLE}.completed_at`,
+                `${JOB_TABLE}.render_amt`,
+            ]);
+            log.info(`getJobBurnAdjustments ${q.compile().sql}`);
+            const res = await q.execute();
+            return res
+                .map((r) => ({
+                id: Number(r.id),
+                createdAt: r.created_at,
+                jobId: Number(r.job_id),
+                downAdjRndrUsed: Number(r.down_adj_rndr_used),
+                downAdjToBurn: Number(r.down_adj_to_burn),
+                adjusted: Number(r.tot_to_burn) - Number(r.tot_burned),
+                job: {
+                    id: Number(r.job_id),
+                    completedAt: r.completed_at,
+                    rndrUsed: BigInt(r.render_amt),
+                },
+            }))
+                .filter((a) => a.adjusted < a.downAdjToBurn);
+        },
+        async getTxBurnAdjustments(trx, ps, log = consoleLogger) {
+            if (!ps.toFill)
+                return [];
+            const q = trx.trx
+                .selectFrom(TX_BURN_ADJUSTMENT_TABLE)
+                .selectAll(TX_BURN_ADJUSTMENT_TABLE)
+                .leftJoin(BURN_TABLE, `${BURN_TABLE}.tx_burn_adjustment_id`, `${TX_BURN_ADJUSTMENT_TABLE}.id`)
+                .select(({ fn, lit }) => [
+                fn
+                    .sum(fn.coalesce(`${BURN_TABLE}.from_escrow_credit`, lit(0)))
+                    .as('tot_from_escrow'),
+            ])
+                .groupBy(`${TX_BURN_ADJUSTMENT_TABLE}.id`);
+            log.info(`getTxBurnAdjustments ${q.compile().sql}`);
+            const res = await q.execute();
+            return res
+                .map((r) => ({
+                id: Number(r.id),
+                createdAt: r.created_at,
+                solTxId: Number(r.sol_tx_id),
+                surplusRender: Number(r.surplus_render),
+                consumed: Number(r.tot_from_escrow),
+            }))
+                .filter((a) => a.consumed < a.surplusRender);
         },
         async insertPolygonUpgrade(db, p, log = consoleLogger) {
             const solTxId = await getOrInsertTx(db.trx, {