@moneypot/hub 1.19.1 → 1.19.4

package/README.md

@@ -80,6 +80,7 @@ You should always keep your hub server up to date as soon as possible. Never ins
 ### 1.19.x
 
 - All of hub's internal graphql-schema-modifying plugins have been moved to hub's set of required plugins.
+- Added `FAILED` take request auto-handling
 
 ### 1.18.x
 

package/dist/src/db/types.d.ts

@@ -114,6 +114,8 @@ export type DbTakeRequest = {
     mp_transfer_status: DbTransferStatusKind | null;
     transfer_needs_completion: boolean;
     transfer_completion_attempted_at: Date | null;
+    transfer_failure_count: number;
+    transfer_first_failure_at: Date | null;
     refunded_at: Date | null;
     updated_at: Date;
 };

package/dist/src/pg-versions/014-take-request-transfer-retry.sql

@@ -0,0 +1,11 @@
+ALTER TABLE hub.take_request
+ADD COLUMN IF NOT EXISTS transfer_failure_count int DEFAULT 0,
+ADD COLUMN IF NOT EXISTS transfer_first_failure_at timestamptz;
+
+CREATE INDEX IF NOT EXISTS take_request_transfer_retry_idx
+ON hub.take_request(casino_id, status, transfer_first_failure_at)
+WHERE mp_transfer_id IS NULL AND status = 'PROCESSING';
+
+CREATE INDEX IF NOT EXISTS take_request_failed_refund_idx
+ON hub.take_request(casino_id, status, refunded_at)
+WHERE mp_transfer_id IS NULL AND status = 'FAILED' AND refunded_at IS NULL;

package/dist/src/process-transfers/websocket-processor.js

@@ -61,6 +61,11 @@ export function startWebsocketProcessor({ casinoId, graphqlUrl, signal, controll
             return true;
         },
         keepAlive: 1_000,
+        retryWait: async (retries) => {
+            const baseDelay = Math.min(1000 * Math.pow(2, retries), 30_000);
+            const jitter = Math.floor(Math.random() * 10_000);
+            await new Promise((resolve) => setTimeout(resolve, baseDelay + jitter));
+        },
     });
     client.on("connected", () => {
         logger.info(`[websocketProcessor] Connected to websocket ${graphqlUrl}`);

package/dist/src/server/graphile.config.d.ts

@@ -37,4 +37,7 @@ export declare function createPreset({ plugins, exportSchemaSDLPath, extraPgSche
     context: ServerContext;
     enableChat: boolean;
     enablePlayground: boolean;
-}): GraphileConfig.Preset;
+}): {
+    preset: GraphileConfig.Preset;
+    pgService: GraphileConfig.PgServiceConfiguration<"@dataplan/pg/adaptors/pg">;
+};

package/dist/src/server/graphile.config.js

@@ -91,15 +91,14 @@ export function createPreset({ plugins, exportSchemaSDLPath, extraPgSchemas, abo
     if (enableChat) {
         mutablePlugins.push(HubChatCreateUserMessagePlugin, HubChatCreateSystemMessagePlugin, HubChatSubscriptionPlugin, HubChatMuteUserPlugin, HubChatUnmuteUserPlugin, HubChatAfterIdConditionPlugin, HubChatModManagementPlugin);
     }
+    const pgService = makePgService({
+        connectionString: config.DATABASE_URL,
+        schemas: [...extraPgSchemas, "hub"],
+    });
     const preset = {
         extends: [PostGraphileAmberPreset],
         disablePlugins: ["NodePlugin"],
-        pgServices: [
-            makePgService({
-                connectionString: config.DATABASE_URL,
-                schemas: [...extraPgSchemas, "hub"],
-            }),
-        ],
+        pgServices: [pgService],
         schema: {
             dontSwallowErrors: true,
             exportSchemaSDLPath,
@@ -175,7 +174,7 @@ export function createPreset({ plugins, exportSchemaSDLPath, extraPgSchemas, abo
         },
         plugins: mutablePlugins,
     };
-    return preset;
+    return { preset, pgService };
 }
 function getSessionIdFromWebsocketCtx(ws) {
     const value = ws.normalizedConnectionParams?.authorization;

package/dist/src/server/index.js

@@ -46,7 +46,7 @@ function createExpressServer(context) {
 }
 export function createHubServer({ configureApp, plugins, exportSchemaSDLPath, extraPgSchemas, abortSignal, context, enableChat, enablePlayground, port, }) {
     const expressServer = createExpressServer(context);
-    const preset = createPreset({
+    const { preset, pgService } = createPreset({
         plugins: plugins ?? defaultPlugins,
         exportSchemaSDLPath,
         extraPgSchemas: extraPgSchemas ?? [],
@@ -78,10 +78,12 @@ export function createHubServer({ configureApp, plugins, exportSchemaSDLPath, ex
             });
         },
         shutdown: async () => {
+            nodeServer.closeAllConnections();
+            await pgService.release?.();
+            await pgl.release();
             await new Promise((resolve) => {
                 nodeServer.close(() => resolve());
             });
-            await pgl.release();
         },
     };
 }

package/dist/src/take-request/process-take-request.d.ts

@@ -25,3 +25,8 @@ export declare function completeTransfer({ mpTakeRequestId, takeRequestId, mpTra
     casinoId: string;
     pool: pg.Pool;
 }): Promise<void>;
+export declare function processFailedRequestRefunds({ casinoId, abortSignal, pool, }: {
+    casinoId: string;
+    abortSignal: AbortSignal;
+    pool: pg.Pool;
+}): Promise<void>;

package/dist/src/take-request/process-take-request.js

@@ -193,6 +193,7 @@ export async function processTakeRequests({ abortSignal, controllerId, casinoId,
         pool,
     });
     await processStuckRequests({ casinoId, graphqlClient, abortSignal, pool });
+    await processFailedRequestRefunds({ casinoId, abortSignal, pool });
 }
 async function fetchPendingTakeRequests(graphqlClient, controllerId) {
     const result = await graphqlClient.request(MP_PAGINATE_PENDING_TAKE_REQUESTS, {
@@ -382,6 +383,7 @@ async function attemptTransfer({ pgClient, takeRequestId, mpTakeRequestId, mpExp
         }
         const result = transferResult.transferCurrencyExperienceToUser.result;
         let transferId = null;
+        let needsImmediateRefund = false;
         let resultStatus = LocalTakeRequestStatus.PROCESSING;
         switch (result.__typename) {
             case "TransferSuccess":
@@ -400,6 +402,7 @@ async function attemptTransfer({ pgClient, takeRequestId, mpTakeRequestId, mpExp
                 }
                 else {
                     resultStatus = LocalTakeRequestStatus.FAILED;
+                    needsImmediateRefund = true;
                 }
                 break;
             }
@@ -409,8 +412,8 @@ async function attemptTransfer({ pgClient, takeRequestId, mpTakeRequestId, mpExp
         await pgClient.query({
             text: `
         UPDATE hub.take_request
-        SET status = $1, 
-            mp_transfer_id = $2, 
+        SET status = $1,
+            mp_transfer_id = $2,
             mp_transfer_status = $3,
             transfer_needs_completion = $4,
             mp_status = $5
@@ -427,18 +430,61 @@ async function attemptTransfer({ pgClient, takeRequestId, mpTakeRequestId, mpExp
                 takeRequestId,
             ],
         });
+        if (needsImmediateRefund) {
+            logger.info({ takeRequestId }, `[attemptTransfer] Take request already terminal on MP, refunding`);
+            await refundTakeRequest(pgClient, takeRequestId);
+        }
         return transferId;
     }
     catch (error) {
-        logger.error(error, `[attemptTransfer] Error`);
+        logger.error(error, `[attemptTransfer] Error creating transfer`);
+        const errorMessage = error instanceof Error ? error.message : "Unknown error";
+        const shouldRetry = error instanceof Error && isNetworkError(error);
         await pgClient.query({
             text: `
         UPDATE hub.take_request
-        SET status = $1
-        WHERE id = $2
+        SET transfer_failure_count = COALESCE(transfer_failure_count, 0) + 1,
+            transfer_first_failure_at = COALESCE(transfer_first_failure_at, now()),
+            debug = COALESCE(debug, '') || ' | Transfer attempt failed: ' || $2::text
+        WHERE id = $1
       `,
-            values: [LocalTakeRequestStatus.FAILED, takeRequestId],
+            values: [takeRequestId, errorMessage.slice(0, 200)],
         });
+        if (!shouldRetry) {
+            logger.info({ takeRequestId }, `[attemptTransfer] Non-retryable error, marking as FAILED and refunding`);
+            await pgClient.query({
+                text: `
+          UPDATE hub.take_request
+          SET status = $1,
+              debug = COALESCE(debug, '') || ' | Non-retryable error, refunding'
+          WHERE id = $2
+        `,
+                values: [LocalTakeRequestStatus.FAILED, takeRequestId],
+            });
+            await refundTakeRequest(pgClient, takeRequestId);
+            return null;
+        }
+        const TRANSFER_RETRY_TIMEOUT_SECONDS = 30;
+        const takeRequestAfterAttempt = await pgClient
+            .query({
+            text: `SELECT transfer_first_failure_at FROM hub.take_request WHERE id = $1`,
+            values: [takeRequestId],
+        })
+            .then(exactlyOneRow);
+        const elapsedMs = Date.now() - takeRequestAfterAttempt.transfer_first_failure_at.getTime();
+        if (elapsedMs >= TRANSFER_RETRY_TIMEOUT_SECONDS * 1000) {
+            logger.error({ takeRequestId, elapsedMs }, `[attemptTransfer] Retry timeout (${TRANSFER_RETRY_TIMEOUT_SECONDS}s) reached, marking as FAILED and refunding`);
+            await pgClient.query({
+                text: `
+          UPDATE hub.take_request
+          SET status = $1,
+              debug = COALESCE(debug, '') || ' | Transfer retry timeout reached, refunding'
+          WHERE id = $2
+        `,
+                values: [LocalTakeRequestStatus.FAILED, takeRequestId],
+            });
+            await refundTakeRequest(pgClient, takeRequestId);
+        }
         return null;
     }
 }
@@ -720,7 +766,10 @@ async function processStuckRequests({ casinoId, graphqlClient, abortSignal, pool
       WHERE casino_id = $1
         AND status = $2
         AND mp_transfer_id IS NULL
-        AND updated_at < now() - interval '5 minutes'
+        AND (
+          transfer_first_failure_at IS NULL
+          OR transfer_first_failure_at < now() - interval '10 seconds'
+        )
       LIMIT 10
     `,
         values: [casinoId, LocalTakeRequestStatus.PROCESSING],
@@ -741,6 +790,41 @@ async function processStuckRequests({ casinoId, graphqlClient, abortSignal, pool
         });
     }
 }
+export async function processFailedRequestRefunds({ casinoId, abortSignal, pool, }) {
+    if (abortSignal.aborted) {
+        return;
+    }
+    const failedRequests = await pool
+        .query({
+        text: `
+      SELECT id, mp_take_request_id
+      FROM hub.take_request
+      WHERE casino_id = $1
+        AND status = $2
+        AND mp_transfer_id IS NULL
+        AND refunded_at IS NULL
+        AND reserved_amount > 0
+      LIMIT 10
+    `,
+        values: [casinoId, LocalTakeRequestStatus.FAILED],
+    })
+        .then((result) => result.rows);
+    if (failedRequests.length > 0) {
+        logger.info(`[processFailedRequestRefunds] Found ${failedRequests.length} FAILED requests needing refunds`);
+    }
+    for (const request of failedRequests) {
+        if (abortSignal.aborted) {
+            break;
+        }
+        await withPgPoolTransaction(pool, async (pgClient) => {
+            await PgAdvisoryLock.forMpTakeRequestProcessing(pgClient, {
+                mpTakeRequestId: request.mp_take_request_id,
+                casinoId,
+            });
+            await refundTakeRequest(pgClient, request.id);
+        });
+    }
+}
 async function loadRequiredEntities(pgClient, params) {
     const { type, currencyKey, casinoId } = params;
     const dbCurrency = await pgClient
@@ -926,6 +1010,90 @@ async function updateTakeRequestStatus(pgClient, takeRequestId, status) {
         values: [status, takeRequestId],
     });
 }
+async function refundTakeRequest(pgClient, takeRequestId) {
+    const takeRequestData = await pgClient
+        .query({
+        text: `
+        SELECT *
+        FROM hub.take_request
+        WHERE id = $1
+        FOR UPDATE
+      `,
+        values: [takeRequestId],
+    })
+        .then(exactlyOneRow);
+    if (takeRequestData.refunded_at) {
+        logger.debug({ takeRequestId }, `[refundTakeRequest] Already refunded, skipping`);
+        return false;
+    }
+    if (takeRequestData.reserved_amount <= 0) {
+        logger.debug({ takeRequestId }, `[refundTakeRequest] No reserved amount to refund`);
+        return false;
+    }
+    const dbLockedBalance = await pgClient
+        .query({
+        text: `
+        SELECT id, amount
+        FROM hub.balance
+        WHERE user_id = $1
+          AND experience_id = $2
+          AND casino_id = $3
+          AND currency_key = $4
+        FOR UPDATE
+      `,
+        values: [
+            takeRequestData.user_id,
+            takeRequestData.experience_id,
+            takeRequestData.casino_id,
+            takeRequestData.currency_key,
+        ],
+    })
+        .then(maybeOneRow);
+    if (!dbLockedBalance) {
+        logger.error({ takeRequestId, takeRequestData }, `[refundTakeRequest] Balance record not found - cannot refund`);
+        await pgClient.query({
+            text: `
+        UPDATE hub.take_request
+        SET debug = COALESCE(debug, '') || ' | Refund failed: balance record not found'
+        WHERE id = $1
+      `,
+            values: [takeRequestId],
+        });
+        return false;
+    }
+    await pgClient.query({
+        text: `
+      UPDATE hub.balance
+      SET amount = amount + $1
+      WHERE id = $2
+    `,
+        values: [takeRequestData.reserved_amount, dbLockedBalance.id],
+    });
+    await insertAuditLog(pgClient, "player-balance", {
+        balanceId: dbLockedBalance.id,
+        balanceOld: dbLockedBalance.amount,
+        balanceNew: dbLockedBalance.amount + takeRequestData.reserved_amount,
+        balanceDelta: takeRequestData.reserved_amount,
+        action: "hub:take_request:refund",
+        refType: "hub.take_request",
+        refId: takeRequestId,
+    });
+    await pgClient.query({
+        text: `
+      UPDATE hub.take_request
+      SET refunded_at = now(),
+          debug = COALESCE(debug, '') || $2,
+          updated_at = now()
+      WHERE id = $1
+    `,
+        values: [
+            takeRequestId,
+            ` | Refunded ${takeRequestData.reserved_amount} base units of ${takeRequestData.currency_key}`,
+        ],
+    });
+    logger.info({ takeRequestId, amount: takeRequestData.reserved_amount }, `[refundTakeRequest] Successfully refunded`);
+    return true;
+}
 function isNetworkError(e) {
     return (e.message?.includes("ECONNREFUSED") ||
         e.message?.includes("ECONNRESET") ||

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moneypot/hub",
-  "version": "1.19.1",
+  "version": "1.19.4",
   "author": "moneypot.com",
   "homepage": "https://moneypot.com/hub",
   "keywords": [
@@ -48,47 +48,50 @@
     "build-dashboard": "cd ./dashboard && pnpm run build && rm -rf ../dist/dashboard && cp -Rv dist ../dist/dashboard",
     "test:startup": "dropdb --if-exists hub-test && createdb hub-test && SUPERUSER_DATABASE_URL=postgres://localhost:5432/hub-test DATABASE_URL=postgres://localhost:5432/hub-test tsx src/test-startup.ts",
     "test": "vitest run",
-    "test:watch": "vitest"
+    "test:watch": "vitest",
+    "test:bail": "pnpm test -- --bail --silent --reporter=basic 2>&1 | head -25"
   },
   "dependencies": {
-    "@graphile-contrib/pg-omit-archived": "^4.0.0-rc.1",
+    "@graphile-contrib/pg-omit-archived": "4.0.0-rc.1",
     "@moneypot/hash-herald": "^1.0.0",
     "@moneypot/pg-upgrade-schema": "^2.1.0",
-    "@noble/curves": "^1.5.0",
     "dotenv": "^17.2.3",
-    "express": "^5.0.1",
-    "graphql": "^16.8.1",
-    "graphql-request": "^7.0.0",
-    "graphql-ws": "^6.0.5",
-    "jose": "^6.0.11",
-    "pg": "^8.12.0",
-    "pg-connection-string": "^2.6.4",
+    "express": "^5.2.1",
+    "graphql": "^16.12.0",
+    "graphql-request": "^7.4.0",
+    "graphql-ws": "^6.0.6",
+    "jose": "^6.1.3",
+    "pg": "^8.16.3",
+    "pg-connection-string": "^2.9.1",
     "pino": "^10.1.0",
-    "postgraphile": "^5.0.0-rc.3",
-    "tsafe": "^1.6.6",
-    "zod": "^4.1.12"
+    "postgraphile": "5.0.0-rc.3",
+    "tsafe": "^1.8.12",
+    "zod": "^4.1.13"
   },
   "devDependencies": {
-    "@eslint/js": "^9.8.0",
+    "@eslint/js": "^9.39.2",
     "@graphql-codegen/cli": "^6.1.0",
     "@graphql-codegen/client-preset": "^5.2.1",
     "@graphql-typed-document-node/core": "^3.2.0",
-    "@types/express": "^5.0.0",
-    "@types/node": "^25.0.0",
-    "@types/pg": "^8.11.5",
-    "@types/supertest": "^6.0.2",
-    "eslint": "^9.8.0",
-    "globals": "^16.0.0",
-    "pino-pretty": "^13.0.0",
-    "prettier": "^3.6.2",
-    "supertest": "^7.0.0",
-    "tsx": "^4.20.3",
-    "typescript": "^5.4.5",
-    "typescript-eslint": "^8.0.1",
+    "@types/express": "^5.0.6",
+    "@types/node": "^25.0.1",
+    "@types/pg": "^8.16.0",
+    "@types/supertest": "^6.0.3",
+    "eslint": "^9.39.2",
+    "globals": "^16.5.0",
+    "pino-pretty": "^13.1.3",
+    "prettier": "^3.7.4",
+    "supertest": "^7.1.4",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.49.0",
     "vitest": "^4.0.15"
   },
   "peerDependencies": {
     "graphile-config": "^1.0.0-rc.2"
   },
+  "optionalDependencies": {
+    "@moneypot/server": "file:../moneypot-server"
+  },
   "license": "UNLICENSED"
 }