@moneypot/hub 1.21.0 → 1.22.1

package/dist/src/config.js

@@ -5,9 +5,8 @@ function getEnvVariable(key, transform = (value) => value) {
     return transform(process.env[key] || "");
 }
 export const NODE_ENV = getEnvVariable("NODE_ENV", (value) => {
-    if (!value) {
-        console.warn("Missing NODE_ENV env var. Defaulting to 'development'");
-        return "development";
+    if (value !== "development" && value !== "test" && value !== "production") {
+        throw new Error(`NODE_ENV must be "development", "test", or "production" (got ${JSON.stringify(value)})`);
     }
     return value;
 });

package/dist/src/hash-chain/get-hash.js

@@ -4,6 +4,14 @@ const HASH_HERALD_OPTIONS = {
     baseUrl: config.HASHCHAINSERVER_URL,
     applicationSecret: config.HASHCHAINSERVER_APPLICATION_SECRET,
 };
+const HASH_HERALD_TIMEOUT_MS = 10_000;
+function getHashWithTimeout(req) {
+    let timer;
+    const timeout = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error(`hash-herald did not answer within ${HASH_HERALD_TIMEOUT_MS}ms`)), HASH_HERALD_TIMEOUT_MS);
+    });
+    return Promise.race([getHash(HASH_HERALD_OPTIONS, req), timeout]).finally(() => clearTimeout(timer));
+}
 function resultFromGetHashResponse(response) {
     if (!response.resp) {
         return { type: "bad_hash_chain", reason: "empty_response" };
@@ -26,7 +34,7 @@ export async function getIntermediateHash({ hashChainId, iteration, }) {
         iterations: iteration,
         context: undefined,
     };
-    return getHash(HASH_HERALD_OPTIONS, req).then(resultFromGetHashResponse);
+    return getHashWithTimeout(req).then(resultFromGetHashResponse);
 }
 export async function getPreimageHash({ hashChainId, }) {
     const req = {
@@ -39,7 +47,7 @@ export async function getPreimageHash({ hashChainId, }) {
             },
         },
     };
-    return getHash(HASH_HERALD_OPTIONS, req).then(resultFromGetHashResponse);
+    return getHashWithTimeout(req).then(resultFromGetHashResponse);
 }
 export async function getTerminalHash({ hashChainId, }) {
     const iterations = config.HASHCHAINSERVER_MAX_ITERATIONS;
@@ -53,5 +61,5 @@ export async function getTerminalHash({ hashChainId, }) {
             },
         },
     };
-    return getHash(HASH_HERALD_OPTIONS, req).then(resultFromGetHashResponse);
+    return getHashWithTimeout(req).then(resultFromGetHashResponse);
 }

package/dist/src/liability/internal/processor.d.ts

@@ -2,8 +2,10 @@ import * as pg from "pg";
 import { type PgClientInTransaction } from "../../db/transaction.js";
 import { LiabilityConfigInternal } from "../public/config.js";
 import { QueryExecutor } from "../../db/util.js";
-export declare function dbGetLiabilityIdsPastDeadline(pgClient: QueryExecutor, { limit }?: {
+export declare function dbGetLiabilityIdsPastDeadline(pgClient: QueryExecutor, { limit, excludeIds, before, }?: {
     limit?: number;
+    excludeIds?: string[];
+    before?: Date;
 }): Promise<string[]>;
 export declare function startLiabilityDeadlineProcessor<RefType extends string>({ signal, pool, liabilityConfig, }: {
     signal: AbortSignal;
@@ -14,8 +16,11 @@ export declare function processOneDeadlinedLiability<RefType extends string>(pgC
     liabilityConfig: LiabilityConfigInternal<RefType>;
     liabilityId: string;
 }): Promise<void>;
-export declare function processDeadlinedLiabilities<RefType extends string>({ pool, liabilityConfig, signal, }: {
+export declare function processDeadlinedLiabilities<RefType extends string>({ pool, liabilityConfig, signal, batchSize, }: {
     pool: pg.Pool;
     liabilityConfig: LiabilityConfigInternal<RefType>;
     signal: AbortSignal;
-}): Promise<void>;
+    batchSize?: number;
+}): Promise<{
+    failed: number;
+}>;

package/dist/src/liability/internal/processor.js

@@ -4,17 +4,18 @@ import { dbLockHouseBankroll, dbLockPlayerBalance } from "../../db/public.js";
 import { exactlyOneRow, maybeOneRow } from "../../db/util.js";
 import { abortableTimeout } from "../../util.js";
 const BATCH_SIZE = 100;
-export async function dbGetLiabilityIdsPastDeadline(pgClient, { limit = 100 } = {}) {
+export async function dbGetLiabilityIdsPastDeadline(pgClient, { limit = 100, excludeIds = [], before = new Date(), } = {}) {
     const result = await pgClient.query({
         text: `
       SELECT id
       FROM hub.liability
       WHERE resolved_at IS NULL
-        AND deadline < now()
+        AND deadline < $3
+        AND NOT (id = ANY($2::uuid[]))
       ORDER BY deadline ASC
       LIMIT $1
     `,
-        values: [limit],
+        values: [limit, excludeIds, before],
     });
     return result.rows.map((r) => r.id);
 }
@@ -89,46 +90,55 @@ export async function processOneDeadlinedLiability(pgClient, { liabilityConfig,
         logger.error(`No house bankroll found for liability ${dbLockedLiability.id}.`);
         return;
     }
-    try {
-        await refTypeConfig.handleDeadline({
-            pgClientInTransaction: pgClient,
-            lockedLiability: dbLockedLiability,
-            lockedPlayerBalance: dbLockedPlayerBalance,
-            lockedHouseBankroll: dbLockedHouseBankroll,
-            liabilityConfig: liabilityConfig,
-        });
-        const { resolved_at } = await pgClient
-            .query(`SELECT resolved_at FROM hub.liability WHERE id = $1`, [dbLockedLiability.id])
-            .then(exactlyOneRow);
-        if (!resolved_at) {
-            throw new Error(`Handler returned LIABILITY_RESOLVED but did not resolve liability ${dbLockedLiability.id} (ref_type: ${dbLockedLiability.ref_type}, ref_id: ${dbLockedLiability.ref_id}). ` +
-                `Handler must call dbResolveLiabilityByRef before returning.`);
-        }
-        logger.debug(`Resolved liability ${dbLockedLiability.id}`);
-    }
-    catch (e) {
-        logger.error(e, `Error handling deadline for liability ${dbLockedLiability.id}`);
-        throw e;
-    }
-}
-export async function processDeadlinedLiabilities({ pool, liabilityConfig, signal, }) {
-    const liabilityIds = await dbGetLiabilityIdsPastDeadline(pool, {
-        limit: BATCH_SIZE,
+    await refTypeConfig.handleDeadline({
+        pgClientInTransaction: pgClient,
+        lockedLiability: dbLockedLiability,
+        lockedPlayerBalance: dbLockedPlayerBalance,
+        lockedHouseBankroll: dbLockedHouseBankroll,
+        liabilityConfig: liabilityConfig,
     });
-    if (liabilityIds.length === 0) {
-        return;
+    const { resolved_at } = await pgClient
+        .query(`SELECT resolved_at FROM hub.liability WHERE id = $1`, [dbLockedLiability.id])
+        .then(exactlyOneRow);
+    if (!resolved_at) {
+        throw new Error(`Handler returned LIABILITY_RESOLVED but did not resolve liability ${dbLockedLiability.id} (ref_type: ${dbLockedLiability.ref_type}, ref_id: ${dbLockedLiability.ref_id}). ` +
+            `Handler must call dbResolveLiabilityByRef before returning.`);
     }
-    logger.info(`Processor found ${liabilityIds.length} liability candidates...`);
-    for (const liabilityId of liabilityIds) {
-        if (signal.aborted) {
-            logger.info(`[processExpiredLiabilities] Aborted by graceful shutdown.`);
+    logger.debug(`Resolved liability ${dbLockedLiability.id}`);
+}
+export async function processDeadlinedLiabilities({ pool, liabilityConfig, signal, batchSize = BATCH_SIZE, }) {
+    const before = new Date();
+    const tried = [];
+    let failed = 0;
+    while (!signal.aborted) {
+        const liabilityIds = await dbGetLiabilityIdsPastDeadline(pool, {
+            limit: batchSize,
+            excludeIds: tried,
+            before,
+        });
+        if (liabilityIds.length === 0) {
             break;
         }
-        await withPgPoolTransaction(pool, async (pgClient) => {
-            await processOneDeadlinedLiability(pgClient, {
-                liabilityConfig,
-                liabilityId,
-            });
-        });
+        logger.info(`Processor found ${liabilityIds.length} liability candidates...`);
+        for (const liabilityId of liabilityIds) {
+            if (signal.aborted) {
+                logger.info(`[processExpiredLiabilities] Aborted by graceful shutdown.`);
+                break;
+            }
+            tried.push(liabilityId);
+            try {
+                await withPgPoolTransaction(pool, async (pgClient) => {
+                    await processOneDeadlinedLiability(pgClient, {
+                        liabilityConfig,
+                        liabilityId,
+                    });
+                });
+            }
+            catch (e) {
+                failed++;
+                logger.error({ err: e, liabilityId }, `Error handling deadline for liability ${liabilityId}`);
+            }
+        }
     }
+    return { failed };
 }

package/dist/src/log-serializers.d.ts

@@ -0,0 +1,4 @@
+import { type LoggerOptions } from "pino";
+export declare function errorLogOptions(options: {
+    full: boolean;
+}): Pick<LoggerOptions, "serializers" | "hooks">;

package/dist/src/log-serializers.js

@@ -0,0 +1,118 @@
+import pino from "pino";
+import { ClientError } from "graphql-request";
+const SAFE_CODE = /^[A-Z0-9_]{1,32}$/;
+const OPERATION_NAME = /^\s*(?:query|mutation|subscription)\s+([_A-Za-z][_0-9A-Za-z]{0,99})\b/;
+const MAX_DEPTH = 5;
+const WITHHELD_MESSAGE = "error details withheld: a casino request failed";
+function isClientError(value) {
+    return (value instanceof Error &&
+        typeof value.response === "object" &&
+        value.response !== null &&
+        typeof value.request === "object");
+}
+function nestedErrors(err) {
+    const record = err;
+    const nested = [err.cause, record.originalError];
+    if (Array.isArray(record.errors)) {
+        nested.push(...record.errors);
+    }
+    for (const key in err) {
+        nested.push(record[key]);
+    }
+    return nested.filter((x) => x instanceof Error && x !== err);
+}
+function containsClientError(err, depth = 0) {
+    if (!(err instanceof Error)) {
+        return false;
+    }
+    if (depth > MAX_DEPTH) {
+        return true;
+    }
+    return (isClientError(err) ||
+        nestedErrors(err).some((x) => containsClientError(x, depth + 1)));
+}
+const LOCATION = String.raw `(?:file:\/\/|node:|\/)[^\s()]+:\d+:\d+`;
+const STACK_FRAME = new RegExp(String.raw `^\s+at (?:(?:async |new )?[^\s()]+(?: \[as [^\s\]]+\])? \((?:${LOCATION}|<anonymous>|index \d+)\)|(?:async )?${LOCATION})$`);
+function stackFrames(err) {
+    const header = String(err);
+    if (typeof err.stack !== "string" || !err.stack.startsWith(`${header}\n`)) {
+        return undefined;
+    }
+    const lines = err.stack.slice(header.length + 1).split("\n");
+    return lines.every((line) => STACK_FRAME.test(line))
+        ? lines.map((line) => line.trim())
+        : undefined;
+}
+function operationName(err) {
+    const query = err.request
+        ?.query;
+    return typeof query === "string"
+        ? OPERATION_NAME.exec(query)?.[1]
+        : undefined;
+}
+function describeClientError(err) {
+    const { status, errors } = err.response;
+    const list = Array.isArray(errors) ? errors : [];
+    return {
+        type: "ClientError",
+        operationName: operationName(err),
+        status: typeof status === "number" ? status : undefined,
+        errorCount: list.length,
+        codes: list.flatMap((e) => {
+            const code = e?.extensions
+                ?.code;
+            return typeof code === "string" && SAFE_CODE.test(code) ? [code] : [];
+        }),
+        frames: err instanceof ClientError ? stackFrames(err) : undefined,
+    };
+}
+function describeTaintedError(err, depth) {
+    if (isClientError(err)) {
+        return describeClientError(err);
+    }
+    return {
+        type: err.constructor?.name ?? "Error",
+        withheld: true,
+        ...(depth < MAX_DEPTH
+            ? {
+                nested: nestedErrors(err).map((x) => describeTaintedError(x, depth + 1)),
+            }
+            : {}),
+    };
+}
+function serializeError(err) {
+    if (!(err instanceof Error)) {
+        return err;
+    }
+    return containsClientError(err)
+        ? describeTaintedError(err, 0)
+        : pino.stdSerializers.err(err);
+}
+const withholdingHooks = {
+    logMethod(args, method) {
+        const [first, second] = args;
+        const error = first instanceof Error
+            ? first
+            : (first?.err ??
+                first?.error);
+        if (typeof second !== "string" && containsClientError(error)) {
+            const object = first instanceof Error ? { err: first } : first;
+            return method.apply(this, [object, WITHHELD_MESSAGE]);
+        }
+        return method.apply(this, args);
+    },
+};
+export function errorLogOptions(options) {
+    if (options.full) {
+        return {
+            serializers: {
+                err: pino.stdSerializers.err,
+                error: pino.stdSerializers.err,
+            },
+        };
+    }
+    return {
+        serializers: { err: serializeError, error: serializeError },
+        hooks: withholdingHooks,
+    };
+}

package/dist/src/logger.js

@@ -1,11 +1,13 @@
 import pino from "pino";
 import { LOG_LEVEL, LOG_PRETTY, NODE_ENV } from "./config.js";
+import { errorLogOptions } from "./log-serializers.js";
 function createLogger(options) {
     const prettify = LOG_PRETTY === undefined ? NODE_ENV !== "production" : LOG_PRETTY;
     if (prettify) {
         try {
             return pino({
                 level: options.level,
+                ...errorLogOptions({ full: NODE_ENV === "development" }),
                 transport: {
                     target: "pino-pretty",
                     options: {
@@ -18,12 +20,14 @@ function createLogger(options) {
             console.warn("LOG_PRETTY=true but pino-pretty is not installed, falling back to standard logging");
             return pino({
                 level: options.level,
+                ...errorLogOptions({ full: NODE_ENV === "development" }),
             });
         }
     }
     else {
         return pino({
             level: options.level,
+            ...errorLogOptions({ full: NODE_ENV === "development" }),
         });
     }
 }

package/dist/src/plugins/graphql-error-log.d.ts

@@ -0,0 +1,28 @@
+type ResponseError = {
+    message?: unknown;
+    path?: unknown;
+    locations?: unknown;
+    extensions?: {
+        code?: unknown;
+    };
+};
+type ResponsePayload = {
+    data?: unknown;
+    errors?: ResponseError[];
+};
+export type GraphQLErrorLogLine = {
+    operationName: string | null;
+    kind: "parse" | "request" | "execution";
+    errorCount: number;
+    codes: string[];
+    locations: (string | null)[];
+    unsafe?: {
+        message: unknown;
+        path: unknown;
+    }[];
+};
+export declare function summarizeGraphQLErrors(payload: ResponsePayload, rawOperationName: unknown, options: {
+    includeUnsafe: boolean;
+}): GraphQLErrorLogLine | null;
+export declare const GraphQLErrorLogPlugin: GraphileConfig.Plugin;
+export {};

package/dist/src/plugins/graphql-error-log.js

@@ -0,0 +1,103 @@
+import { NODE_ENV } from "../config.js";
+import { logger } from "../logger.js";
+const SAFE_CODE = /^[A-Z0-9_]{1,32}$/;
+const MAX_INSPECTED_RESPONSE_BYTES = 256 * 1024;
+const SAFE_OPERATION_NAME = /^[_A-Za-z][_0-9A-Za-z]*$/;
+const MAX_OPERATION_NAME_LENGTH = 100;
+const MAX_ERRORS_INSPECTED = 20;
+export function summarizeGraphQLErrors(payload, rawOperationName, options) {
+    const errors = Array.isArray(payload.errors) ? payload.errors : [];
+    if (errors.length === 0) {
+        return null;
+    }
+    const inspected = errors.slice(0, MAX_ERRORS_INSPECTED);
+    const locations = inspected.map((e) => firstLocation(e?.locations));
+    const codes = [
+        ...new Set(inspected.flatMap((e) => {
+            const code = e?.extensions?.code;
+            return typeof code === "string" && SAFE_CODE.test(code) ? [code] : [];
+        })),
+    ];
+    const isParseError = inspected.some((e) => typeof e?.message === "string" && e.message.startsWith("Syntax Error"));
+    return {
+        operationName: safeOperationName(rawOperationName),
+        kind: isParseError ? "parse" : "data" in payload ? "execution" : "request",
+        errorCount: errors.length,
+        codes,
+        locations,
+        ...(options.includeUnsafe
+            ? {
+                unsafe: inspected.map((e) => ({
+                    message: e?.message,
+                    path: e?.path,
+                })),
+            }
+            : {}),
+    };
+}
+function firstLocation(locations) {
+    const first = Array.isArray(locations) ? locations[0] : undefined;
+    const { line, column } = (first ?? {});
+    return Number.isSafeInteger(line) &&
+        Number.isSafeInteger(column) &&
+        line > 0 &&
+        column > 0
+        ? `${line}:${column}`
+        : null;
+}
+function safeOperationName(name) {
+    if (typeof name !== "string" || name === "") {
+        return null;
+    }
+    return name.length <= MAX_OPERATION_NAME_LENGTH &&
+        SAFE_OPERATION_NAME.test(name)
+        ? name
+        : "invalid";
+}
+function responsePayload(result) {
+    if (result?.type === "json") {
+        return result.json;
+    }
+    if (result?.type === "buffer" &&
+        result.buffer.length <= MAX_INSPECTED_RESPONSE_BYTES &&
+        result.buffer.includes('"errors"')) {
+        try {
+            return JSON.parse(result.buffer.toString("utf8"));
+        }
+        catch {
+            return null;
+        }
+    }
+    return null;
+}
+const operationNames = new WeakMap();
+export const GraphQLErrorLogPlugin = {
+    name: "HubGraphQLErrorLogPlugin",
+    description: "Logs GraphQL requests that return errors, without secrets",
+    version: "0.0.1",
+    grafserv: {
+        middleware: {
+            processGraphQLRequestBody(next, event) {
+                if (event.request) {
+                    operationNames.set(event.request, event.body.operationName);
+                }
+                return next();
+            },
+            async processRequest(next, event) {
+                const result = await next();
+                try {
+                    const payload = responsePayload(result);
+                    const line = payload &&
+                        summarizeGraphQLErrors(payload, operationNames.get(event.requestDigest), { includeUnsafe: NODE_ENV === "development" });
+                    if (line) {
+                        logger.warn(line, "GraphQL request returned errors");
+                    }
+                }
+                catch (e) {
+                    logger.warn({ errorType: e instanceof Error ? e.constructor.name : typeof e }, "GraphQL error logging failed");
+                }
+                return result;
+            },
+        },
+    },
+};

package/dist/src/server/graphile.config.js

@@ -16,6 +16,7 @@ import { logger } from "../logger.js";
 import { ValidateCasinoFieldsPlugin } from "../plugins/validate-fields.js";
 import { HubAddCasinoPlugin } from "../plugins/hub-add-casino.js";
 import { HubAdjustBankrollPlugin } from "../plugins/hub-adjust-bankroll.js";
+import { GraphQLErrorLogPlugin } from "../plugins/graphql-error-log.js";
 import { HubBalanceAlertPlugin } from "../plugins/hub-balance-alert.js";
 import { custom as customPgOmitArchivedPlugin } from "@graphile-contrib/pg-omit-archived";
 import { HubCurrentXPlugin } from "../plugins/hub-current-x.js";
@@ -46,6 +47,7 @@ export const requiredPlugins = [
     HubAddCasinoPlugin,
     ValidateCasinoFieldsPlugin,
     HubAdjustBankrollPlugin,
+    GraphQLErrorLogPlugin,
     HubOutcomeInputNonNullFieldsPlugin,
     HubBadHashChainErrorPlugin,
     HubCreateHashChainPlugin,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moneypot/hub",
-  "version": "1.21.0",
+  "version": "1.22.1",
   "author": "moneypot.com",
   "homepage": "https://moneypot.com/hub",
   "keywords": [
@@ -87,7 +87,7 @@
     "codegen": "npx graphql-codegen",
     "codegen-watch": "graphql-codegen --watch",
     "build-dashboard": "cd ./dashboard && pnpm run build && rm -rf ../dist/dashboard && cp -Rv dist ../dist/dashboard",
-    "test:startup": "dropdb --if-exists hub-test && createdb hub-test && SUPERUSER_DATABASE_URL=postgres://localhost:5432/hub-test DATABASE_URL=postgres://localhost:5432/hub-test tsx src/test-startup.ts",
+    "test:startup": "dropdb --if-exists hub-test && createdb hub-test && NODE_ENV=development SUPERUSER_DATABASE_URL=postgres://localhost:5432/hub-test DATABASE_URL=postgres://localhost:5432/hub-test tsx src/test-startup.ts",
     "test": "vitest run",
     "test:watch": "vitest",
     "test:bail": "pnpm test -- --bail --silent --reporter=basic 2>&1 | head -25"