@mohasinac/appkit 4.61.0 → 4.62.1

package/dist/_internal/server/jobs/core/onJobCreated.js

@@ -5,55 +5,88 @@
  * (CLAUDE.md Rule #6 — Vercel routes never do heavy work; they write a
  * `jobs` doc and return, this Function does the actual processing).
  *
- * Flow:
- *   1. Mark the job "processing".
- *   2. Look up a runner in `JOB_RUNNERS` by `job.jobType`.
- *   3. Run it, mark the job "done"/"failed", and best-effort ping the
- *      `bulk_events/{jobId}` RTDB channel the client is subscribed to via
- *      `useBulkEvent` — every RTDB write is its own try/catch so a ping
- *      failure never masks the real Firestore result.
+ * ## 🛑 It no longer runs the job itself — `runJobOnce` does (Phase 8)
+ *
+ * The attempt, the failure classification and the retry scheduling all live in
+ * `runJobOnce`, which the retry sweep also calls. One dispatch path for both
+ * entry points, or the two drift the first time either learns something.
  */
-import { normalizeError } from "../../../../errors/normalize.js";
 import { jobsRepository } from "../../../../repositories/index.js";
-import { getAdminRealtimeDb } from "../../../../providers/db-firebase/index.js";
-import { RTDB_PATHS } from "../../../../providers/db-firebase/rtdb-paths.js";
-import { JOB_RUNNERS } from "./jobRunners.js";
-async function pingBulkEvent(jobId, payload, ctx) {
-    try {
-        await getAdminRealtimeDb()
-            .ref(`${RTDB_PATHS.BULK_EVENTS}/${jobId}`)
-            .set({ ...payload, updatedAt: Date.now() });
-    }
-    catch (err) {
-        void normalizeError(err);
-        ctx.logger.warn("onJobCreated: RTDB ping failed (non-fatal)", {
-            jobId,
-            error: err instanceof Error ? err.message : String(err),
-        });
-    }
-}
+import { runJobOnce } from "./runJobOnce.js";
 export async function handleJobCreated(input, ctx) {
     const { jobId, job } = input;
-    await jobsRepository.markProcessing(jobId);
-    const runner = JOB_RUNNERS[job.jobType];
-    if (!runner) {
-        const error = `No runner registered for jobType: ${job.jobType}`;
-        await jobsRepository.markFailed(jobId, error);
-        await pingBulkEvent(jobId, { status: "failed", action: job.jobType, error }, ctx);
-        ctx.logger.error("onJobCreated: unregistered jobType", null, { jobId, jobType: job.jobType });
-        return;
-    }
-    try {
-        const result = await runner(job.payload, ctx);
-        await jobsRepository.markDone(jobId, result);
-        await pingBulkEvent(jobId, { status: "success", action: job.jobType, ...result }, ctx);
-        ctx.logger.info("onJobCreated: job complete", { jobId, jobType: job.jobType, summary: result.summary });
+    /*
+     * ── 🛑 A SCHEDULED RETRY MUST NOT RUN HERE ────────────────────────────────
+     *
+     * This guard is the whole reason Phase 8's backoff works, and its absence
+     * was a defect in this feature's own first draft.
+     *
+     * A retry is a NEW `jobs` document (it has to be — this trigger fires on
+     * CREATE, so flipping a job back to `pending` would never wake anything).
+     * But creating that document fires THIS trigger immediately — so without
+     * this check, every retry would run the instant it was scheduled, the
+     * 5-minute and 30-minute backoffs would be zero, and three attempts would
+     * burn in under a second against a fault that had not had time to clear.
+     * The backoff fields would still be written, so the database would read as
+     * though it had waited.
+     *
+     * `nextAttemptAt` in the future means "the sweep owns this one". Returning
+     * leaves the document `pending`, which is exactly what `getDueRetries`
+     * queries for.
+     */
+    if (job.nextAttemptAt) {
+        const due = new Date(job.nextAttemptAt).getTime();
+        if (Number.isFinite(due) && due > Date.now()) {
+            ctx.logger.info("onJobCreated: retry scheduled, deferring to the sweep", {
+                jobId,
+                jobType: job.jobType,
+                attempt: job.attempt ?? 1,
+                nextAttemptAt: new Date(due).toISOString(),
+            });
+            return;
+        }
     }
-    catch (err) {
-        void normalizeError(err);
-        const message = err instanceof Error ? err.message : String(err);
-        await jobsRepository.markFailed(jobId, message);
-        await pingBulkEvent(jobId, { status: "failed", action: job.jobType, error: message }, ctx);
-        ctx.logger.error("onJobCreated: job failed", err, { jobId, jobType: job.jobType });
+    await runJobOnce({ jobId, job }, ctx);
+}
+/**
+ * The retry sweep — runs every scheduled attempt whose time has come.
+ *
+ * ## 🛑 Folded into an EXISTING scheduled function, not a new one
+ *
+ * It is called from `paymentWindowTimeout` (every 5 minutes). Cloud Scheduler
+ * bills per registered job and this project already carries ~34 of them
+ * against 3 free; adding a 35th to run a usually-empty query is the cost this
+ * plan explicitly refuses. The same reasoning put the ledger reconciler inside
+ * `dailyDataCleanup` and R1 inside this same function.
+ *
+ * 🛑 And it is placed FIRST at its call site, before that function's early
+ * return — `paymentWindowTimeout` returns as soon as there are no expired
+ * payment windows, which is most ticks, so appending would run the sweep only
+ * on the minority of ticks with unrelated work (Root Cause #27).
+ *
+ * Sequential, not `Promise.all`. A job runner is heavy by definition — that is
+ * why it is a job — and twenty of them concurrently inside one 540s function
+ * is how a sweep exhausts its own budget. Twenty sequential attempts that run
+ * out of time simply leave the rest `pending` for the next tick, which is the
+ * safe failure.
+ */
+export async function runDueJobRetries(ctx) {
+    const due = await jobsRepository.getDueRetries();
+    if (due.length === 0)
+        return { ran: 0 };
+    ctx.logger.info("job retries: running due attempts", { count: due.length });
+    let ran = 0;
+    for (const { id, data } of due) {
+        /*
+         * `runJobOnce` never throws — it records every outcome on the job itself.
+         * The `.catch` is belt-and-braces so one pathological job cannot abort the
+         * rest of the sweep, which would leave the remainder un-retried with no
+         * record of why.
+         */
+        await runJobOnce({ jobId: id, job: data }, ctx).catch((err) => {
+            ctx.logger.error("job retries: attempt threw", err, { jobId: id });
+        });
+        ran += 1;
     }
+    return { ran };
 }

package/dist/_internal/server/jobs/core/onOrderStatusChange.d.ts

@@ -6,9 +6,16 @@ export type OrderAfter = {
     userEmail: string;
     productTitle: string;
     trackingNumber?: string;
+    /**
+     * Phase 7 — the invoice is numbered on the PAYMENT transition, which is a
+     * different axis from `status` and is why these two fields are here.
+     */
+    paymentStatus?: string;
+    invoiceNumber?: string;
 };
 export type OrderBefore = {
     status: OrderStatus;
+    paymentStatus?: string;
 };
 export interface HandleOrderStatusChangeInput {
     orderId: string;

package/dist/_internal/server/jobs/core/onOrderStatusChange.js

@@ -1,5 +1,9 @@
 import { decryptPii } from "../../../../security/index.js";
 import { sendNotification } from "../../../../features/admin/actions/notification-actions.js";
+import { orderRepository } from "../../../../features/orders/repository/orders.repository.js";
+/* Phase 7 — numbering only; the PDF renderer lives in `functions/` because
+ * `pdfkit` must not enter appkit's module graph. */
+import { allocateSeriesNumber } from "../../features/invoicing/allocate.js";
 import { ORDER_MESSAGES } from "../handlers/messages.js";
 const STATUS_CONFIG = {
     confirmed: {
@@ -31,6 +35,64 @@ export async function handleOrderStatusChange(input, ctx) {
     const { orderId, before, after } = input;
     if (!before || !after)
         return;
+    /*
+     * ── PHASE 7: number the invoice on the PAYMENT transition ─────────────────
+     *
+     * 🛑 PLACED FIRST, before the `status`-unchanged early return. That return
+     * is correct for the notification work below it, and it is also why this
+     * cannot go after it: `paymentStatus: pending → paid` frequently happens
+     * with NO change to `status` at all (a COD order verified by an admin, a
+     * manual proof approved), so appending this below would have numbered only
+     * the minority of paid orders whose status happened to move in the same
+     * write. That is Root Cause #27's sweep-coverage trap, and R1 in
+     * `paymentWindowTimeout` had to be placed the same way for the same reason.
+     *
+     * 🛑 Extending THIS trigger rather than adding a second one on `orders`.
+     * A second Firestore trigger on the same collection wakes for every write
+     * and returns — which is how `onShipmentDeleted` burned a million
+     * invocations a day sitting correctly-guarded beside a neighbour's loop
+     * (Root Cause #92). The sticker-trace definitions carry the same 🛑.
+     *
+     * 🛑 And it is numbering ONLY — no PDF. `pdfkit` is a Functions dependency
+     * and must not enter appkit's module graph; the bytes are rendered once,
+     * lazily, by the serve route and stored from there. Numbering is the half
+     * that must be immediate and transactional, because it is the half that
+     * cannot be corrected: a number cannot be withdrawn without leaving a gap.
+     */
+    const becamePaid = before.paymentStatus !== "paid" && after.paymentStatus === "paid";
+    if (becamePaid && !after.invoiceNumber) {
+        try {
+            const allocation = await allocateSeriesNumber({
+                kind: "invoice",
+                subjectId: orderId,
+                orderId,
+            });
+            /*
+             * Only writes when THIS call allocated. A re-entry that found an
+             * existing claim skips the write entirely, which keeps a re-fired
+             * trigger from touching the order at all — the write that would
+             * otherwise re-fire this very trigger.
+             */
+            if (allocation.allocated) {
+                await orderRepository.update(orderId, {
+                    invoiceNumber: allocation.number,
+                    invoiceIssuedAt: new Date(),
+                });
+                ctx.logger.info(`Invoice ${allocation.number} allocated`, { orderId });
+            }
+        }
+        catch (error) {
+            /*
+             * Logged and swallowed, so a numbering failure cannot suppress the
+             * buyer's order notification below. The order stays un-invoiced, which
+             * is a visible, retryable state — the next write that touches
+             * `paymentStatus` re-attempts, and the serve route can allocate on
+             * demand. Throwing here would also make the Firestore trigger retry the
+             * whole handler, re-sending the notification.
+             */
+            ctx.logger.error("Invoice allocation failed", error, { orderId });
+        }
+    }
     if (before.status === after.status)
         return;
     const newStatus = after.status;

package/dist/_internal/server/jobs/core/paymentReconcile.js

@@ -138,81 +138,7 @@ export async function reconcileStuckGatewayEvents(ctx) {
     }
     return { redispatched, escalated, reads: stuck.length };
 }
-/**
- * "The gateway captured a different amount from the one the order records."
- *
- * Extracted from the sweep below, where it sat inside
- * fn → `for record` → `for order` → `if` and the whole case literal was a
- * brace too deep. Pure construction — no branching, and `recorded` is passed
- * in rather than re-derived, so the one money rule still lives at the call
- * site (Root Cause #75).
- */
-async function openAmountMismatchCase(record, orderId, recorded, now) {
-    await paymentCasesRepository().openOrObserve({
-        kind: "amount_mismatch",
-        subjectId: `${record.id}:${orderId}`,
-        severity: "error",
-        title: `Captured ${record.amount} but recorded ${recorded} — ${orderId}`,
-        detail: [
-            `The gateway captured a different amount from the one the order records.`,
-            `Either the buyer was over-charged or the order under-states what was paid.`,
-            "",
-            `Gateway: ${record.gateway}`,
-            `Payment: ${record.id}`,
-            `Order: ${orderId}`,
-            `Captured: ${record.amount} ${record.currency}`,
-            `Order total: ${recorded}`,
-            `Difference: ${(record.amount - recorded).toFixed(2)}`,
-        ].join("\n"),
-        entityType: "order",
-        entityId: orderId,
-        amountAtRisk: Math.abs(record.amount - recorded),
-        orderIds: [orderId],
-        gatewayPaymentId: record.id,
-        now,
-    });
-}
-/**
- * "Our refund total disagrees with the gateway's."
- *
- * Same extraction as `openAmountMismatchCase`. `ours` and `base` are both
- * computed by the caller from the shared helpers, so this never re-derives a
- * refund figure of its own.
- *
- * 🛑 The severity split is the whole point and must not be flattened: the
- * gateway having refunded MORE than we recorded is money that left without a
- * record (`error`); our side showing more is a refund reserved and never
- * completed (`warning`).
- */
-async function openRefundMismatchCase(record, orderId, ours, base, now) {
-    await paymentCasesRepository().openOrObserve({
-        kind: "refund_mismatch",
-        subjectId: `${record.id}:refunds`,
-        severity: record.amountRefunded > ours ? "error" : "warning",
-        title: `Refund total disagrees with the gateway — ${orderId}`,
-        detail: [
-            record.amountRefunded > ours
-                ? `🛑 The gateway has refunded MORE than our records show. Money has left that we did not record.`
-                : `Our records show more refunded than the gateway did — a refund may have been reserved and never completed.`,
-            "",
-            `Gateway: ${record.gateway}`,
-            `Payment: ${record.id}`,
-            `Order: ${orderId}`,
-            `Gateway refunded: ${record.amountRefunded}`,
-            `Our committed refunds: ${ours}`,
-            `Refundable base: ${base}`,
-            record.refundStatus ? `Gateway refund status: ${record.refundStatus}` : null,
-        ]
-            .filter((l) => l !== null)
-            .join("\n"),
-        entityType: "order",
-        entityId: orderId,
-        amountAtRisk: Math.abs(record.amountRefunded - ours),
-        orderIds: [orderId],
-        gatewayPaymentId: record.id,
-        now,
-    });
-}
+/* ──────────────── R2: the gateway has it — do we? ───────────────────────── */
 async function reconcileGatewayPayments(ctx) {
     const payment = getProviders().payment;
     let examined = 0;
@@ -298,28 +224,9 @@ async function reconcileGatewayPayments(ctx) {
                 continue;
             }
             for (const order of orders) {
-                /*
-                 * Amount check. Compared against the order's own recorded total rather
-                 * than recomputed from its lines — recomputing would re-derive the
-                 * pricing pipeline here and disagree with it the day a rate changes,
-                 * which is a sixth copy of the money rule (Root Cause #75).
-                 */
-                const recorded = order.totalPrice ?? 0;
-                if (Math.abs(recorded - record.amount) > MONEY_EPSILON) {
-                    amountMismatch += 1;
-                    await openAmountMismatchCase(record, order.id, recorded, ctx.now);
-                }
-                /*
-                 * Refund check — the gateway's `amount_refunded` against our own
-                 * non-failed refund events. `committedRefundTotal` is the same helper
-                 * the refund cap uses, so "what we think we refunded" has ONE
-                 * definition rather than a second one written here.
-                 */
-                const ours = committedRefundTotal(order);
-                if (Math.abs(ours - record.amountRefunded) > MONEY_EPSILON) {
-                    refundMismatch += 1;
-                    await openRefundMismatchCase(record, order.id, ours, refundableBase(order), ctx.now);
-                }
+                const outcome = await reconcilePaymentAgainstOrder(record, order, ctx);
+                amountMismatch += outcome.amountMismatch;
+                refundMismatch += outcome.refundMismatch;
             }
         }
         catch (err) {
@@ -331,6 +238,89 @@ async function reconcileGatewayPayments(ctx) {
     }
     return { examined, missing, amountMismatch, refundMismatch, reads };
 }
+/**
+ * The two per-order checks R2 runs against one gateway record.
+ *
+ * Extracted so each `openOrObserve` call is a statement in a function body
+ * rather than five braces deep inside `for record -> try -> for order -> if`.
+ * Returns the counts rather than mutating the caller's accumulators, so the
+ * loop above reads as a sum rather than as two side effects.
+ */
+async function reconcilePaymentAgainstOrder(record, order, ctx) {
+    let amountMismatch = 0;
+    let refundMismatch = 0;
+    /*
+     * Amount check. Compared against the order's own recorded total rather
+     * than recomputed from its lines — recomputing would re-derive the
+     * pricing pipeline here and disagree with it the day a rate changes,
+     * which is a sixth copy of the money rule (Root Cause #75).
+     */
+    const recorded = order.totalPrice ?? 0;
+    if (Math.abs(recorded - record.amount) > MONEY_EPSILON) {
+        amountMismatch += 1;
+        await paymentCasesRepository().openOrObserve({
+            kind: "amount_mismatch",
+            subjectId: `${record.id}:${order.id}`,
+            severity: "error",
+            title: `Captured ${record.amount} but recorded ${recorded} — ${order.id}`,
+            detail: [
+                `The gateway captured a different amount from the one the order records.`,
+                `Either the buyer was over-charged or the order under-states what was paid.`,
+                "",
+                `Gateway: ${record.gateway}`,
+                `Payment: ${record.id}`,
+                `Order: ${order.id}`,
+                `Captured: ${record.amount} ${record.currency}`,
+                `Order total: ${recorded}`,
+                `Difference: ${(record.amount - recorded).toFixed(2)}`,
+            ].join("\n"),
+            entityType: "order",
+            entityId: order.id,
+            amountAtRisk: Math.abs(record.amount - recorded),
+            orderIds: [order.id],
+            gatewayPaymentId: record.id,
+            now: ctx.now,
+        });
+    }
+    /*
+     * Refund check — the gateway's `amount_refunded` against our own
+     * non-failed refund events. `committedRefundTotal` is the same helper
+     * the refund cap uses, so "what we think we refunded" has ONE
+     * definition rather than a second one written here.
+     */
+    const ours = committedRefundTotal(order);
+    if (Math.abs(ours - record.amountRefunded) > MONEY_EPSILON) {
+        refundMismatch += 1;
+        await paymentCasesRepository().openOrObserve({
+            kind: "refund_mismatch",
+            subjectId: `${record.id}:refunds`,
+            severity: record.amountRefunded > ours ? "error" : "warning",
+            title: `Refund total disagrees with the gateway — ${order.id}`,
+            detail: [
+                record.amountRefunded > ours
+                    ? `🛑 The gateway has refunded MORE than our records show. Money has left that we did not record.`
+                    : `Our records show more refunded than the gateway did — a refund may have been reserved and never completed.`,
+                "",
+                `Gateway: ${record.gateway}`,
+                `Payment: ${record.id}`,
+                `Order: ${order.id}`,
+                `Gateway refunded: ${record.amountRefunded}`,
+                `Our committed refunds: ${ours}`,
+                `Refundable base: ${refundableBase(order)}`,
+                record.refundStatus ? `Gateway refund status: ${record.refundStatus}` : null,
+            ]
+                .filter((l) => l !== null)
+                .join("\n"),
+            entityType: "order",
+            entityId: order.id,
+            amountAtRisk: Math.abs(record.amountRefunded - ours),
+            orderIds: [order.id],
+            gatewayPaymentId: record.id,
+            now: ctx.now,
+        });
+    }
+    return { amountMismatch, refundMismatch };
+}
 /* ──────────────── R3: refund claimed, never settled ─────────────────────── */
 /**
  * A `refundClaims` row still `claimed` an hour later is a crash between

package/dist/_internal/server/jobs/core/paymentWindowTimeout.js

@@ -6,6 +6,9 @@ import { OrderStatusValues } from "../../../../features/orders/schemas/firestore
 import { PAYMENT_WINDOW_EXPIRED_REASON } from "../../../../features/orders/constants/payment-window.js";
 import { ORDER_MESSAGES } from "../handlers/messages.js";
 import { reconcileStuckGatewayEvents } from "./paymentReconcile.js";
+/* Phase 8 — the job retry sweep. Folded in here rather than given its own
+ * Scheduler job; see its own docstring and the 🛑 at the call site. */
+import { runDueJobRetries } from "./onJobCreated.js";
 /**
  * 15-minute payment window sweep — cancels + restocks any `upi_manual`/
  * `cash`/`emi` order still `pending`/`pending` with no proof uploaded past
@@ -53,6 +56,36 @@ export async function runPaymentWindowTimeout(ctx) {
             err,
         });
     }
+    /*
+     * ── Phase 8 rides along too: job retries whose backoff has elapsed ────────
+     *
+     * Same two reasons as R1 above, and the same placement rule: **FIRST, before
+     * the early return**. A retry is a `pending` job document with a future
+     * `nextAttemptAt`, and nothing else in the system will ever run it — the
+     * create trigger deliberately defers to this sweep, because creating the
+     * retry document fires that trigger instantly and would collapse the backoff
+     * to zero.
+     *
+     * So if this call sat after the early return, a retry would run only on a
+     * tick that happened to have an expired payment window — which is the
+     * minority — and a job scheduled on a quiet afternoon could wait hours.
+     * Root Cause #27 again, now with a second mechanism depending on it.
+     *
+     * Never throws, for the same reason R1 does not: a stuck job must not stop
+     * the payment-window cancellations this function exists for.
+     */
+    try {
+        const retries = await runDueJobRetries(ctx);
+        if (retries.ran > 0) {
+            ctx.logger.info("Phase 8: ran due job retries", { ...retries });
+        }
+    }
+    catch (err) {
+        void normalizeError(err);
+        ctx.logger.error("job retry sweep failed inside paymentWindowTimeout — continuing", {
+            err,
+        });
+    }
     ctx.logger.info("Scanning orders past their 15-minute payment deadline");
     const expired = await orderRepository.getExpiredPaymentDeadlines();
     if (expired.length === 0) {

package/dist/_internal/server/jobs/core/runJobOnce.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Run ONE attempt of a job, and decide what happens next (Phase 8).
+ *
+ * ## 🛑 ONE dispatch path, shared by the trigger and the retry sweep
+ *
+ * `handleJobCreated` is the `jobs/{jobId}` create trigger; `runDueJobRetries`
+ * is the sweep that picks up scheduled retries. Both call THIS, so there is
+ * exactly one place that knows how to run a job, classify its failure and
+ * schedule the next attempt. Two dispatch paths would drift the first time one
+ * of them learned something — which is the shape of every defect in this
+ * plan's § "Recurrent Root Cause Patterns" that begins "a fix applied to the
+ * route but never back-ported".
+ *
+ * ## 🛑 A retry is a NEW DOCUMENT
+ *
+ * `onJobCreated` fires on CREATE. Re-running a job by flipping its status back
+ * to `pending` would never wake the trigger, so the job would sit `pending`
+ * forever looking scheduled — a stall that reads as "queued". So a retry is a
+ * fresh `jobs` document carrying `attempt + 1`, `parentJobId` and a
+ * `nextAttemptAt`, and the sweep is what runs it when due.
+ *
+ * ## What is retryable
+ *
+ * `mapToHttpError(err).status >= 500`. Reused rather than invented: that
+ * function already encodes "is this our fault or the caller's" for every error
+ * class in the codebase, and a second classification would answer differently
+ * the first time either changed.
+ *
+ * A 4xx is terminal IMMEDIATELY — no backoff, no attempts burned. A validation
+ * failure does not become valid by waiting, and retrying it three times over
+ * 35 minutes just delays the case by 35 minutes.
+ */
+import { type JobDocument } from "../../../../features/jobs/schemas/firestore";
+import type { JobContext } from "../runtime/types";
+export interface RunJobOnceInput {
+    jobId: string;
+    job: JobDocument;
+}
+/**
+ * Run one attempt. Marks the job done, schedules a retry, or declares it dead.
+ *
+ * Never throws: a job's failure is recorded on the job, and letting it escape
+ * would make the Firebase trigger retry the whole handler — a second,
+ * uncontrolled retry mechanism layered under this one, which is precisely the
+ * doubling that cost 55 sessions in Root Cause #91 (`run.mjs`'s internal retry
+ * multiplying the pool's).
+ */
+export declare function runJobOnce(input: RunJobOnceInput, ctx: JobContext): Promise<void>;