@teincfood/core 0.9.15 → 0.9.17

package/dist/adapters/sync-failure.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Sync-failure reporter adapter.
+ *
+ * Core never depends on the observability SDK. The host app registers a
+ * reporter at boot (built on `@teincfood/observability`) and core forwards
+ * every failed command attempt so operators can see which device/app/user is
+ * struggling and go fix it.
+ */
+export type SyncFailurePhase = "outbox" | "sync_jobs";
+export interface SyncFailureReport {
+    /** Which queue produced the failure. */
+    phase: SyncFailurePhase;
+    business_id: string;
+    user_id?: string;
+    /** Stable per-install device id (matches the telemetry device). */
+    device_id?: string;
+    command_type: string;
+    command_id?: string;
+    entity_id?: string;
+    /** HTTP status when the failure came from a response. */
+    status?: number;
+    /** Backend/guard error code. */
+    code?: string;
+    message?: string;
+    attempts?: number;
+    /** True when the command was dead-lettered (no further auto-retry). */
+    terminal: boolean;
+    transport?: string;
+}
+export type SyncFailureReporter = (report: SyncFailureReport) => void;
+export declare function setSyncFailureReporter(next: SyncFailureReporter | null): void;
+/** Fire-and-forget; never throws, safe before a reporter is registered. */
+export declare function reportSyncFailure(report: SyncFailureReport): void;

package/dist/adapters/sync-failure.js

@@ -0,0 +1,23 @@
+/**
+ * Sync-failure reporter adapter.
+ *
+ * Core never depends on the observability SDK. The host app registers a
+ * reporter at boot (built on `@teincfood/observability`) and core forwards
+ * every failed command attempt so operators can see which device/app/user is
+ * struggling and go fix it.
+ */
+let reporter = null;
+export function setSyncFailureReporter(next) {
+    reporter = next;
+}
+/** Fire-and-forget; never throws, safe before a reporter is registered. */
+export function reportSyncFailure(report) {
+    if (!reporter)
+        return;
+    try {
+        reporter(report);
+    }
+    catch {
+        /* telemetry must never break sync */
+    }
+}

package/dist/index.d.ts

@@ -29,6 +29,8 @@ export * from "./utils/customer-display";
 export * from "./utils/order-filter";
 export { useReferenceData } from "./reference/hooks";
 export { setNotifyHandlers } from "./adapters/notify";
+export { setSyncFailureReporter, reportSyncFailure, type SyncFailureReport, type SyncFailureReporter, type SyncFailurePhase, } from "./adapters/sync-failure";
+export { noteRateLimited, isRateLimited, getRateLimitWaitMs, extractRetryAfterMs, shouldReportRateLimit, paceSync, waitForRateLimitClear, SYNC_MIN_INTERVAL_MS, resetRateLimitForTests, } from "./sync/rate-limit";
 export { walletService, riderService, fleetService, deliveryService, dashboardService, analyticsService, placesService } from "./adapters/services";
 export { buildMenuItemEnvelope, buildMenuCategoryEnvelope } from "./reference/menu-record";
 export { validateMenuItemInput, validateMenuCategoryInput, validateOrderCreateInput, validateInviteInput } from "./validation/offline";

package/dist/index.js

@@ -30,6 +30,8 @@ export * from "./utils/customer-display";
 export * from "./utils/order-filter";
 export { useReferenceData } from "./reference/hooks";
 export { setNotifyHandlers } from "./adapters/notify";
+export { setSyncFailureReporter, reportSyncFailure, } from "./adapters/sync-failure";
+export { noteRateLimited, isRateLimited, getRateLimitWaitMs, extractRetryAfterMs, shouldReportRateLimit, paceSync, waitForRateLimitClear, SYNC_MIN_INTERVAL_MS, resetRateLimitForTests, } from "./sync/rate-limit";
 export { walletService, riderService, fleetService, deliveryService, dashboardService, analyticsService, placesService } from "./adapters/services";
 export { buildMenuItemEnvelope, buildMenuCategoryEnvelope } from "./reference/menu-record";
 export { validateMenuItemInput, validateMenuCategoryInput, validateOrderCreateInput, validateInviteInput } from "./validation/offline";

package/dist/providers/query-client.js

@@ -28,6 +28,14 @@ export function createAppQueryClient(onMutationError) {
                             return false;
                     }
                     catch { }
+                    // 429 is back-pressure, not a transient blip — retrying immediately
+                    // deepens the storm. Background sync auto-resumes after the cooldown;
+                    // queries refetch on reconnect/invalidation.
+                    const status = error?.status;
+                    const code = error?.code
+                        ?? error?.response?.data?.error?.code;
+                    if (status === 429 || code === "RATE_LIMIT_EXCEEDED")
+                        return false;
                     // Never retry explicit offline short-circuits (axios guard / adapters).
                     const msg = error instanceof Error ? error.message : String(error ?? "");
                     if (msg.startsWith("Offline") || error?.code === "OFFLINE")

package/dist/reference/service.js

@@ -14,6 +14,7 @@ import { menuService, businessService, rolesService } from "../adapters/services
 import { cacheImageBatch, pruneImageCache } from "../adapters/image-cache";
 import { getConnectivityState } from "../adapters/connectivity";
 import { canBusinessSync } from "../sync/sync-guard";
+import { paceSync, isRateLimited } from "../sync/rate-limit";
 import { getReferenceData, getReferenceVersion, mergeReferenceData, writeServerReferenceData, } from "./operations";
 import { anchorServerTime } from "./entitlement-clock";
 import { isServerAuthoritativeKey } from "./entitlements";
@@ -90,6 +91,7 @@ class ReferenceSyncService {
             // endpoints, fall back to legacy full-fetch per key.
             let versions = null;
             try {
+                await paceSync();
                 versions = await fetchReferenceVersions(options.businessId);
             }
             catch (error) {
@@ -98,7 +100,14 @@ class ReferenceSyncService {
                 });
             }
             for (const key of keys) {
+                // Honour a global 429 cooldown: stop this pass; it resumes on the next
+                // trigger once the window clears.
+                if (isRateLimited()) {
+                    result.skipped.push(key);
+                    continue;
+                }
                 try {
+                    await paceSync();
                     const synced = await this.syncKey(db, key, options, versions);
                     if (synced) {
                         result.synced.push(key);

package/dist/sync/command-queue.service.d.ts

@@ -24,6 +24,8 @@ declare class CommandQueueService {
      * Try to execute a single pending command through the current transport.
      */
     processCommand(command: PersistedCommand): Promise<void>;
+    /** Forward a failed attempt to the host's observability reporter. */
+    private reportFailure;
     /**
      * Mark a command permanently failed: loud toast + optimistic revert, then
      * replay continues to the next command. Shared by the exhausted-retry and

package/dist/sync/command-queue.service.js

@@ -21,7 +21,9 @@ import { generateLocalCalloutNumber, generateOrderNumberForBusiness, generateOrd
 import { getConnectivityState, subscribeConnectivity } from "../adapters/connectivity";
 import { canBusinessSync } from "./sync-guard";
 import { backoffDelayMs } from "./backoff";
-const MAX_RETRIES = 5;
+import { reportSyncFailure } from "../adapters/sync-failure";
+import { paceSync, isRateLimited, getRateLimitWaitMs, extractRetryAfterMs, noteRateLimited, shouldReportRateLimit, } from "./rate-limit";
+const MAX_ATTEMPTS = 8;
 /**
  * Stop a replay pass after this many consecutive transport-level failures
  * (network/timeout/5xx) so a dead server is not hammered once per queued
@@ -30,14 +32,8 @@ const MAX_RETRIES = 5;
  */
 const TRANSPORT_FAILURE_BREAKER = 3;
 /**
- * 4xx that deserve another chance: transient races and rate limits, never
- * permanent verdicts. Anything else 4xx (400 validation, 401 session,
- * 402 plan, 403 forbidden) fails fast with a toast. 409 ORDER_NUMBER_TAKEN
- * has its own regen path below and is not capped (every attempt mints a
- * new order number, so it is never a blind retry).
+ * In-memory next-eligible time per command (shared 2s→5min curve).
  */
-const RETRYABLE_4XX = new Set([404, 408, 409, 422, 429]);
-/** In-memory next-eligible time per command (shared 2s→5min curve). */
 const retryNotBefore = new Map();
 const QUEUED_STATUS_BY_COMMAND = {
     "order:accept": "accepted",
@@ -267,7 +263,7 @@ class CommandQueueService {
             timestamp: command.timestamp,
             payload: command.payload,
         };
-        if (command.retry_count >= MAX_RETRIES) {
+        if (command.retry_count >= MAX_ATTEMPTS) {
             await updateCommandStatus(db, command.command_id, "failed", "Max retry count exceeded");
             return;
         }
@@ -280,6 +276,13 @@ class CommandQueueService {
         if (!this.syncEngine) {
             throw new Error("CommandQueueService: SyncEngine not initialized");
         }
+        // Global 429 cooldown: defer without consuming an attempt.
+        if (isRateLimited()) {
+            retryNotBefore.set(command.command_id, Date.now() + getRateLimitWaitMs());
+            throw new TransitionError(TRANSITION_ERROR_CODES.REPLAY_DEFERRED, "Rate limited, deferring queued command");
+        }
+        // Pace background replay so a deep queue drains at ~3 req/sec, not a burst.
+        await paceSync();
         await markCommandInFlight(db, command.command_id);
         try {
             const events = await this.syncEngine.execute(parsedCommand, { forReplay: true });
@@ -313,8 +316,31 @@ class CommandQueueService {
             }
             const apiErr = error;
             const message = error instanceof Error ? error.message : String(error);
-            // 409 ORDER_NUMBER_TAKEN is retryable once with new RRR (exact backend code, any envelope)
             const code = extractApiErrorCode(error);
+            // 429: open the global cooldown, defer this entity, and DO NOT consume a
+            // retry attempt — the command is fine, the server is only busy.
+            if (apiErr?.status === 429) {
+                const retryAfterMs = extractRetryAfterMs(error);
+                noteRateLimited(retryAfterMs);
+                if (shouldReportRateLimit()) {
+                    this.reportFailure(command, {
+                        status: 429,
+                        code: code ?? "RATE_LIMIT_EXCEEDED",
+                        message,
+                        attempts: command.retry_count,
+                        terminal: false,
+                    });
+                }
+                retryNotBefore.set(command.command_id, Date.now() + (retryAfterMs ?? getRateLimitWaitMs()));
+                await updateCommandStatus(db, command.command_id, "pending", `retry_429:${message}`);
+                syncLogger.warn("CommandQueueService", "Rate limited (429) — backing off", {
+                    command_id: command.command_id,
+                    command_type: command.command_type,
+                    retry_after_ms: retryAfterMs,
+                });
+                throw new TransitionError(TRANSITION_ERROR_CODES.REPLAY_DEFERRED, `Rate limited on ${command.command_type}, backing off`);
+            }
+            // 409 ORDER_NUMBER_TAKEN is retryable once with new RRR (exact backend code, any envelope)
             if (apiErr?.status === 409 && code === "ORDER_NUMBER_TAKEN") {
                 if (command.command_type === "order:create") {
                     try {
@@ -336,6 +362,13 @@ class CommandQueueService {
                     catch { }
                 }
                 await updateCommandStatus(db, command.command_id, "failed", `order_number_taken:${message}`);
+                this.reportFailure(command, {
+                    status: 409,
+                    code: "ORDER_NUMBER_TAKEN",
+                    message,
+                    attempts: command.retry_count + 1,
+                    terminal: true,
+                });
                 throw error;
             }
             const status = typeof apiErr?.status === "number" ? apiErr.status : undefined;
@@ -352,19 +385,28 @@ class CommandQueueService {
                 }
                 catch { }
             }
-            // Retryable 4xx (FIFO races, rate limits): back off on the shared
-            // 2s→5min curve, then fail loudly after MAX_RETRIES. Throw a
-            // transport-neutral deferral so replay STOPS here (FIFO ordering) —
-            // rethrowing the 4xx itself would make the loop `continue` past it.
-            if (status !== undefined && RETRYABLE_4XX.has(status)) {
+            // Every other failure is retryable on the shared 2s→5min backoff.
+            //
+            // 4xx (validation/permission/plan races) is bounded: a device stuck in
+            // a bad local state keeps retrying, then dead-letters so an operator is
+            // alerted and can visit the device. 5xx/network is an outage, not a
+            // device problem — it keeps retrying indefinitely so a long outage can
+            // never lose good orders. Every attempt is reported to observability.
+            const is4xxResponse = status !== undefined && status >= 400 && status < 500;
+            if (is4xxResponse) {
                 const attempts = command.retry_count + 1;
-                if (attempts >= MAX_RETRIES) {
-                    await this.failPermanently(db, command, message, `retryable 4xx exhausted after ${attempts} attempts`);
+                if (attempts >= MAX_ATTEMPTS) {
+                    await this.failPermanently(db, command, message, `4xx exhausted after ${attempts} attempts`, {
+                        status,
+                        code,
+                        attempts,
+                    });
                     return;
                 }
+                this.reportFailure(command, { status, code, message, attempts, terminal: false });
                 retryNotBefore.set(command.command_id, Date.now() + backoffDelayMs(command.retry_count));
                 await updateCommandStatus(db, command.command_id, "pending", `retry_4xx:${message}`, true);
-                syncLogger.warn("CommandQueueService", "Retryable 4xx — backing off", {
+                syncLogger.warn("CommandQueueService", "4xx — backing off", {
                     command_id: command.command_id,
                     command_type: command.command_type,
                     status,
@@ -372,30 +414,37 @@ class CommandQueueService {
                 });
                 throw new TransitionError(TRANSITION_ERROR_CODES.REPLAY_DEFERRED, `Retryable ${status} on ${command.command_type}, backing off`);
             }
-            const isNonRetryable = status !== undefined &&
-                status >= 400 &&
-                status < 500;
-            if (isNonRetryable) {
-                await this.failPermanently(db, command, message, "permanent 4xx");
-                // Do not throw — allow replay to continue to next command.
-                return;
-            }
-            // Network/timeout/5xx: NOT a rejection — keep the command retryable
-            // indefinitely (status back to pending, retry_count untouched) so a
-            // long outage never dead-letters the outbox via MAX_RETRIES. Replay
-            // stops here to preserve ordering; the next trigger retries.
+            // 5xx / network / timeout: keep the command retryable indefinitely
+            // (status back to pending, retry_count untouched) so a long outage never
+            // dead-letters the outbox. Replay defers this entity; the next trigger
+            // retries.
+            this.reportFailure(command, { status, code, message, attempts: command.retry_count, terminal: false });
             await updateCommandStatus(db, command.command_id, "pending", `network:${message}`);
             syncLogger.command(parsedCommand, "queued", { error: message, deferred: true });
             throw error;
         }
     }
+    /** Forward a failed attempt to the host's observability reporter. */
+    reportFailure(command, extra) {
+        reportSyncFailure({
+            phase: "outbox",
+            business_id: command.business_id,
+            user_id: command.user_id,
+            device_id: command.device_id,
+            command_type: command.command_type,
+            command_id: command.command_id,
+            entity_id: command.entity_id,
+            ...extra,
+        });
+    }
     /**
      * Mark a command permanently failed: loud toast + optimistic revert, then
      * replay continues to the next command. Shared by the exhausted-retry and
      * permanent-rejection paths.
      */
-    async failPermanently(db, command, message, reason) {
+    async failPermanently(db, command, message, reason, extra) {
         await updateCommandStatus(db, command.command_id, "failed", `permanent:${message}`);
+        this.reportFailure(command, { ...extra, message, terminal: true });
         syncLogger.warn("CommandQueueService", `Queued command failed (${reason})`, {
             command_id: command.command_id,
             command_type: command.command_type,

package/dist/sync/engine.js

@@ -25,6 +25,7 @@ import { openSyncDatabase } from "../db/connection";
 import { canBusinessSync } from "./sync-guard";
 import { TRANSITION_ERROR_CODES, TransitionError } from "./transition-errors";
 import { assertEntitledForCommand } from "../reference/entitlement-guard";
+import { noteRateLimited, extractRetryAfterMs } from "./rate-limit";
 function withTimeout(promise, ms) {
     return new Promise((resolve, reject) => {
         const timer = setTimeout(() => {
@@ -243,6 +244,12 @@ export class SyncEngine {
                     ? error.status
                     : undefined;
                 if (rejectionStatus !== undefined && rejectionStatus >= 400 && rejectionStatus < 500) {
+                    // 429 is transient back-pressure, not a verdict: open the global
+                    // cooldown so background sync pauses, then surface as usual (the
+                    // outbox replay path classifies 429 separately and defers).
+                    if (rejectionStatus === 429) {
+                        noteRateLimited(extractRetryAfterMs(error));
+                    }
                     syncLogger.warn("SyncEngine", "Command rejected by server", {
                         command_type: command.command_type,
                         command_id: command.command_id,

package/dist/sync/rate-limit.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Global rate-limit gate + pacer for outgoing SYNC requests.
+ *
+ * Goal: drain the offline queue "naturally" — paced at roughly the rate of
+ * real user activity (~3 req/sec) so the backend never sees a burst that looks
+ * like an attack. When the server does throttle us (429), every sync path
+ * pauses until the Retry-After window elapses instead of hammering.
+ *
+ * User-initiated taps are NOT paced here; human pace is already the natural
+ * rate. Only background drains/replays/pulls/reference go through paceSync().
+ */
+/** ~3 requests/second ceiling for background sync. */
+export declare const SYNC_MIN_INTERVAL_MS = 333;
+/** Record a 429: pause all sync until retryAfterMs (default 60s). */
+export declare function noteRateLimited(retryAfterMs?: number): void;
+/** Milliseconds remaining in the current cooldown (0 when not throttled). */
+export declare function getRateLimitWaitMs(): number;
+export declare function isRateLimited(): boolean;
+/**
+ * Whether this throttle window has already been reported. The first 429 of a
+ * window reports (terminal:false); repeats stay silent so a storm does not
+ * spam observability.
+ */
+export declare function shouldReportRateLimit(): boolean;
+export declare function getRateLimitHitCount(): number;
+/** Test/reset hook. */
+export declare function resetRateLimitForTests(): void;
+/**
+ * Extract Retry-After from an error: backend body
+ * `{ error: { retry_after_seconds } }` or a `retry-after` header.
+ */
+export declare function extractRetryAfterMs(error: unknown): number | undefined;
+/**
+ * Await a paced slot for the next background sync request. Serialized via a
+ * promise chain so concurrent callers still space out at SYNC_MIN_INTERVAL_MS.
+ */
+export declare function paceSync(): Promise<void>;
+/**
+ * Wait out an active cooldown. Returns false if we should abort this sync
+ * attempt instead of waiting (callers schedule their own retry).
+ */
+export declare function waitForRateLimitClear(): Promise<boolean>;

package/dist/sync/rate-limit.js

@@ -0,0 +1,104 @@
+/**
+ * Global rate-limit gate + pacer for outgoing SYNC requests.
+ *
+ * Goal: drain the offline queue "naturally" — paced at roughly the rate of
+ * real user activity (~3 req/sec) so the backend never sees a burst that looks
+ * like an attack. When the server does throttle us (429), every sync path
+ * pauses until the Retry-After window elapses instead of hammering.
+ *
+ * User-initiated taps are NOT paced here; human pace is already the natural
+ * rate. Only background drains/replays/pulls/reference go through paceSync().
+ */
+const DEFAULT_RETRY_AFTER_MS = 60000;
+/** ~3 requests/second ceiling for background sync. */
+export const SYNC_MIN_INTERVAL_MS = 333;
+let rateLimitedUntil = 0;
+let reportedWindowUntil = 0;
+let rateLimitHitsInWindow = 0;
+/** Record a 429: pause all sync until retryAfterMs (default 60s). */
+export function noteRateLimited(retryAfterMs) {
+    const delay = retryAfterMs && retryAfterMs > 0 ? retryAfterMs : DEFAULT_RETRY_AFTER_MS;
+    const until = Date.now() + delay;
+    if (until > rateLimitedUntil)
+        rateLimitedUntil = until;
+    if (until > reportedWindowUntil) {
+        reportedWindowUntil = until;
+        rateLimitHitsInWindow = 0;
+    }
+    rateLimitHitsInWindow += 1;
+}
+/** Milliseconds remaining in the current cooldown (0 when not throttled). */
+export function getRateLimitWaitMs() {
+    return Math.max(0, rateLimitedUntil - Date.now());
+}
+export function isRateLimited() {
+    return getRateLimitWaitMs() > 0;
+}
+/**
+ * Whether this throttle window has already been reported. The first 429 of a
+ * window reports (terminal:false); repeats stay silent so a storm does not
+ * spam observability.
+ */
+export function shouldReportRateLimit() {
+    return rateLimitHitsInWindow <= 1;
+}
+export function getRateLimitHitCount() {
+    return rateLimitHitsInWindow;
+}
+/** Test/reset hook. */
+export function resetRateLimitForTests() {
+    rateLimitedUntil = 0;
+    reportedWindowUntil = 0;
+    rateLimitHitsInWindow = 0;
+    lastSyncRequestAt = 0;
+    pacerChain = Promise.resolve();
+}
+/**
+ * Extract Retry-After from an error: backend body
+ * `{ error: { retry_after_seconds } }` or a `retry-after` header.
+ */
+export function extractRetryAfterMs(error) {
+    if (!error || typeof error !== "object")
+        return undefined;
+    const e = error;
+    const body = (e.response?.data ?? e);
+    const secs = body?.error?.retry_after_seconds ?? body?.retry_after_seconds ?? undefined;
+    if (typeof secs === "number" && secs > 0)
+        return secs * 1000;
+    const header = e.response?.headers?.["retry-after"];
+    const headerSecs = typeof header === "string" ? Number(header) : NaN;
+    if (Number.isFinite(headerSecs) && headerSecs > 0)
+        return headerSecs * 1000;
+    return undefined;
+}
+// ── Pacer ────────────────────────────────────────────────────────────────────
+let lastSyncRequestAt = 0;
+let pacerChain = Promise.resolve();
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Await a paced slot for the next background sync request. Serialized via a
+ * promise chain so concurrent callers still space out at SYNC_MIN_INTERVAL_MS.
+ */
+export function paceSync() {
+    const run = pacerChain.then(async () => {
+        const wait = Math.max(0, lastSyncRequestAt + SYNC_MIN_INTERVAL_MS - Date.now());
+        if (wait > 0)
+            await sleep(wait);
+        lastSyncRequestAt = Date.now();
+    });
+    pacerChain = run.catch(() => undefined);
+    return run;
+}
+/**
+ * Wait out an active cooldown. Returns false if we should abort this sync
+ * attempt instead of waiting (callers schedule their own retry).
+ */
+export async function waitForRateLimitClear() {
+    const wait = getRateLimitWaitMs();
+    if (wait <= 0)
+        return true;
+    await sleep(wait);
+    return getRateLimitWaitMs() <= 0;
+}

package/dist/sync/sync-jobs.service.js

@@ -19,11 +19,14 @@ import { getTransitionPath, DELIVERY_COMMANDS } from "./transition-endpoints";
 import { getQueuedStatus } from "../utils/order-transitions";
 import { extractApiErrorCode } from "../utils/error";
 import { canBusinessSync, isAuthenticated } from "./sync-guard";
+import { reportSyncFailure } from "../adapters/sync-failure";
+import { getAppInstanceId } from "../adapters/devices";
+import { paceSync, isRateLimited, getRateLimitWaitMs, extractRetryAfterMs, noteRateLimited, shouldReportRateLimit, } from "./rate-limit";
 const inflight = new Map();
 const retryTimers = new Map();
 const backoffTracker = new BackoffTracker();
-/** Blind 4xx retries stop here, loudly. 5xx/network retry indefinitely. */
-const MAX_SYNC_JOB_ATTEMPTS = 5;
+/** Bounded 4xx retries stop here, loudly. 5xx/network retry indefinitely. */
+const MAX_SYNC_JOB_ATTEMPTS = 8;
 function isOnline() {
     const s = getConnectivityState();
     return s.isOnline && s.isCloudReachable !== "unreachable";
@@ -37,15 +40,16 @@ function isOnlineForPhx() {
     const s = getConnectivityState();
     return s.isOnline && s.isCloudReachable === "reachable";
 }
-function scheduleRetry(businessId) {
+function scheduleRetry(businessId, delayOverrideMs) {
     if (!canBusinessSync(businessId))
         return;
     if (retryTimers.has(businessId))
         return;
     // Shared backoff curve: 2s doubling to a 5min cap with jitter. Attempts
     // reset on any success (clearRetry) so long outages go quiet instead of
-    // hammering a down backend every 30s.
-    const delay = backoffTracker.next(businessId);
+    // hammering a down backend every 30s. A 429 cooldown overrides with the
+    // server-provided Retry-After window.
+    const delay = delayOverrideMs ?? backoffTracker.next(businessId);
     recordAttempt("sync_drain", `retry-scheduled delay=${delay}`);
     syncLogger.debug("SyncJobs", "scheduleRetry backoff", { businessId, delay });
     const t = setTimeout(() => {
@@ -65,6 +69,36 @@ function clearRetry(businessId) {
     }
     backoffTracker.reset(businessId);
 }
+/** Best-effort device id for failure reports (never throws). */
+function safeDeviceId() {
+    try {
+        return getAppInstanceId();
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Forward a failed job attempt to the host's observability reporter. */
+function reportJobFailure(businessId, row, extra) {
+    let userId;
+    try {
+        const p = JSON.parse(row.payload);
+        const candidate = p.user_id ?? p.user?.id;
+        if (typeof candidate === "string")
+            userId = candidate;
+    }
+    catch { }
+    reportSyncFailure({
+        phase: "sync_jobs",
+        business_id: businessId,
+        user_id: userId,
+        device_id: safeDeviceId(),
+        command_type: row.command_type,
+        command_id: row.id,
+        entity_id: row.entity_id,
+        ...extra,
+    });
+}
 async function tryPhxPush(businessId, payload) {
     try {
         const socket = phoenixSocket.getChannel?.(`business:${businessId}`);
@@ -110,6 +144,8 @@ async function tryPhxPush(businessId, payload) {
 }
 async function tryRest(businessId, payload, commandType) {
     try {
+        // Pace background sync so a deep queue drains at ~3 req/sec, never a burst.
+        await paceSync();
         const http = getHttpClient();
         if (!http)
             return { ok: false };
@@ -210,6 +246,18 @@ async function tryRest(businessId, payload, commandType) {
         // Backend code via every envelope shape (core ApiError, axios, app ApiError).
         const bodyCode = extractApiErrorCode(e);
         const bodyMsg = err.response?.data?.error?.message;
+        // 429: open the global cooldown and let the caller reschedule — never count
+        // this as a job attempt (the command is fine; the server is busy).
+        if (status === 429) {
+            const retryAfterMs = extractRetryAfterMs(e);
+            noteRateLimited(retryAfterMs);
+            syncLogger.warn("SyncJobs", "Rate limited (429) — pausing sync", {
+                businessId,
+                commandType,
+                retryAfterMs,
+            });
+            return { ok: false, status, code: bodyCode ?? "RATE_LIMIT_EXCEEDED", message: bodyMsg ?? err.message, rateLimited: true, retryAfterMs };
+        }
         return {
             ok: false,
             status,
@@ -260,6 +308,13 @@ export async function drainOne(businessId, entityId) {
         syncLogger.debug("SyncJobs", "drainOne early-return gate", { businessId, entityId, onlineForRest, hasToken: isAuthenticated(), connectivity: state });
         return;
     }
+    // Global 429 cooldown: don't touch the network until the window clears.
+    if (isRateLimited()) {
+        const waitMs = getRateLimitWaitMs();
+        syncLogger.debug("SyncJobs", "drainOne deferred — rate limited", { businessId, entityId, waitMs });
+        scheduleRetry(businessId, waitMs);
+        return;
+    }
     const key = `${businessId}:${entityId}`;
     // Join an in-flight drain instead of skipping: concurrent drains (e.g. the
     // fire-and-forget drainOne after a write racing an awaited drainPending)
@@ -327,6 +382,22 @@ export async function drainOne(businessId, entityId) {
             catch { }
             return;
         }
+        // 429: global cooldown already opened by tryRest. Keep the job pending
+        // (no attempt consumed) and reschedule at the Retry-After window.
+        if (rest.rateLimited) {
+            if (shouldReportRateLimit()) {
+                reportJobFailure(businessId, row, {
+                    status: 429,
+                    code: rest.code ?? "RATE_LIMIT_EXCEEDED",
+                    message: rest.message,
+                    attempts: row.attempts,
+                    terminal: false,
+                });
+            }
+            const waitMs = rest.retryAfterMs && rest.retryAfterMs > 0 ? rest.retryAfterMs : getRateLimitWaitMs();
+            scheduleRetry(businessId, waitMs);
+            return;
+        }
         // 3) Handle 409 RRR regen same UUID (POS create only, exact backend code)
         syncLogger.debug("SyncJobs", "check 409", { businessId, entityId, status: rest.status, code: rest.code });
         const is409 = rest.status === 409 && rest.code === "ORDER_NUMBER_TAKEN";
@@ -385,38 +456,13 @@ export async function drainOne(businessId, entityId) {
             syncLogger.debug("SyncJobs", "401 deferred until login", { businessId });
             return;
         }
-        // Permanent 4xx: mark failed to stop the retry loop.
-        // Retryable (bounded, 5 attempts): 401 (refresh owns it below), 408,
-        // 409, 422, 429, any 404 (FIFO race: earlier command still in flight),
-        // BUSINESS_CLOSED (pending-by-design). Everything else 4xx (400
-        // validation, 402 plan, 403 forbidden) fails fast. Backend codes are
+        // Any 4xx is retryable with backoff (bounded); 5xx/network keeps retrying
+        // indefinitely. Every failed attempt is reported to observability so an
+        // operator can see the device/app/user and visit it. BUSINESS_CLOSED is a
+        // legitimate closed state (keep pending, re-pull hours). Backend codes are
         // exact (no message matching).
         const isBusinessClosed = rest.code === "BUSINESS_CLOSED" || rest.code === "HOURS_NOT_SYNCED";
-        const is4xx = rest.status && rest.status >= 400 && rest.status < 500;
-        const isRetryable4xx = rest.status === 401 || rest.status === 408 || rest.status === 409
-            || rest.status === 422 || rest.status === 429 || rest.status === 404
-            || isBusinessClosed;
-        if (is4xx && !isRetryable4xx) {
-            syncLogger.warn("SyncJobs", "Permanent 4xx — marking job failed to stop retry loop", { businessId, entityId, status: rest.status, code: rest.code, message: rest.message });
-            try {
-                await db.runAsync(`UPDATE sync_jobs SET status='failed', last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
-            }
-            catch { }
-            clearRetry(businessId);
-            try {
-                const { notifyError } = await import("../adapters/notify");
-                notifyError(`Server rejected ${row.command_type}: ${rest.message ?? rest.code ?? rest.status}`);
-            }
-            catch { }
-            // Retain optimistic row — pull will converge for transitions, create stays for manual retry.
-            if (row.command_type !== "order:create") {
-                try {
-                    void pullRemoteOrders(businessId).catch(() => { });
-                }
-                catch { }
-            }
-            return;
-        }
+        const is4xx = !!rest.status && rest.status >= 400 && rest.status < 500;
         // Already-applied absorb for transitions: the server already moved the
         // order (duplicate tap, earlier drain won the race) — complete the job
         // instead of failing it. Server truth only (hlc-guarded pull), never the
@@ -432,39 +478,43 @@ export async function drainOne(businessId, entityId) {
             }
             catch { }
         }
-        // Bounded retries: blind 4xx retries stop after 5 attempts with a loud
-        // toast (attempts only increments on the keep-pending path below).
-        if (is4xx && isRetryable4xx && (row.attempts ?? 0) + 1 >= 5) {
-            syncLogger.warn("SyncJobs", "Retryable 4xx exhausted — marking job failed", { businessId, entityId, status: rest.status, code: rest.code, attempts: row.attempts });
+        if (isBusinessClosed) {
+            syncLogger.warn("SyncJobs", "Business closed — keeping pending for retry after hours sync", { businessId, entityId, code: rest.code });
+            reportJobFailure(businessId, row, { status: rest.status, code: rest.code, message: rest.message, attempts: row.attempts, terminal: false });
             try {
-                await db.runAsync(`UPDATE sync_jobs SET status='failed', last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""} (after ${MAX_SYNC_JOB_ATTEMPTS} attempts)`.trim(), new Date().toISOString(), row.id);
+                await db.runAsync(`UPDATE sync_jobs SET last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
             }
             catch { }
-            clearRetry(businessId);
+            // Re-pull reference so hours merge converges, then retry with backoff.
             try {
-                const { notifyError } = await import("../adapters/notify");
-                notifyError(`Server rejected ${row.command_type}: ${rest.message ?? rest.code ?? rest.status} (after retries)`);
+                const { referenceSyncService } = await import("../reference/service");
+                void referenceSyncService.sync({ businessId, keys: ["operating-hours", "business:profile"] }).catch(() => { });
             }
             catch { }
+            scheduleRetry(businessId);
             return;
         }
-        if (isBusinessClosed) {
-            syncLogger.warn("SyncJobs", "Business closed — keeping pending for retry after hours sync", { businessId, entityId, code: rest.code });
+        const attempts = (row.attempts ?? 0) + 1;
+        // Bounded 4xx: after MAX_SYNC_JOB_ATTEMPTS, dead-letter (retained) with a
+        // loud toast + observability report so an operator can visit the device.
+        if (is4xx && attempts >= MAX_SYNC_JOB_ATTEMPTS) {
+            syncLogger.warn("SyncJobs", "4xx exhausted — marking job failed", { businessId, entityId, status: rest.status, code: rest.code, attempts: row.attempts });
             try {
-                await db.runAsync(`UPDATE sync_jobs SET last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
+                await db.runAsync(`UPDATE sync_jobs SET status='failed', last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""} (after ${MAX_SYNC_JOB_ATTEMPTS} attempts)`.trim(), new Date().toISOString(), row.id);
             }
             catch { }
-            // Re-pull reference so hours merge converges, then retry with backoff.
+            reportJobFailure(businessId, row, { status: rest.status, code: rest.code, message: rest.message, attempts, terminal: true });
+            clearRetry(businessId);
             try {
-                const { referenceSyncService } = await import("../reference/service");
-                void referenceSyncService.sync({ businessId, keys: ["operating-hours", "business:profile"] }).catch(() => { });
+                const { notifyError } = await import("../adapters/notify");
+                notifyError(`Server rejected ${row.command_type}: ${rest.message ?? rest.code ?? rest.status} (after retries)`);
             }
             catch { }
-            scheduleRetry(businessId);
             return;
         }
-        // Keep pending for retry (5xx / network) — auto retry with backoff
+        // 4xx (bounded) or 5xx/network (indefinite): keep pending and back off.
         syncLogger.debug("SyncJobs", "drainOne kept pending for retry", { businessId, entityId, status: rest.status, code: rest.code });
+        reportJobFailure(businessId, row, { status: rest.status, code: rest.code, message: rest.message, attempts, terminal: false });
         try {
             await db.runAsync(`UPDATE sync_jobs SET attempts = attempts + 1, last_error = ?, updated_at = ? WHERE id = ?;`, `${rest.status ?? "network"} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
         }
@@ -491,6 +541,11 @@ export async function pullRemoteOrders(businessId) {
         syncLogger.debug("SyncJobs", "pullRemoteOrders early-return gate", { businessId, onlineForRest, hasToken: isAuthenticated(), connectivity: getConnectivityState() });
         return;
     }
+    // Respect a global 429 cooldown for background pulls.
+    if (isRateLimited()) {
+        syncLogger.debug("SyncJobs", "pullRemoteOrders skipped — rate limited", { businessId });
+        return;
+    }
     try {
         const http = getHttpClient();
         if (!http) {
@@ -502,6 +557,7 @@ export async function pullRemoteOrders(businessId) {
         let totalPages = 1;
         do {
             syncLogger.debug("SyncJobs", "pullRemoteOrders GET start", { businessId, page });
+            await paceSync();
             const res = await http.get(`/businesses/${businessId}/orders`, { params: { page, page_size: 20 } });
             const payload = res.data;
             const data = payload?.data ?? res.data ?? [];
@@ -538,7 +594,8 @@ export async function pullRemoteOrders(businessId) {
     // Also pull recent deliveries (single page recent) for single business
     try {
         const http = getHttpClient();
-        if (http) {
+        if (http && !isRateLimited()) {
+            await paceSync();
             const r2 = await http.get(`/businesses/${businessId}/deliveries`, { params: { page: 1, page_size: 20 } });
             const d = r2.data?.data ?? [];
             syncLogger.debug("SyncJobs", "pullRemoteDeliveries result", { businessId, dataLen: d.length });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@teincfood/core",
-  "version": "0.9.15",
+  "version": "0.9.17",
   "description": "TeincFood shared offline-first core — types, sync engine, local DB, reference data, and repositories for mobile + desktop",
   "type": "module",
   "main": "dist/index.js",