@teincfood/core 0.9.16 → 0.9.17

package/dist/index.d.ts

@@ -30,6 +30,7 @@ export * from "./utils/order-filter";
 export { useReferenceData } from "./reference/hooks";
 export { setNotifyHandlers } from "./adapters/notify";
 export { setSyncFailureReporter, reportSyncFailure, type SyncFailureReport, type SyncFailureReporter, type SyncFailurePhase, } from "./adapters/sync-failure";
+export { noteRateLimited, isRateLimited, getRateLimitWaitMs, extractRetryAfterMs, shouldReportRateLimit, paceSync, waitForRateLimitClear, SYNC_MIN_INTERVAL_MS, resetRateLimitForTests, } from "./sync/rate-limit";
 export { walletService, riderService, fleetService, deliveryService, dashboardService, analyticsService, placesService } from "./adapters/services";
 export { buildMenuItemEnvelope, buildMenuCategoryEnvelope } from "./reference/menu-record";
 export { validateMenuItemInput, validateMenuCategoryInput, validateOrderCreateInput, validateInviteInput } from "./validation/offline";

package/dist/index.js

@@ -31,6 +31,7 @@ export * from "./utils/order-filter";
 export { useReferenceData } from "./reference/hooks";
 export { setNotifyHandlers } from "./adapters/notify";
 export { setSyncFailureReporter, reportSyncFailure, } from "./adapters/sync-failure";
+export { noteRateLimited, isRateLimited, getRateLimitWaitMs, extractRetryAfterMs, shouldReportRateLimit, paceSync, waitForRateLimitClear, SYNC_MIN_INTERVAL_MS, resetRateLimitForTests, } from "./sync/rate-limit";
 export { walletService, riderService, fleetService, deliveryService, dashboardService, analyticsService, placesService } from "./adapters/services";
 export { buildMenuItemEnvelope, buildMenuCategoryEnvelope } from "./reference/menu-record";
 export { validateMenuItemInput, validateMenuCategoryInput, validateOrderCreateInput, validateInviteInput } from "./validation/offline";

package/dist/providers/query-client.js

@@ -28,6 +28,14 @@ export function createAppQueryClient(onMutationError) {
                             return false;
                     }
                     catch { }
+                    // 429 is back-pressure, not a transient blip — retrying immediately
+                    // deepens the storm. Background sync auto-resumes after the cooldown;
+                    // queries refetch on reconnect/invalidation.
+                    const status = error?.status;
+                    const code = error?.code
+                        ?? error?.response?.data?.error?.code;
+                    if (status === 429 || code === "RATE_LIMIT_EXCEEDED")
+                        return false;
                     // Never retry explicit offline short-circuits (axios guard / adapters).
                     const msg = error instanceof Error ? error.message : String(error ?? "");
                     if (msg.startsWith("Offline") || error?.code === "OFFLINE")

package/dist/reference/service.js

@@ -14,6 +14,7 @@ import { menuService, businessService, rolesService } from "../adapters/services
 import { cacheImageBatch, pruneImageCache } from "../adapters/image-cache";
 import { getConnectivityState } from "../adapters/connectivity";
 import { canBusinessSync } from "../sync/sync-guard";
+import { paceSync, isRateLimited } from "../sync/rate-limit";
 import { getReferenceData, getReferenceVersion, mergeReferenceData, writeServerReferenceData, } from "./operations";
 import { anchorServerTime } from "./entitlement-clock";
 import { isServerAuthoritativeKey } from "./entitlements";
@@ -90,6 +91,7 @@ class ReferenceSyncService {
             // endpoints, fall back to legacy full-fetch per key.
             let versions = null;
             try {
+                await paceSync();
                 versions = await fetchReferenceVersions(options.businessId);
             }
             catch (error) {
@@ -98,7 +100,14 @@ class ReferenceSyncService {
                 });
             }
             for (const key of keys) {
+                // Honour a global 429 cooldown: stop this pass; it resumes on the next
+                // trigger once the window clears.
+                if (isRateLimited()) {
+                    result.skipped.push(key);
+                    continue;
+                }
                 try {
+                    await paceSync();
                     const synced = await this.syncKey(db, key, options, versions);
                     if (synced) {
                         result.synced.push(key);

package/dist/sync/command-queue.service.js

@@ -22,6 +22,7 @@ import { getConnectivityState, subscribeConnectivity } from "../adapters/connect
 import { canBusinessSync } from "./sync-guard";
 import { backoffDelayMs } from "./backoff";
 import { reportSyncFailure } from "../adapters/sync-failure";
+import { paceSync, isRateLimited, getRateLimitWaitMs, extractRetryAfterMs, noteRateLimited, shouldReportRateLimit, } from "./rate-limit";
 const MAX_ATTEMPTS = 8;
 /**
  * Stop a replay pass after this many consecutive transport-level failures
@@ -275,6 +276,13 @@ class CommandQueueService {
         if (!this.syncEngine) {
             throw new Error("CommandQueueService: SyncEngine not initialized");
         }
+        // Global 429 cooldown: defer without consuming an attempt.
+        if (isRateLimited()) {
+            retryNotBefore.set(command.command_id, Date.now() + getRateLimitWaitMs());
+            throw new TransitionError(TRANSITION_ERROR_CODES.REPLAY_DEFERRED, "Rate limited, deferring queued command");
+        }
+        // Pace background replay so a deep queue drains at ~3 req/sec, not a burst.
+        await paceSync();
         await markCommandInFlight(db, command.command_id);
         try {
             const events = await this.syncEngine.execute(parsedCommand, { forReplay: true });
@@ -308,8 +316,31 @@ class CommandQueueService {
             }
             const apiErr = error;
             const message = error instanceof Error ? error.message : String(error);
-            // 409 ORDER_NUMBER_TAKEN is retryable once with new RRR (exact backend code, any envelope)
             const code = extractApiErrorCode(error);
+            // 429: open the global cooldown, defer this entity, and DO NOT consume a
+            // retry attempt — the command is fine, the server is only busy.
+            if (apiErr?.status === 429) {
+                const retryAfterMs = extractRetryAfterMs(error);
+                noteRateLimited(retryAfterMs);
+                if (shouldReportRateLimit()) {
+                    this.reportFailure(command, {
+                        status: 429,
+                        code: code ?? "RATE_LIMIT_EXCEEDED",
+                        message,
+                        attempts: command.retry_count,
+                        terminal: false,
+                    });
+                }
+                retryNotBefore.set(command.command_id, Date.now() + (retryAfterMs ?? getRateLimitWaitMs()));
+                await updateCommandStatus(db, command.command_id, "pending", `retry_429:${message}`);
+                syncLogger.warn("CommandQueueService", "Rate limited (429) — backing off", {
+                    command_id: command.command_id,
+                    command_type: command.command_type,
+                    retry_after_ms: retryAfterMs,
+                });
+                throw new TransitionError(TRANSITION_ERROR_CODES.REPLAY_DEFERRED, `Rate limited on ${command.command_type}, backing off`);
+            }
+            // 409 ORDER_NUMBER_TAKEN is retryable once with new RRR (exact backend code, any envelope)
             if (apiErr?.status === 409 && code === "ORDER_NUMBER_TAKEN") {
                 if (command.command_type === "order:create") {
                     try {

package/dist/sync/engine.js

@@ -25,6 +25,7 @@ import { openSyncDatabase } from "../db/connection";
 import { canBusinessSync } from "./sync-guard";
 import { TRANSITION_ERROR_CODES, TransitionError } from "./transition-errors";
 import { assertEntitledForCommand } from "../reference/entitlement-guard";
+import { noteRateLimited, extractRetryAfterMs } from "./rate-limit";
 function withTimeout(promise, ms) {
     return new Promise((resolve, reject) => {
         const timer = setTimeout(() => {
@@ -243,6 +244,12 @@ export class SyncEngine {
                     ? error.status
                     : undefined;
                 if (rejectionStatus !== undefined && rejectionStatus >= 400 && rejectionStatus < 500) {
+                    // 429 is transient back-pressure, not a verdict: open the global
+                    // cooldown so background sync pauses, then surface as usual (the
+                    // outbox replay path classifies 429 separately and defers).
+                    if (rejectionStatus === 429) {
+                        noteRateLimited(extractRetryAfterMs(error));
+                    }
                     syncLogger.warn("SyncEngine", "Command rejected by server", {
                         command_type: command.command_type,
                         command_id: command.command_id,

package/dist/sync/rate-limit.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Global rate-limit gate + pacer for outgoing SYNC requests.
+ *
+ * Goal: drain the offline queue "naturally" — paced at roughly the rate of
+ * real user activity (~3 req/sec) so the backend never sees a burst that looks
+ * like an attack. When the server does throttle us (429), every sync path
+ * pauses until the Retry-After window elapses instead of hammering.
+ *
+ * User-initiated taps are NOT paced here; human pace is already the natural
+ * rate. Only background drains/replays/pulls/reference go through paceSync().
+ */
+/** ~3 requests/second ceiling for background sync. */
+export declare const SYNC_MIN_INTERVAL_MS = 333;
+/** Record a 429: pause all sync until retryAfterMs (default 60s). */
+export declare function noteRateLimited(retryAfterMs?: number): void;
+/** Milliseconds remaining in the current cooldown (0 when not throttled). */
+export declare function getRateLimitWaitMs(): number;
+export declare function isRateLimited(): boolean;
+/**
+ * Whether this throttle window has already been reported. The first 429 of a
+ * window reports (terminal:false); repeats stay silent so a storm does not
+ * spam observability.
+ */
+export declare function shouldReportRateLimit(): boolean;
+export declare function getRateLimitHitCount(): number;
+/** Test/reset hook. */
+export declare function resetRateLimitForTests(): void;
+/**
+ * Extract Retry-After from an error: backend body
+ * `{ error: { retry_after_seconds } }` or a `retry-after` header.
+ */
+export declare function extractRetryAfterMs(error: unknown): number | undefined;
+/**
+ * Await a paced slot for the next background sync request. Serialized via a
+ * promise chain so concurrent callers still space out at SYNC_MIN_INTERVAL_MS.
+ */
+export declare function paceSync(): Promise<void>;
+/**
+ * Wait out an active cooldown. Returns false if we should abort this sync
+ * attempt instead of waiting (callers schedule their own retry).
+ */
+export declare function waitForRateLimitClear(): Promise<boolean>;

package/dist/sync/rate-limit.js

@@ -0,0 +1,104 @@
+/**
+ * Global rate-limit gate + pacer for outgoing SYNC requests.
+ *
+ * Goal: drain the offline queue "naturally" — paced at roughly the rate of
+ * real user activity (~3 req/sec) so the backend never sees a burst that looks
+ * like an attack. When the server does throttle us (429), every sync path
+ * pauses until the Retry-After window elapses instead of hammering.
+ *
+ * User-initiated taps are NOT paced here; human pace is already the natural
+ * rate. Only background drains/replays/pulls/reference go through paceSync().
+ */
+const DEFAULT_RETRY_AFTER_MS = 60000;
+/** ~3 requests/second ceiling for background sync. */
+export const SYNC_MIN_INTERVAL_MS = 333;
+let rateLimitedUntil = 0;
+let reportedWindowUntil = 0;
+let rateLimitHitsInWindow = 0;
+/** Record a 429: pause all sync until retryAfterMs (default 60s). */
+export function noteRateLimited(retryAfterMs) {
+    const delay = retryAfterMs && retryAfterMs > 0 ? retryAfterMs : DEFAULT_RETRY_AFTER_MS;
+    const until = Date.now() + delay;
+    if (until > rateLimitedUntil)
+        rateLimitedUntil = until;
+    if (until > reportedWindowUntil) {
+        reportedWindowUntil = until;
+        rateLimitHitsInWindow = 0;
+    }
+    rateLimitHitsInWindow += 1;
+}
+/** Milliseconds remaining in the current cooldown (0 when not throttled). */
+export function getRateLimitWaitMs() {
+    return Math.max(0, rateLimitedUntil - Date.now());
+}
+export function isRateLimited() {
+    return getRateLimitWaitMs() > 0;
+}
+/**
+ * Whether this throttle window has already been reported. The first 429 of a
+ * window reports (terminal:false); repeats stay silent so a storm does not
+ * spam observability.
+ */
+export function shouldReportRateLimit() {
+    return rateLimitHitsInWindow <= 1;
+}
+export function getRateLimitHitCount() {
+    return rateLimitHitsInWindow;
+}
+/** Test/reset hook. */
+export function resetRateLimitForTests() {
+    rateLimitedUntil = 0;
+    reportedWindowUntil = 0;
+    rateLimitHitsInWindow = 0;
+    lastSyncRequestAt = 0;
+    pacerChain = Promise.resolve();
+}
+/**
+ * Extract Retry-After from an error: backend body
+ * `{ error: { retry_after_seconds } }` or a `retry-after` header.
+ */
+export function extractRetryAfterMs(error) {
+    if (!error || typeof error !== "object")
+        return undefined;
+    const e = error;
+    const body = (e.response?.data ?? e);
+    const secs = body?.error?.retry_after_seconds ?? body?.retry_after_seconds ?? undefined;
+    if (typeof secs === "number" && secs > 0)
+        return secs * 1000;
+    const header = e.response?.headers?.["retry-after"];
+    const headerSecs = typeof header === "string" ? Number(header) : NaN;
+    if (Number.isFinite(headerSecs) && headerSecs > 0)
+        return headerSecs * 1000;
+    return undefined;
+}
+// ── Pacer ────────────────────────────────────────────────────────────────────
+let lastSyncRequestAt = 0;
+let pacerChain = Promise.resolve();
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Await a paced slot for the next background sync request. Serialized via a
+ * promise chain so concurrent callers still space out at SYNC_MIN_INTERVAL_MS.
+ */
+export function paceSync() {
+    const run = pacerChain.then(async () => {
+        const wait = Math.max(0, lastSyncRequestAt + SYNC_MIN_INTERVAL_MS - Date.now());
+        if (wait > 0)
+            await sleep(wait);
+        lastSyncRequestAt = Date.now();
+    });
+    pacerChain = run.catch(() => undefined);
+    return run;
+}
+/**
+ * Wait out an active cooldown. Returns false if we should abort this sync
+ * attempt instead of waiting (callers schedule their own retry).
+ */
+export async function waitForRateLimitClear() {
+    const wait = getRateLimitWaitMs();
+    if (wait <= 0)
+        return true;
+    await sleep(wait);
+    return getRateLimitWaitMs() <= 0;
+}

package/dist/sync/sync-jobs.service.js

@@ -21,6 +21,7 @@ import { extractApiErrorCode } from "../utils/error";
 import { canBusinessSync, isAuthenticated } from "./sync-guard";
 import { reportSyncFailure } from "../adapters/sync-failure";
 import { getAppInstanceId } from "../adapters/devices";
+import { paceSync, isRateLimited, getRateLimitWaitMs, extractRetryAfterMs, noteRateLimited, shouldReportRateLimit, } from "./rate-limit";
 const inflight = new Map();
 const retryTimers = new Map();
 const backoffTracker = new BackoffTracker();
@@ -39,15 +40,16 @@ function isOnlineForPhx() {
     const s = getConnectivityState();
     return s.isOnline && s.isCloudReachable === "reachable";
 }
-function scheduleRetry(businessId) {
+function scheduleRetry(businessId, delayOverrideMs) {
     if (!canBusinessSync(businessId))
         return;
     if (retryTimers.has(businessId))
         return;
     // Shared backoff curve: 2s doubling to a 5min cap with jitter. Attempts
     // reset on any success (clearRetry) so long outages go quiet instead of
-    // hammering a down backend every 30s.
-    const delay = backoffTracker.next(businessId);
+    // hammering a down backend every 30s. A 429 cooldown overrides with the
+    // server-provided Retry-After window.
+    const delay = delayOverrideMs ?? backoffTracker.next(businessId);
     recordAttempt("sync_drain", `retry-scheduled delay=${delay}`);
     syncLogger.debug("SyncJobs", "scheduleRetry backoff", { businessId, delay });
     const t = setTimeout(() => {
@@ -142,6 +144,8 @@ async function tryPhxPush(businessId, payload) {
 }
 async function tryRest(businessId, payload, commandType) {
     try {
+        // Pace background sync so a deep queue drains at ~3 req/sec, never a burst.
+        await paceSync();
         const http = getHttpClient();
         if (!http)
             return { ok: false };
@@ -242,6 +246,18 @@ async function tryRest(businessId, payload, commandType) {
         // Backend code via every envelope shape (core ApiError, axios, app ApiError).
         const bodyCode = extractApiErrorCode(e);
         const bodyMsg = err.response?.data?.error?.message;
+        // 429: open the global cooldown and let the caller reschedule — never count
+        // this as a job attempt (the command is fine; the server is busy).
+        if (status === 429) {
+            const retryAfterMs = extractRetryAfterMs(e);
+            noteRateLimited(retryAfterMs);
+            syncLogger.warn("SyncJobs", "Rate limited (429) — pausing sync", {
+                businessId,
+                commandType,
+                retryAfterMs,
+            });
+            return { ok: false, status, code: bodyCode ?? "RATE_LIMIT_EXCEEDED", message: bodyMsg ?? err.message, rateLimited: true, retryAfterMs };
+        }
         return {
             ok: false,
             status,
@@ -292,6 +308,13 @@ export async function drainOne(businessId, entityId) {
         syncLogger.debug("SyncJobs", "drainOne early-return gate", { businessId, entityId, onlineForRest, hasToken: isAuthenticated(), connectivity: state });
         return;
     }
+    // Global 429 cooldown: don't touch the network until the window clears.
+    if (isRateLimited()) {
+        const waitMs = getRateLimitWaitMs();
+        syncLogger.debug("SyncJobs", "drainOne deferred — rate limited", { businessId, entityId, waitMs });
+        scheduleRetry(businessId, waitMs);
+        return;
+    }
     const key = `${businessId}:${entityId}`;
     // Join an in-flight drain instead of skipping: concurrent drains (e.g. the
     // fire-and-forget drainOne after a write racing an awaited drainPending)
@@ -359,6 +382,22 @@ export async function drainOne(businessId, entityId) {
             catch { }
             return;
         }
+        // 429: global cooldown already opened by tryRest. Keep the job pending
+        // (no attempt consumed) and reschedule at the Retry-After window.
+        if (rest.rateLimited) {
+            if (shouldReportRateLimit()) {
+                reportJobFailure(businessId, row, {
+                    status: 429,
+                    code: rest.code ?? "RATE_LIMIT_EXCEEDED",
+                    message: rest.message,
+                    attempts: row.attempts,
+                    terminal: false,
+                });
+            }
+            const waitMs = rest.retryAfterMs && rest.retryAfterMs > 0 ? rest.retryAfterMs : getRateLimitWaitMs();
+            scheduleRetry(businessId, waitMs);
+            return;
+        }
         // 3) Handle 409 RRR regen same UUID (POS create only, exact backend code)
         syncLogger.debug("SyncJobs", "check 409", { businessId, entityId, status: rest.status, code: rest.code });
         const is409 = rest.status === 409 && rest.code === "ORDER_NUMBER_TAKEN";
@@ -502,6 +541,11 @@ export async function pullRemoteOrders(businessId) {
         syncLogger.debug("SyncJobs", "pullRemoteOrders early-return gate", { businessId, onlineForRest, hasToken: isAuthenticated(), connectivity: getConnectivityState() });
         return;
     }
+    // Respect a global 429 cooldown for background pulls.
+    if (isRateLimited()) {
+        syncLogger.debug("SyncJobs", "pullRemoteOrders skipped — rate limited", { businessId });
+        return;
+    }
     try {
         const http = getHttpClient();
         if (!http) {
@@ -513,6 +557,7 @@ export async function pullRemoteOrders(businessId) {
         let totalPages = 1;
         do {
             syncLogger.debug("SyncJobs", "pullRemoteOrders GET start", { businessId, page });
+            await paceSync();
             const res = await http.get(`/businesses/${businessId}/orders`, { params: { page, page_size: 20 } });
             const payload = res.data;
             const data = payload?.data ?? res.data ?? [];
@@ -549,7 +594,8 @@ export async function pullRemoteOrders(businessId) {
     // Also pull recent deliveries (single page recent) for single business
     try {
         const http = getHttpClient();
-        if (http) {
+        if (http && !isRateLimited()) {
+            await paceSync();
             const r2 = await http.get(`/businesses/${businessId}/deliveries`, { params: { page: 1, page_size: 20 } });
             const d = r2.data?.data ?? [];
             syncLogger.debug("SyncJobs", "pullRemoteDeliveries result", { businessId, dataLen: d.length });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@teincfood/core",
-  "version": "0.9.16",
+  "version": "0.9.17",
   "description": "TeincFood shared offline-first core — types, sync engine, local DB, reference data, and repositories for mobile + desktop",
   "type": "module",
   "main": "dist/index.js",