@teincfood/core 0.9.15 → 0.9.16

package/dist/adapters/sync-failure.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Sync-failure reporter adapter.
+ *
+ * Core never depends on the observability SDK. The host app registers a
+ * reporter at boot (built on `@teincfood/observability`) and core forwards
+ * every failed command attempt so operators can see which device/app/user is
+ * struggling and go fix it.
+ */
+export type SyncFailurePhase = "outbox" | "sync_jobs";
+export interface SyncFailureReport {
+    /** Which queue produced the failure. */
+    phase: SyncFailurePhase;
+    business_id: string;
+    user_id?: string;
+    /** Stable per-install device id (matches the telemetry device). */
+    device_id?: string;
+    command_type: string;
+    command_id?: string;
+    entity_id?: string;
+    /** HTTP status when the failure came from a response. */
+    status?: number;
+    /** Backend/guard error code. */
+    code?: string;
+    message?: string;
+    attempts?: number;
+    /** True when the command was dead-lettered (no further auto-retry). */
+    terminal: boolean;
+    transport?: string;
+}
+export type SyncFailureReporter = (report: SyncFailureReport) => void;
+export declare function setSyncFailureReporter(next: SyncFailureReporter | null): void;
+/** Fire-and-forget; never throws, safe before a reporter is registered. */
+export declare function reportSyncFailure(report: SyncFailureReport): void;

package/dist/adapters/sync-failure.js

@@ -0,0 +1,23 @@
+/**
+ * Sync-failure reporter adapter.
+ *
+ * Core never depends on the observability SDK. The host app registers a
+ * reporter at boot (built on `@teincfood/observability`) and core forwards
+ * every failed command attempt so operators can see which device/app/user is
+ * struggling and go fix it.
+ */
+let reporter = null;
+export function setSyncFailureReporter(next) {
+    reporter = next;
+}
+/** Fire-and-forget; never throws, safe before a reporter is registered. */
+export function reportSyncFailure(report) {
+    if (!reporter)
+        return;
+    try {
+        reporter(report);
+    }
+    catch {
+        /* telemetry must never break sync */
+    }
+}

package/dist/index.d.ts

@@ -29,6 +29,7 @@ export * from "./utils/customer-display";
 export * from "./utils/order-filter";
 export { useReferenceData } from "./reference/hooks";
 export { setNotifyHandlers } from "./adapters/notify";
+export { setSyncFailureReporter, reportSyncFailure, type SyncFailureReport, type SyncFailureReporter, type SyncFailurePhase, } from "./adapters/sync-failure";
 export { walletService, riderService, fleetService, deliveryService, dashboardService, analyticsService, placesService } from "./adapters/services";
 export { buildMenuItemEnvelope, buildMenuCategoryEnvelope } from "./reference/menu-record";
 export { validateMenuItemInput, validateMenuCategoryInput, validateOrderCreateInput, validateInviteInput } from "./validation/offline";

package/dist/index.js

@@ -30,6 +30,7 @@ export * from "./utils/customer-display";
 export * from "./utils/order-filter";
 export { useReferenceData } from "./reference/hooks";
 export { setNotifyHandlers } from "./adapters/notify";
+export { setSyncFailureReporter, reportSyncFailure, } from "./adapters/sync-failure";
 export { walletService, riderService, fleetService, deliveryService, dashboardService, analyticsService, placesService } from "./adapters/services";
 export { buildMenuItemEnvelope, buildMenuCategoryEnvelope } from "./reference/menu-record";
 export { validateMenuItemInput, validateMenuCategoryInput, validateOrderCreateInput, validateInviteInput } from "./validation/offline";

package/dist/sync/command-queue.service.d.ts

@@ -24,6 +24,8 @@ declare class CommandQueueService {
      * Try to execute a single pending command through the current transport.
      */
     processCommand(command: PersistedCommand): Promise<void>;
+    /** Forward a failed attempt to the host's observability reporter. */
+    private reportFailure;
     /**
      * Mark a command permanently failed: loud toast + optimistic revert, then
      * replay continues to the next command. Shared by the exhausted-retry and

package/dist/sync/command-queue.service.js

@@ -21,7 +21,8 @@ import { generateLocalCalloutNumber, generateOrderNumberForBusiness, generateOrd
 import { getConnectivityState, subscribeConnectivity } from "../adapters/connectivity";
 import { canBusinessSync } from "./sync-guard";
 import { backoffDelayMs } from "./backoff";
-const MAX_RETRIES = 5;
+import { reportSyncFailure } from "../adapters/sync-failure";
+const MAX_ATTEMPTS = 8;
 /**
  * Stop a replay pass after this many consecutive transport-level failures
  * (network/timeout/5xx) so a dead server is not hammered once per queued
@@ -30,14 +31,8 @@ const MAX_RETRIES = 5;
  */
 const TRANSPORT_FAILURE_BREAKER = 3;
 /**
- * 4xx that deserve another chance: transient races and rate limits, never
- * permanent verdicts. Anything else 4xx (400 validation, 401 session,
- * 402 plan, 403 forbidden) fails fast with a toast. 409 ORDER_NUMBER_TAKEN
- * has its own regen path below and is not capped (every attempt mints a
- * new order number, so it is never a blind retry).
+ * In-memory next-eligible time per command (shared 2s→5min curve).
  */
-const RETRYABLE_4XX = new Set([404, 408, 409, 422, 429]);
-/** In-memory next-eligible time per command (shared 2s→5min curve). */
 const retryNotBefore = new Map();
 const QUEUED_STATUS_BY_COMMAND = {
     "order:accept": "accepted",
@@ -267,7 +262,7 @@ class CommandQueueService {
             timestamp: command.timestamp,
             payload: command.payload,
         };
-        if (command.retry_count >= MAX_RETRIES) {
+        if (command.retry_count >= MAX_ATTEMPTS) {
             await updateCommandStatus(db, command.command_id, "failed", "Max retry count exceeded");
             return;
         }
@@ -336,6 +331,13 @@ class CommandQueueService {
                     catch { }
                 }
                 await updateCommandStatus(db, command.command_id, "failed", `order_number_taken:${message}`);
+                this.reportFailure(command, {
+                    status: 409,
+                    code: "ORDER_NUMBER_TAKEN",
+                    message,
+                    attempts: command.retry_count + 1,
+                    terminal: true,
+                });
                 throw error;
             }
             const status = typeof apiErr?.status === "number" ? apiErr.status : undefined;
@@ -352,19 +354,28 @@ class CommandQueueService {
                 }
                 catch { }
             }
-            // Retryable 4xx (FIFO races, rate limits): back off on the shared
-            // 2s→5min curve, then fail loudly after MAX_RETRIES. Throw a
-            // transport-neutral deferral so replay STOPS here (FIFO ordering) —
-            // rethrowing the 4xx itself would make the loop `continue` past it.
-            if (status !== undefined && RETRYABLE_4XX.has(status)) {
+            // Every other failure is retryable on the shared 2s→5min backoff.
+            //
+            // 4xx (validation/permission/plan races) is bounded: a device stuck in
+            // a bad local state keeps retrying, then dead-letters so an operator is
+            // alerted and can visit the device. 5xx/network is an outage, not a
+            // device problem — it keeps retrying indefinitely so a long outage can
+            // never lose good orders. Every attempt is reported to observability.
+            const is4xxResponse = status !== undefined && status >= 400 && status < 500;
+            if (is4xxResponse) {
                 const attempts = command.retry_count + 1;
-                if (attempts >= MAX_RETRIES) {
-                    await this.failPermanently(db, command, message, `retryable 4xx exhausted after ${attempts} attempts`);
+                if (attempts >= MAX_ATTEMPTS) {
+                    await this.failPermanently(db, command, message, `4xx exhausted after ${attempts} attempts`, {
+                        status,
+                        code,
+                        attempts,
+                    });
                     return;
                 }
+                this.reportFailure(command, { status, code, message, attempts, terminal: false });
                 retryNotBefore.set(command.command_id, Date.now() + backoffDelayMs(command.retry_count));
                 await updateCommandStatus(db, command.command_id, "pending", `retry_4xx:${message}`, true);
-                syncLogger.warn("CommandQueueService", "Retryable 4xx — backing off", {
+                syncLogger.warn("CommandQueueService", "4xx — backing off", {
                     command_id: command.command_id,
                     command_type: command.command_type,
                     status,
@@ -372,30 +383,37 @@ class CommandQueueService {
                 });
                 throw new TransitionError(TRANSITION_ERROR_CODES.REPLAY_DEFERRED, `Retryable ${status} on ${command.command_type}, backing off`);
             }
-            const isNonRetryable = status !== undefined &&
-                status >= 400 &&
-                status < 500;
-            if (isNonRetryable) {
-                await this.failPermanently(db, command, message, "permanent 4xx");
-                // Do not throw — allow replay to continue to next command.
-                return;
-            }
-            // Network/timeout/5xx: NOT a rejection — keep the command retryable
-            // indefinitely (status back to pending, retry_count untouched) so a
-            // long outage never dead-letters the outbox via MAX_RETRIES. Replay
-            // stops here to preserve ordering; the next trigger retries.
+            // 5xx / network / timeout: keep the command retryable indefinitely
+            // (status back to pending, retry_count untouched) so a long outage never
+            // dead-letters the outbox. Replay defers this entity; the next trigger
+            // retries.
+            this.reportFailure(command, { status, code, message, attempts: command.retry_count, terminal: false });
             await updateCommandStatus(db, command.command_id, "pending", `network:${message}`);
             syncLogger.command(parsedCommand, "queued", { error: message, deferred: true });
             throw error;
         }
     }
+    /** Forward a failed attempt to the host's observability reporter. */
+    reportFailure(command, extra) {
+        reportSyncFailure({
+            phase: "outbox",
+            business_id: command.business_id,
+            user_id: command.user_id,
+            device_id: command.device_id,
+            command_type: command.command_type,
+            command_id: command.command_id,
+            entity_id: command.entity_id,
+            ...extra,
+        });
+    }
     /**
      * Mark a command permanently failed: loud toast + optimistic revert, then
      * replay continues to the next command. Shared by the exhausted-retry and
      * permanent-rejection paths.
      */
-    async failPermanently(db, command, message, reason) {
+    async failPermanently(db, command, message, reason, extra) {
         await updateCommandStatus(db, command.command_id, "failed", `permanent:${message}`);
+        this.reportFailure(command, { ...extra, message, terminal: true });
         syncLogger.warn("CommandQueueService", `Queued command failed (${reason})`, {
             command_id: command.command_id,
             command_type: command.command_type,

package/dist/sync/sync-jobs.service.js

@@ -19,11 +19,13 @@ import { getTransitionPath, DELIVERY_COMMANDS } from "./transition-endpoints";
 import { getQueuedStatus } from "../utils/order-transitions";
 import { extractApiErrorCode } from "../utils/error";
 import { canBusinessSync, isAuthenticated } from "./sync-guard";
+import { reportSyncFailure } from "../adapters/sync-failure";
+import { getAppInstanceId } from "../adapters/devices";
 const inflight = new Map();
 const retryTimers = new Map();
 const backoffTracker = new BackoffTracker();
-/** Blind 4xx retries stop here, loudly. 5xx/network retry indefinitely. */
-const MAX_SYNC_JOB_ATTEMPTS = 5;
+/** Bounded 4xx retries stop here, loudly. 5xx/network retry indefinitely. */
+const MAX_SYNC_JOB_ATTEMPTS = 8;
 function isOnline() {
     const s = getConnectivityState();
     return s.isOnline && s.isCloudReachable !== "unreachable";
@@ -65,6 +67,36 @@ function clearRetry(businessId) {
     }
     backoffTracker.reset(businessId);
 }
+/** Best-effort device id for failure reports (never throws). */
+function safeDeviceId() {
+    try {
+        return getAppInstanceId();
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Forward a failed job attempt to the host's observability reporter. */
+function reportJobFailure(businessId, row, extra) {
+    let userId;
+    try {
+        const p = JSON.parse(row.payload);
+        const candidate = p.user_id ?? p.user?.id;
+        if (typeof candidate === "string")
+            userId = candidate;
+    }
+    catch { }
+    reportSyncFailure({
+        phase: "sync_jobs",
+        business_id: businessId,
+        user_id: userId,
+        device_id: safeDeviceId(),
+        command_type: row.command_type,
+        command_id: row.id,
+        entity_id: row.entity_id,
+        ...extra,
+    });
+}
 async function tryPhxPush(businessId, payload) {
     try {
         const socket = phoenixSocket.getChannel?.(`business:${businessId}`);
@@ -385,38 +417,13 @@ export async function drainOne(businessId, entityId) {
             syncLogger.debug("SyncJobs", "401 deferred until login", { businessId });
             return;
         }
-        // Permanent 4xx: mark failed to stop the retry loop.
-        // Retryable (bounded, 5 attempts): 401 (refresh owns it below), 408,
-        // 409, 422, 429, any 404 (FIFO race: earlier command still in flight),
-        // BUSINESS_CLOSED (pending-by-design). Everything else 4xx (400
-        // validation, 402 plan, 403 forbidden) fails fast. Backend codes are
+        // Any 4xx is retryable with backoff (bounded); 5xx/network keeps retrying
+        // indefinitely. Every failed attempt is reported to observability so an
+        // operator can see the device/app/user and visit it. BUSINESS_CLOSED is a
+        // legitimate closed state (keep pending, re-pull hours). Backend codes are
         // exact (no message matching).
         const isBusinessClosed = rest.code === "BUSINESS_CLOSED" || rest.code === "HOURS_NOT_SYNCED";
-        const is4xx = rest.status && rest.status >= 400 && rest.status < 500;
-        const isRetryable4xx = rest.status === 401 || rest.status === 408 || rest.status === 409
-            || rest.status === 422 || rest.status === 429 || rest.status === 404
-            || isBusinessClosed;
-        if (is4xx && !isRetryable4xx) {
-            syncLogger.warn("SyncJobs", "Permanent 4xx — marking job failed to stop retry loop", { businessId, entityId, status: rest.status, code: rest.code, message: rest.message });
-            try {
-                await db.runAsync(`UPDATE sync_jobs SET status='failed', last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
-            }
-            catch { }
-            clearRetry(businessId);
-            try {
-                const { notifyError } = await import("../adapters/notify");
-                notifyError(`Server rejected ${row.command_type}: ${rest.message ?? rest.code ?? rest.status}`);
-            }
-            catch { }
-            // Retain optimistic row — pull will converge for transitions, create stays for manual retry.
-            if (row.command_type !== "order:create") {
-                try {
-                    void pullRemoteOrders(businessId).catch(() => { });
-                }
-                catch { }
-            }
-            return;
-        }
+        const is4xx = !!rest.status && rest.status >= 400 && rest.status < 500;
         // Already-applied absorb for transitions: the server already moved the
         // order (duplicate tap, earlier drain won the race) — complete the job
         // instead of failing it. Server truth only (hlc-guarded pull), never the
@@ -432,39 +439,43 @@ export async function drainOne(businessId, entityId) {
             }
             catch { }
         }
-        // Bounded retries: blind 4xx retries stop after 5 attempts with a loud
-        // toast (attempts only increments on the keep-pending path below).
-        if (is4xx && isRetryable4xx && (row.attempts ?? 0) + 1 >= 5) {
-            syncLogger.warn("SyncJobs", "Retryable 4xx exhausted — marking job failed", { businessId, entityId, status: rest.status, code: rest.code, attempts: row.attempts });
+        if (isBusinessClosed) {
+            syncLogger.warn("SyncJobs", "Business closed — keeping pending for retry after hours sync", { businessId, entityId, code: rest.code });
+            reportJobFailure(businessId, row, { status: rest.status, code: rest.code, message: rest.message, attempts: row.attempts, terminal: false });
             try {
-                await db.runAsync(`UPDATE sync_jobs SET status='failed', last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""} (after ${MAX_SYNC_JOB_ATTEMPTS} attempts)`.trim(), new Date().toISOString(), row.id);
+                await db.runAsync(`UPDATE sync_jobs SET last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
             }
             catch { }
-            clearRetry(businessId);
+            // Re-pull reference so hours merge converges, then retry with backoff.
             try {
-                const { notifyError } = await import("../adapters/notify");
-                notifyError(`Server rejected ${row.command_type}: ${rest.message ?? rest.code ?? rest.status} (after retries)`);
+                const { referenceSyncService } = await import("../reference/service");
+                void referenceSyncService.sync({ businessId, keys: ["operating-hours", "business:profile"] }).catch(() => { });
             }
             catch { }
+            scheduleRetry(businessId);
             return;
         }
-        if (isBusinessClosed) {
-            syncLogger.warn("SyncJobs", "Business closed — keeping pending for retry after hours sync", { businessId, entityId, code: rest.code });
+        const attempts = (row.attempts ?? 0) + 1;
+        // Bounded 4xx: after MAX_SYNC_JOB_ATTEMPTS, dead-letter (retained) with a
+        // loud toast + observability report so an operator can visit the device.
+        if (is4xx && attempts >= MAX_SYNC_JOB_ATTEMPTS) {
+            syncLogger.warn("SyncJobs", "4xx exhausted — marking job failed", { businessId, entityId, status: rest.status, code: rest.code, attempts: row.attempts });
             try {
-                await db.runAsync(`UPDATE sync_jobs SET last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
+                await db.runAsync(`UPDATE sync_jobs SET status='failed', last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""} (after ${MAX_SYNC_JOB_ATTEMPTS} attempts)`.trim(), new Date().toISOString(), row.id);
             }
             catch { }
-            // Re-pull reference so hours merge converges, then retry with backoff.
+            reportJobFailure(businessId, row, { status: rest.status, code: rest.code, message: rest.message, attempts, terminal: true });
+            clearRetry(businessId);
             try {
-                const { referenceSyncService } = await import("../reference/service");
-                void referenceSyncService.sync({ businessId, keys: ["operating-hours", "business:profile"] }).catch(() => { });
+                const { notifyError } = await import("../adapters/notify");
+                notifyError(`Server rejected ${row.command_type}: ${rest.message ?? rest.code ?? rest.status} (after retries)`);
             }
             catch { }
-            scheduleRetry(businessId);
             return;
         }
-        // Keep pending for retry (5xx / network) — auto retry with backoff
+        // 4xx (bounded) or 5xx/network (indefinite): keep pending and back off.
         syncLogger.debug("SyncJobs", "drainOne kept pending for retry", { businessId, entityId, status: rest.status, code: rest.code });
+        reportJobFailure(businessId, row, { status: rest.status, code: rest.code, message: rest.message, attempts, terminal: false });
         try {
             await db.runAsync(`UPDATE sync_jobs SET attempts = attempts + 1, last_error = ?, updated_at = ? WHERE id = ?;`, `${rest.status ?? "network"} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@teincfood/core",
-  "version": "0.9.15",
+  "version": "0.9.16",
   "description": "TeincFood shared offline-first core — types, sync engine, local DB, reference data, and repositories for mobile + desktop",
   "type": "module",
   "main": "dist/index.js",