@teincfood/core 0.9.14 → 0.9.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,33 @@
1
+ /**
2
+ * Sync-failure reporter adapter.
3
+ *
4
+ * Core never depends on the observability SDK. The host app registers a
5
+ * reporter at boot (built on `@teincfood/observability`) and core forwards
6
+ * every failed command attempt so operators can see which device/app/user is
7
+ * struggling and go fix it.
8
+ */
9
+ export type SyncFailurePhase = "outbox" | "sync_jobs";
10
+ export interface SyncFailureReport {
11
+ /** Which queue produced the failure. */
12
+ phase: SyncFailurePhase;
13
+ business_id: string;
14
+ user_id?: string;
15
+ /** Stable per-install device id (matches the telemetry device). */
16
+ device_id?: string;
17
+ command_type: string;
18
+ command_id?: string;
19
+ entity_id?: string;
20
+ /** HTTP status when the failure came from a response. */
21
+ status?: number;
22
+ /** Backend/guard error code. */
23
+ code?: string;
24
+ message?: string;
25
+ attempts?: number;
26
+ /** True when the command was dead-lettered (no further auto-retry). */
27
+ terminal: boolean;
28
+ transport?: string;
29
+ }
30
+ export type SyncFailureReporter = (report: SyncFailureReport) => void;
31
+ export declare function setSyncFailureReporter(next: SyncFailureReporter | null): void;
32
+ /** Fire-and-forget; never throws, safe before a reporter is registered. */
33
+ export declare function reportSyncFailure(report: SyncFailureReport): void;
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Sync-failure reporter adapter.
3
+ *
4
+ * Core never depends on the observability SDK. The host app registers a
5
+ * reporter at boot (built on `@teincfood/observability`) and core forwards
6
+ * every failed command attempt so operators can see which device/app/user is
7
+ * struggling and go fix it.
8
+ */
9
+ let reporter = null;
10
+ export function setSyncFailureReporter(next) {
11
+ reporter = next;
12
+ }
13
+ /** Fire-and-forget; never throws, safe before a reporter is registered. */
14
+ export function reportSyncFailure(report) {
15
+ if (!reporter)
16
+ return;
17
+ try {
18
+ reporter(report);
19
+ }
20
+ catch {
21
+ /* telemetry must never break sync */
22
+ }
23
+ }
@@ -19,6 +19,7 @@ export async function openSyncDatabase() {
19
19
  const driver = getSqliteDriver();
20
20
  initPromise = (async () => {
21
21
  await runMigrations(driver);
22
+ await recoverInFlightCommands(driver);
22
23
  dbInstance = driver;
23
24
  syncLogger.info("SQLite", "Database opened and migrated", {
24
25
  name: DB_NAME,
@@ -71,6 +72,30 @@ async function runMigrations(db) {
71
72
  await doMigrate();
72
73
  }
73
74
  }
75
+ /**
76
+ * Crash recovery: a process kill between `markCommandInFlight` and the
77
+ * command's terminal update leaves rows stuck in `in_flight`. They are
78
+ * invisible to the outbox query (which selects pending only), so the order
79
+ * would silently never sync. Reset them to pending on open so the next replay
80
+ * picks them up. Idempotent and safe: a fresh process owns all in-flight work.
81
+ */
82
+ async function recoverInFlightCommands(db) {
83
+ try {
84
+ const result = await db.runAsync("UPDATE commands SET status = 'pending', error = NULL, updated_at = ? WHERE status = 'in_flight';", new Date().toISOString());
85
+ const changes = result?.changes ?? 0;
86
+ if (changes > 0) {
87
+ syncLogger.warn("SQLite", "Recovered in-flight commands after restart", {
88
+ count: changes,
89
+ });
90
+ }
91
+ }
92
+ catch (error) {
93
+ // Never block DB open on recovery — replay/backoff still works.
94
+ syncLogger.error("SQLite", "in-flight recovery failed", {
95
+ error: String(error),
96
+ });
97
+ }
98
+ }
74
99
  export async function closeSyncDatabase() {
75
100
  if (dbInstance) {
76
101
  if (dbInstance.closeAsync)
@@ -8,7 +8,7 @@ export interface InsertCommandInput {
8
8
  status?: PersistedCommand["status"];
9
9
  }
10
10
  export declare function insertCommand(db: SQLiteDatabase, input: InsertCommandInput): Promise<PersistedCommand>;
11
- export declare function getPendingCommands(db: SQLiteDatabase, limit?: number): Promise<PersistedCommand[]>;
11
+ export declare function getPendingCommands(db: SQLiteDatabase, limit?: number, includeFailed?: boolean): Promise<PersistedCommand[]>;
12
12
  export declare function updateCommandStatus(db: SQLiteDatabase, commandId: string, status: PersistedCommand["status"], error?: string, incrementRetry?: boolean): Promise<void>;
13
13
  export declare function markCommandInFlight(db: SQLiteDatabase, commandId: string): Promise<void>;
14
14
  export declare function deleteCommand(db: SQLiteDatabase, commandId: string): Promise<void>;
@@ -24,10 +24,15 @@ export async function insertCommand(db, input) {
24
24
  syncLogger.command(command, "queued");
25
25
  return rowToPersistedCommand(row);
26
26
  }
27
- export async function getPendingCommands(db, limit = 100) {
27
+ export async function getPendingCommands(db, limit = 100, includeFailed = false) {
28
+ // `failed` rows are terminal (permanent rejection / retries exhausted) — they
29
+ // must NOT be re-fetched by the replay loop, or every failed command would be
30
+ // re-POSTed and re-toasted on each pass. Diagnostics opts in via
31
+ // includeFailed for the full outbox dump.
32
+ const statuses = includeFailed ? "('pending', 'failed')" : "('pending')";
28
33
  const rows = await db.getAllAsync(`
29
34
  SELECT * FROM commands
30
- WHERE status IN ('pending', 'failed')
35
+ WHERE status IN ${statuses}
31
36
  ORDER BY created_at ASC
32
37
  LIMIT ?;
33
38
  `, limit);
@@ -0,0 +1,24 @@
1
+ /**
2
+ * React hook for startup readiness (seller POS/order surfaces).
3
+ *
4
+ * Runs one bounded catch-up on mount (and whenever the cloud becomes
5
+ * reachable), then exposes the local cache readiness so the app can block the
6
+ * POS with an explicit "preparing offline data" state instead of surprising
7
+ * the seller with a guard failure at order-tap time.
8
+ */
9
+ import { type StartupReadiness } from "../sync/startup-readiness";
10
+ export interface UseStartupReadinessOptions {
11
+ /** Bound the online catch-up attempt. Defaults to 15000ms. */
12
+ timeoutMs?: number;
13
+ /** Set false to skip automatic checking. */
14
+ enabled?: boolean;
15
+ }
16
+ export interface UseStartupReadinessResult {
17
+ businessId: string | undefined;
18
+ readiness: StartupReadiness | null;
19
+ /** True while the catch-up check is running (first load only). */
20
+ checking: boolean;
21
+ /** Re-run the readiness check on demand (e.g. a Retry button). */
22
+ recheck: () => Promise<void>;
23
+ }
24
+ export declare function useStartupReadiness(options?: UseStartupReadinessOptions): UseStartupReadinessResult;
@@ -0,0 +1,61 @@
1
+ /**
2
+ * React hook for startup readiness (seller POS/order surfaces).
3
+ *
4
+ * Runs one bounded catch-up on mount (and whenever the cloud becomes
5
+ * reachable), then exposes the local cache readiness so the app can block the
6
+ * POS with an explicit "preparing offline data" state instead of surprising
7
+ * the seller with a guard failure at order-tap time.
8
+ */
9
+ import { useCallback, useEffect, useRef, useState } from "react";
10
+ import { useSyncExternalStore } from "react";
11
+ import { getActiveBusiness, subscribeBusiness } from "../adapters/business";
12
+ import { getConnectivityState, subscribeConnectivity } from "../adapters/connectivity";
13
+ import { ensureStartupReady, } from "../sync/startup-readiness";
14
+ function useActiveBusinessId() {
15
+ const id = useSyncExternalStore(subscribeBusiness, () => getActiveBusiness()?.id ?? null, () => null);
16
+ return id ?? undefined;
17
+ }
18
+ function useCloudStatus() {
19
+ return useSyncExternalStore(subscribeConnectivity, () => getConnectivityState().isCloudReachable, () => "unknown");
20
+ }
21
+ export function useStartupReadiness(options = {}) {
22
+ const { timeoutMs, enabled = true } = options;
23
+ const businessId = useActiveBusinessId();
24
+ const cloudStatus = useCloudStatus();
25
+ const [readiness, setReadiness] = useState(null);
26
+ const [checking, setChecking] = useState(false);
27
+ const runRef = useRef(false);
28
+ const check = useCallback(async () => {
29
+ if (!businessId)
30
+ return;
31
+ if (runRef.current)
32
+ return;
33
+ runRef.current = true;
34
+ setChecking(true);
35
+ try {
36
+ const result = await ensureStartupReady(businessId, { timeoutMs });
37
+ setReadiness(result);
38
+ }
39
+ finally {
40
+ runRef.current = false;
41
+ setChecking(false);
42
+ }
43
+ }, [businessId, timeoutMs]);
44
+ // First check when a business is available.
45
+ useEffect(() => {
46
+ if (!enabled || !businessId)
47
+ return;
48
+ if (readiness)
49
+ return;
50
+ void check();
51
+ }, [enabled, businessId, readiness, check]);
52
+ // Re-check whenever cloud reachability flips to reachable (data may now sync).
53
+ useEffect(() => {
54
+ if (!enabled || !businessId)
55
+ return;
56
+ if (cloudStatus !== "reachable")
57
+ return;
58
+ void check();
59
+ }, [enabled, businessId, cloudStatus, check]);
60
+ return { businessId, readiness, checking, recheck: check };
61
+ }
package/dist/index.d.ts CHANGED
@@ -29,6 +29,7 @@ export * from "./utils/customer-display";
29
29
  export * from "./utils/order-filter";
30
30
  export { useReferenceData } from "./reference/hooks";
31
31
  export { setNotifyHandlers } from "./adapters/notify";
32
+ export { setSyncFailureReporter, reportSyncFailure, type SyncFailureReport, type SyncFailureReporter, type SyncFailurePhase, } from "./adapters/sync-failure";
32
33
  export { walletService, riderService, fleetService, deliveryService, dashboardService, analyticsService, placesService } from "./adapters/services";
33
34
  export { buildMenuItemEnvelope, buildMenuCategoryEnvelope } from "./reference/menu-record";
34
35
  export { validateMenuItemInput, validateMenuCategoryInput, validateOrderCreateInput, validateInviteInput } from "./validation/offline";
@@ -54,6 +55,8 @@ export { buildCommand } from "./sync/command-builder";
54
55
  export { generateLocalOrderNumber, generateLocalCalloutNumber, isLocalOrderNumber, generateOrderNumber, generateOrderNumberForBusiness, isOfflineOrderNumber, parseOrderNumber, resetCalloutForTests, } from "./sync/order-number";
55
56
  export { deviceLeaseService, getDeviceNumber, isOnlineOnly, ensureLease } from "./sync/device-lease.service";
56
57
  export { useDeviceLease, ensureDeviceLease } from "./sync/use-device-lease";
58
+ export { getStartupReadiness, ensureStartupReady, REQUIRED_ORDER_KEYS, type StartupReadiness, } from "./sync/startup-readiness";
59
+ export { useStartupReadiness, type UseStartupReadinessOptions, type UseStartupReadinessResult, } from "./hooks/startup-readiness.hooks";
57
60
  export * from "./utils/hlc";
58
61
  export * from "./reference/lww";
59
62
  export { orderSnapshotsService } from "./sync/order-snapshots.service";
package/dist/index.js CHANGED
@@ -30,6 +30,7 @@ export * from "./utils/customer-display";
30
30
  export * from "./utils/order-filter";
31
31
  export { useReferenceData } from "./reference/hooks";
32
32
  export { setNotifyHandlers } from "./adapters/notify";
33
+ export { setSyncFailureReporter, reportSyncFailure, } from "./adapters/sync-failure";
33
34
  export { walletService, riderService, fleetService, deliveryService, dashboardService, analyticsService, placesService } from "./adapters/services";
34
35
  export { buildMenuItemEnvelope, buildMenuCategoryEnvelope } from "./reference/menu-record";
35
36
  export { validateMenuItemInput, validateMenuCategoryInput, validateOrderCreateInput, validateInviteInput } from "./validation/offline";
@@ -57,6 +58,8 @@ export { buildCommand } from "./sync/command-builder";
57
58
  export { generateLocalOrderNumber, generateLocalCalloutNumber, isLocalOrderNumber, generateOrderNumber, generateOrderNumberForBusiness, isOfflineOrderNumber, parseOrderNumber, resetCalloutForTests, } from "./sync/order-number";
58
59
  export { deviceLeaseService, getDeviceNumber, isOnlineOnly, ensureLease } from "./sync/device-lease.service";
59
60
  export { useDeviceLease, ensureDeviceLease } from "./sync/use-device-lease";
61
+ export { getStartupReadiness, ensureStartupReady, REQUIRED_ORDER_KEYS, } from "./sync/startup-readiness";
62
+ export { useStartupReadiness, } from "./hooks/startup-readiness.hooks";
60
63
  export * from "./utils/hlc";
61
64
  export * from "./reference/lww";
62
65
  export { orderSnapshotsService } from "./sync/order-snapshots.service";
@@ -24,6 +24,8 @@ declare class CommandQueueService {
24
24
  * Try to execute a single pending command through the current transport.
25
25
  */
26
26
  processCommand(command: PersistedCommand): Promise<void>;
27
+ /** Forward a failed attempt to the host's observability reporter. */
28
+ private reportFailure;
27
29
  /**
28
30
  * Mark a command permanently failed: loud toast + optimistic revert, then
29
31
  * replay continues to the next command. Shared by the exhausted-retry and
@@ -39,8 +41,13 @@ declare class CommandQueueService {
39
41
  */
40
42
  private absorbIfApplied;
41
43
  /**
42
- * Replay all pending commands in order. Stops on the first failure to keep
43
- * ordering consistent; the next replay will retry from the failed command.
44
+ * Replay pending commands in FIFO order with entity-aware skip-and-continue.
45
+ *
46
+ * Ordering is preserved per entity: a failing command defers only its own
47
+ * entity_id, so that order's later commands stay queued behind it, while
48
+ * independent entities keep syncing in the same pass. The pass stops only
49
+ * when nothing can make progress: no transport at all (REPLAY_NO_TRANSPORT)
50
+ * or TRANSPORT_FAILURE_BREAKER consecutive network/5xx failures.
44
51
  */
45
52
  replay(): Promise<void>;
46
53
  /**
@@ -12,7 +12,7 @@ import { getSyncDatabase, openSyncDatabase } from "../db/connection";
12
12
  import { insertCommand, getPendingCommands, markCommandInFlight, updateCommandStatus, deleteCommand, insertEvent, } from "../db/operations";
13
13
  import { syncLogger } from "../utils/sync-logger";
14
14
  import { recordAttempt } from "../utils/attempt-stats";
15
- import { TransitionError, isReplayDeferred, TRANSITION_ERROR_CODES } from "./transition-errors";
15
+ import { TransitionError, isReplayDeferred, isReplayNoTransport, TRANSITION_ERROR_CODES } from "./transition-errors";
16
16
  import { assertBusinessOpenForCreate } from "../utils/business-hours";
17
17
  import { extractApiErrorCode } from "../utils/error";
18
18
  import { generateEventId } from "../utils/uuid";
@@ -21,16 +21,18 @@ import { generateLocalCalloutNumber, generateOrderNumberForBusiness, generateOrd
21
21
  import { getConnectivityState, subscribeConnectivity } from "../adapters/connectivity";
22
22
  import { canBusinessSync } from "./sync-guard";
23
23
  import { backoffDelayMs } from "./backoff";
24
- const MAX_RETRIES = 5;
24
+ import { reportSyncFailure } from "../adapters/sync-failure";
25
+ const MAX_ATTEMPTS = 8;
25
26
  /**
26
- * 4xx that deserve another chance: transient races and rate limits, never
27
- * permanent verdicts. Anything else 4xx (400 validation, 401 session,
28
- * 402 plan, 403 forbidden) fails fast with a toast. 409 ORDER_NUMBER_TAKEN
29
- * has its own regen path below and is not capped (every attempt mints a
30
- * new order number, so it is never a blind retry).
27
+ * Stop a replay pass after this many consecutive transport-level failures
28
+ * (network/timeout/5xx) so a dead server is not hammered once per queued
29
+ * command. Any server response (2xx, permanent 4xx) or deferral resets the
30
+ * counter. The next trigger (connectivity/backoff) starts a fresh pass.
31
+ */
32
+ const TRANSPORT_FAILURE_BREAKER = 3;
33
+ /**
34
+ * In-memory next-eligible time per command (shared 2s→5min curve).
31
35
  */
32
- const RETRYABLE_4XX = new Set([404, 408, 409, 422, 429]);
33
- /** In-memory next-eligible time per command (shared 2s→5min curve). */
34
36
  const retryNotBefore = new Map();
35
37
  const QUEUED_STATUS_BY_COMMAND = {
36
38
  "order:accept": "accepted",
@@ -260,7 +262,7 @@ class CommandQueueService {
260
262
  timestamp: command.timestamp,
261
263
  payload: command.payload,
262
264
  };
263
- if (command.retry_count >= MAX_RETRIES) {
265
+ if (command.retry_count >= MAX_ATTEMPTS) {
264
266
  await updateCommandStatus(db, command.command_id, "failed", "Max retry count exceeded");
265
267
  return;
266
268
  }
@@ -294,10 +296,16 @@ class CommandQueueService {
294
296
  // the next trigger (do not mark, do not count). Must reset the
295
297
  // in_flight flag first — rows stuck in_flight are invisible to the
296
298
  // outbox query and would wedge the queue forever.
297
- if (isReplayDeferred(error)) {
299
+ if (isReplayNoTransport(error)) {
298
300
  await updateCommandStatus(db, command.command_id, "pending", "deferred:no-transport");
299
301
  throw error;
300
302
  }
303
+ // Per-command backoff (retry window or retryable-4xx backoff): leave the
304
+ // row pending and rethrow so the pass defers just this entity.
305
+ if (isReplayDeferred(error)) {
306
+ await updateCommandStatus(db, command.command_id, "pending", "deferred:backoff");
307
+ throw error;
308
+ }
301
309
  const apiErr = error;
302
310
  const message = error instanceof Error ? error.message : String(error);
303
311
  // 409 ORDER_NUMBER_TAKEN is retryable once with new RRR (exact backend code, any envelope)
@@ -323,6 +331,13 @@ class CommandQueueService {
323
331
  catch { }
324
332
  }
325
333
  await updateCommandStatus(db, command.command_id, "failed", `order_number_taken:${message}`);
334
+ this.reportFailure(command, {
335
+ status: 409,
336
+ code: "ORDER_NUMBER_TAKEN",
337
+ message,
338
+ attempts: command.retry_count + 1,
339
+ terminal: true,
340
+ });
326
341
  throw error;
327
342
  }
328
343
  const status = typeof apiErr?.status === "number" ? apiErr.status : undefined;
@@ -339,19 +354,28 @@ class CommandQueueService {
339
354
  }
340
355
  catch { }
341
356
  }
342
- // Retryable 4xx (FIFO races, rate limits): back off on the shared
343
- // 2s→5min curve, then fail loudly after MAX_RETRIES. Throw a
344
- // transport-neutral deferral so replay STOPS here (FIFO ordering) —
345
- // rethrowing the 4xx itself would make the loop `continue` past it.
346
- if (status !== undefined && RETRYABLE_4XX.has(status)) {
357
+ // Every other failure is retryable on the shared 2s→5min backoff.
358
+ //
359
+ // 4xx (validation/permission/plan races) is bounded: a device stuck in
360
+ // a bad local state keeps retrying, then dead-letters so an operator is
361
+ // alerted and can visit the device. 5xx/network is an outage, not a
362
+ // device problem — it keeps retrying indefinitely so a long outage can
363
+ // never lose good orders. Every attempt is reported to observability.
364
+ const is4xxResponse = status !== undefined && status >= 400 && status < 500;
365
+ if (is4xxResponse) {
347
366
  const attempts = command.retry_count + 1;
348
- if (attempts >= MAX_RETRIES) {
349
- await this.failPermanently(db, command, message, `retryable 4xx exhausted after ${attempts} attempts`);
367
+ if (attempts >= MAX_ATTEMPTS) {
368
+ await this.failPermanently(db, command, message, `4xx exhausted after ${attempts} attempts`, {
369
+ status,
370
+ code,
371
+ attempts,
372
+ });
350
373
  return;
351
374
  }
375
+ this.reportFailure(command, { status, code, message, attempts, terminal: false });
352
376
  retryNotBefore.set(command.command_id, Date.now() + backoffDelayMs(command.retry_count));
353
377
  await updateCommandStatus(db, command.command_id, "pending", `retry_4xx:${message}`, true);
354
- syncLogger.warn("CommandQueueService", "Retryable 4xx — backing off", {
378
+ syncLogger.warn("CommandQueueService", "4xx — backing off", {
355
379
  command_id: command.command_id,
356
380
  command_type: command.command_type,
357
381
  status,
@@ -359,30 +383,37 @@ class CommandQueueService {
359
383
  });
360
384
  throw new TransitionError(TRANSITION_ERROR_CODES.REPLAY_DEFERRED, `Retryable ${status} on ${command.command_type}, backing off`);
361
385
  }
362
- const isNonRetryable = status !== undefined &&
363
- status >= 400 &&
364
- status < 500;
365
- if (isNonRetryable) {
366
- await this.failPermanently(db, command, message, "permanent 4xx");
367
- // Do not throw — allow replay to continue to next command.
368
- return;
369
- }
370
- // Network/timeout/5xx: NOT a rejection — keep the command retryable
371
- // indefinitely (status back to pending, retry_count untouched) so a
372
- // long outage never dead-letters the outbox via MAX_RETRIES. Replay
373
- // stops here to preserve ordering; the next trigger retries.
386
+ // 5xx / network / timeout: keep the command retryable indefinitely
387
+ // (status back to pending, retry_count untouched) so a long outage never
388
+ // dead-letters the outbox. Replay defers this entity; the next trigger
389
+ // retries.
390
+ this.reportFailure(command, { status, code, message, attempts: command.retry_count, terminal: false });
374
391
  await updateCommandStatus(db, command.command_id, "pending", `network:${message}`);
375
392
  syncLogger.command(parsedCommand, "queued", { error: message, deferred: true });
376
393
  throw error;
377
394
  }
378
395
  }
396
+ /** Forward a failed attempt to the host's observability reporter. */
397
+ reportFailure(command, extra) {
398
+ reportSyncFailure({
399
+ phase: "outbox",
400
+ business_id: command.business_id,
401
+ user_id: command.user_id,
402
+ device_id: command.device_id,
403
+ command_type: command.command_type,
404
+ command_id: command.command_id,
405
+ entity_id: command.entity_id,
406
+ ...extra,
407
+ });
408
+ }
379
409
  /**
380
410
  * Mark a command permanently failed: loud toast + optimistic revert, then
381
411
  * replay continues to the next command. Shared by the exhausted-retry and
382
412
  * permanent-rejection paths.
383
413
  */
384
- async failPermanently(db, command, message, reason) {
414
+ async failPermanently(db, command, message, reason, extra) {
385
415
  await updateCommandStatus(db, command.command_id, "failed", `permanent:${message}`);
416
+ this.reportFailure(command, { ...extra, message, terminal: true });
386
417
  syncLogger.warn("CommandQueueService", `Queued command failed (${reason})`, {
387
418
  command_id: command.command_id,
388
419
  command_type: command.command_type,
@@ -437,8 +468,13 @@ class CommandQueueService {
437
468
  return false;
438
469
  }
439
470
  /**
440
- * Replay all pending commands in order. Stops on the first failure to keep
441
- * ordering consistent; the next replay will retry from the failed command.
471
+ * Replay pending commands in FIFO order with entity-aware skip-and-continue.
472
+ *
473
+ * Ordering is preserved per entity: a failing command defers only its own
474
+ * entity_id, so that order's later commands stay queued behind it, while
475
+ * independent entities keep syncing in the same pass. The pass stops only
476
+ * when nothing can make progress: no transport at all (REPLAY_NO_TRANSPORT)
477
+ * or TRANSPORT_FAILURE_BREAKER consecutive network/5xx failures.
442
478
  */
443
479
  async replay() {
444
480
  if (!canBusinessSync())
@@ -469,21 +505,59 @@ class CommandQueueService {
469
505
  syncLogger.info("CommandQueueService", "Replaying commands", {
470
506
  count: pending.length,
471
507
  });
508
+ // Entities deferred in THIS pass (backoff window or a transport failure):
509
+ // later commands for the same entity are skipped to preserve per-entity
510
+ // FIFO without blocking independent commands.
511
+ const deferredEntities = new Set();
512
+ let consecutiveTransportFailures = 0;
472
513
  for (const command of pending) {
514
+ const entity = command.entity_id;
515
+ if (entity && deferredEntities.has(entity)) {
516
+ syncLogger.debug("CommandQueueService", "Skipping command for deferred entity", {
517
+ command_id: command.command_id,
518
+ entity_id: entity,
519
+ });
520
+ continue;
521
+ }
473
522
  try {
474
523
  await this.processCommand(command);
524
+ consecutiveTransportFailures = 0;
475
525
  }
476
526
  catch (e) {
477
- const replayErr = e;
478
- const isNonRetryable = typeof replayErr?.status === "number" &&
479
- replayErr.status >= 400 &&
480
- replayErr.status < 500;
481
- if (isNonRetryable) {
482
- // Already marked permanent in processCommand — continue to next.
527
+ // No transport at all: nothing in this pass can succeed — stop.
528
+ if (isReplayNoTransport(e)) {
529
+ syncLogger.info("CommandQueueService", "Replay stopped: no transport available", {
530
+ command_id: command.command_id,
531
+ });
532
+ break;
533
+ }
534
+ // Per-command deferral (backoff window / retryable-4xx backoff):
535
+ // defer this entity only and keep draining the rest.
536
+ if (isReplayDeferred(e)) {
537
+ if (entity)
538
+ deferredEntities.add(entity);
539
+ consecutiveTransportFailures = 0;
483
540
  continue;
484
541
  }
485
- // Network/5xx — stop to preserve ordering; retry later.
486
- break;
542
+ const status = e?.status;
543
+ const is4xx = typeof status === "number" && status >= 400 && status < 500;
544
+ if (is4xx) {
545
+ // Permanent/retryable-exhausted 4xx: processCommand already marked
546
+ // the row failed (or absorbed it) — continue to the next command.
547
+ consecutiveTransportFailures = 0;
548
+ continue;
549
+ }
550
+ // Network/timeout/5xx: defer this entity, continue, but trip the
551
+ // breaker if the transport looks dead so we stop hammering.
552
+ if (entity)
553
+ deferredEntities.add(entity);
554
+ consecutiveTransportFailures += 1;
555
+ if (consecutiveTransportFailures >= TRANSPORT_FAILURE_BREAKER) {
556
+ syncLogger.warn("CommandQueueService", "Replay stopped: transport failure breaker tripped", {
557
+ consecutiveTransportFailures,
558
+ });
559
+ break;
560
+ }
487
561
  }
488
562
  }
489
563
  }
@@ -520,7 +594,7 @@ class CommandQueueService {
520
594
  async debugDumpCommands(businessId) {
521
595
  try {
522
596
  const db = getSyncDatabase();
523
- const rows = await getPendingCommands(db, Number.MAX_SAFE_INTEGER);
597
+ const rows = await getPendingCommands(db, Number.MAX_SAFE_INTEGER, true);
524
598
  return rows
525
599
  .filter((c) => !businessId || c.business_id === businessId)
526
600
  .map((c) => ({
@@ -151,9 +151,11 @@ export class SyncEngine {
151
151
  if (attempt === "queue") {
152
152
  // Replays must never re-enqueue-then-delete (the command would eat
153
153
  // itself without executing). If no real transport is available, leave
154
- // the row untouched for a later trigger.
154
+ // the row untouched for a later trigger. Distinct code so the outbox
155
+ // stops the whole replay pass (nothing can succeed) rather than
156
+ // skipping just this entity and hammering the rest.
155
157
  if (opts?.forReplay) {
156
- throw new TransitionError(TRANSITION_ERROR_CODES.REPLAY_DEFERRED, "No transport available, keeping command queued");
158
+ throw new TransitionError(TRANSITION_ERROR_CODES.REPLAY_NO_TRANSPORT, "No transport available, keeping command queued");
157
159
  }
158
160
  const event = await commandQueueService.enqueue(command);
159
161
  if (queueDespiteEntitlement) {
@@ -1,6 +1,14 @@
1
1
  /**
2
- * Full sync: push → pull → push.
3
- * Push first so local edits (already HLC-bumped) reach server before pull
4
- * overwrites them; then pull merges; final drain pushes any remainder.
2
+ * Full sync: reference → push → pull → push.
3
+ *
4
+ * Reference (menu/hours/tax/capabilities) is pulled FIRST and non-blockingly:
5
+ * every phase is independently swallowed, so a reference failure never
6
+ * prevents queued orders from flushing. Getting reference in before the order
7
+ * push means newly-launched terminals can satisfy order-creation prerequisites
8
+ * (hour/tax/capability guards) in the same cycle.
9
+ *
10
+ * Orders still push before the order pull, so local edits (already HLC-bumped)
11
+ * reach the server before the pull merge could overwrite them. The final
12
+ * drain flushes anything newly queued or retried after the pull.
5
13
  */
6
14
  export declare function performFullSync(businessId?: string): Promise<void>;
@@ -1,7 +1,15 @@
1
1
  /**
2
- * Full sync: push → pull → push.
3
- * Push first so local edits (already HLC-bumped) reach server before pull
4
- * overwrites them; then pull merges; final drain pushes any remainder.
2
+ * Full sync: reference → push → pull → push.
3
+ *
4
+ * Reference (menu/hours/tax/capabilities) is pulled FIRST and non-blockingly:
5
+ * every phase is independently swallowed, so a reference failure never
6
+ * prevents queued orders from flushing. Getting reference in before the order
7
+ * push means newly-launched terminals can satisfy order-creation prerequisites
8
+ * (hour/tax/capability guards) in the same cycle.
9
+ *
10
+ * Orders still push before the order pull, so local edits (already HLC-bumped)
11
+ * reach the server before the pull merge could overwrite them. The final
12
+ * drain flushes anything newly queued or retried after the pull.
5
13
  */
6
14
  import { getConnectivityState } from "../adapters/connectivity";
7
15
  import { getActiveBusiness } from "../adapters/business";
@@ -26,28 +34,30 @@ export async function performFullSync(businessId) {
26
34
  }
27
35
  const p = (async () => {
28
36
  try {
29
- syncLogger.info("FullSync", "Starting push→pull→push", { businessId: bid });
30
- // 1. Push pending FIFO first — HLC-bumped local edits outrank stale server
37
+ syncLogger.info("FullSync", "Starting reference→push→pull→push", { businessId: bid });
38
+ // 1. Reference first (non-blocking): menu/hours/tax/caps so order
39
+ // creation prerequisites are present before we flush orders.
31
40
  try {
32
- await drainPending(bid);
41
+ await referenceSyncService.sync({ businessId: bid });
33
42
  }
34
43
  catch (e) {
35
- syncLogger.warn("FullSync", "drainPending(pre) failed", { err: String(e) });
44
+ syncLogger.warn("FullSync", "reference sync failed", { err: String(e) });
36
45
  }
37
- // 2. Pull remote orders (HLC-guarded) and reference (LWW merge)
46
+ // 2. Push pending FIFO — HLC-bumped local edits outrank stale server
38
47
  try {
39
- await pullRemoteOrders(bid);
48
+ await drainPending(bid);
40
49
  }
41
50
  catch (e) {
42
- syncLogger.warn("FullSync", "pullRemoteOrders failed", { err: String(e) });
51
+ syncLogger.warn("FullSync", "drainPending(pre) failed", { err: String(e) });
43
52
  }
53
+ // 3. Pull remote orders (HLC-guarded)
44
54
  try {
45
- await referenceSyncService.sync({ businessId: bid });
55
+ await pullRemoteOrders(bid);
46
56
  }
47
57
  catch (e) {
48
- syncLogger.warn("FullSync", "reference sync failed", { err: String(e) });
58
+ syncLogger.warn("FullSync", "pullRemoteOrders failed", { err: String(e) });
49
59
  }
50
- // 3. Final drain — anything newly queued or retried after pull
60
+ // 4. Final drain — anything newly queued or retried after pull
51
61
  try {
52
62
  await drainPending(bid);
53
63
  }
@@ -0,0 +1,44 @@
1
+ /**
2
+ * Startup readiness for the POS/order surfaces.
3
+ *
4
+ * The offline-first guards intentionally fail closed (HOURS_NOT_SYNCED,
5
+ * SUBSCRIPTION_REQUIRED, "no device leased"), which is correct — but if the
6
+ * required reference data was never loaded, those guards fire at order-tap
7
+ * time as a surprise. This module lets the app discover the same conditions at
8
+ * launch: it evaluates the local cache (no network) and, when online, performs
9
+ * one bounded catch-up sync so a freshly-launched terminal can satisfy the
10
+ * guards before the seller starts taking orders.
11
+ *
12
+ * Seller-only. Fleet/rider surfaces are fully online and never gate here.
13
+ */
14
+ import type { ReferenceDataKey } from "../reference/types";
15
+ /**
16
+ * Reference keys required before the app presents itself as ready to take
17
+ * orders: menu to sell, taxes to price, hours to open, profile/capabilities
18
+ * for routing + guards, and the signed entitlement proof for gated commands.
19
+ */
20
+ export declare const REQUIRED_ORDER_KEYS: readonly ReferenceDataKey[];
21
+ export interface StartupReadiness {
22
+ /** True when every required key is cached and a device number is leased. */
23
+ ready: boolean;
24
+ /** Required keys with no cached payload. */
25
+ missingKeys: ReferenceDataKey[];
26
+ /** True when no DD-RRR-CCC device number is available (needs one online lease). */
27
+ needsLease: boolean;
28
+ /**
29
+ * True when an entitlement envelope is absent or does not evaluate as
30
+ * entitled. Informational: cash orders still queue offline, so this does not
31
+ * by itself block readiness.
32
+ */
33
+ entitlementUnverified: boolean;
34
+ }
35
+ /** Evaluate the local cache only (no network). */
36
+ export declare function getStartupReadiness(businessId: string): Promise<StartupReadiness>;
37
+ /**
38
+ * One bounded catch-up attempt, then evaluate the cache. When offline (or the
39
+ * cloud is unreachable) this performs no network work and simply reports what
40
+ * is cached. Never throws.
41
+ */
42
+ export declare function ensureStartupReady(businessId: string, opts?: {
43
+ timeoutMs?: number;
44
+ }): Promise<StartupReadiness>;
@@ -0,0 +1,115 @@
1
+ /**
2
+ * Startup readiness for the POS/order surfaces.
3
+ *
4
+ * The offline-first guards intentionally fail closed (HOURS_NOT_SYNCED,
5
+ * SUBSCRIPTION_REQUIRED, "no device leased"), which is correct — but if the
6
+ * required reference data was never loaded, those guards fire at order-tap
7
+ * time as a surprise. This module lets the app discover the same conditions at
8
+ * launch: it evaluates the local cache (no network) and, when online, performs
9
+ * one bounded catch-up sync so a freshly-launched terminal can satisfy the
10
+ * guards before the seller starts taking orders.
11
+ *
12
+ * Seller-only. Fleet/rider surfaces are fully online and never gate here.
13
+ */
14
+ import { openSyncDatabase } from "../db/connection";
15
+ import { getReferenceData } from "../reference/operations";
16
+ import { referenceSyncService } from "../reference/service";
17
+ import { evaluateBusinessEntitlements } from "../reference/entitlement-guard";
18
+ import { ensureLease } from "./device-lease.service";
19
+ import { getConnectivityState } from "../adapters/connectivity";
20
+ import { syncLogger } from "../utils/sync-logger";
21
+ /**
22
+ * Reference keys required before the app presents itself as ready to take
23
+ * orders: menu to sell, taxes to price, hours to open, profile/capabilities
24
+ * for routing + guards, and the signed entitlement proof for gated commands.
25
+ */
26
+ export const REQUIRED_ORDER_KEYS = [
27
+ "subscription:entitlements",
28
+ "capabilities",
29
+ "menu:categories",
30
+ "menu:items",
31
+ "tax-rules",
32
+ "business:profile",
33
+ "operating-hours",
34
+ ];
35
+ /** Evaluate the local cache only (no network). */
36
+ export async function getStartupReadiness(businessId) {
37
+ const db = await openSyncDatabase();
38
+ const records = await Promise.all(REQUIRED_ORDER_KEYS.map(async (key) => ({
39
+ key,
40
+ rec: await getReferenceData(db, businessId, key),
41
+ })));
42
+ const missingKeys = records
43
+ .filter((r) => !r.rec || r.rec.payload == null)
44
+ .map((r) => r.key);
45
+ // ensureLease() hydrates the sync KV from async KV and leases online when
46
+ // possible; offline with no cache it returns null (caller blocks POS).
47
+ let deviceNumber = null;
48
+ try {
49
+ deviceNumber = await ensureLease(businessId);
50
+ }
51
+ catch (e) {
52
+ syncLogger.warn("StartupReadiness", "lease check failed", { err: String(e) });
53
+ }
54
+ const needsLease = !deviceNumber;
55
+ let entitlementUnverified = missingKeys.includes("subscription:entitlements");
56
+ if (!entitlementUnverified) {
57
+ try {
58
+ const { evaluation } = await evaluateBusinessEntitlements(businessId);
59
+ entitlementUnverified = !evaluation.entitled;
60
+ }
61
+ catch {
62
+ entitlementUnverified = true;
63
+ }
64
+ }
65
+ return {
66
+ ready: missingKeys.length === 0 && !needsLease,
67
+ missingKeys,
68
+ needsLease,
69
+ entitlementUnverified,
70
+ };
71
+ }
72
+ /**
73
+ * One bounded catch-up attempt, then evaluate the cache. When offline (or the
74
+ * cloud is unreachable) this performs no network work and simply reports what
75
+ * is cached. Never throws.
76
+ */
77
+ export async function ensureStartupReady(businessId, opts) {
78
+ const timeoutMs = opts?.timeoutMs ?? 15000;
79
+ const { isOnline, isCloudReachable } = getConnectivityState();
80
+ if (isOnline && isCloudReachable !== "unreachable") {
81
+ const catchUp = (async () => {
82
+ try {
83
+ await referenceSyncService.sync({ businessId, keys: [...REQUIRED_ORDER_KEYS] });
84
+ }
85
+ catch (e) {
86
+ syncLogger.warn("StartupReadiness", "reference catch-up failed", { err: String(e) });
87
+ }
88
+ try {
89
+ await ensureLease(businessId);
90
+ }
91
+ catch (e) {
92
+ syncLogger.warn("StartupReadiness", "lease catch-up failed", { err: String(e) });
93
+ }
94
+ })();
95
+ await raceWithTimeout(catchUp, timeoutMs);
96
+ }
97
+ return getStartupReadiness(businessId);
98
+ }
99
+ /** Resolve after either the promise settles or the timeout elapses. */
100
+ async function raceWithTimeout(promise, ms) {
101
+ let timer;
102
+ const timeout = new Promise((resolve) => {
103
+ timer = setTimeout(resolve, ms);
104
+ });
105
+ try {
106
+ await Promise.race([
107
+ promise.then(() => undefined, () => undefined),
108
+ timeout,
109
+ ]);
110
+ }
111
+ finally {
112
+ if (timer)
113
+ clearTimeout(timer);
114
+ }
115
+ }
@@ -19,11 +19,13 @@ import { getTransitionPath, DELIVERY_COMMANDS } from "./transition-endpoints";
19
19
  import { getQueuedStatus } from "../utils/order-transitions";
20
20
  import { extractApiErrorCode } from "../utils/error";
21
21
  import { canBusinessSync, isAuthenticated } from "./sync-guard";
22
+ import { reportSyncFailure } from "../adapters/sync-failure";
23
+ import { getAppInstanceId } from "../adapters/devices";
22
24
  const inflight = new Map();
23
25
  const retryTimers = new Map();
24
26
  const backoffTracker = new BackoffTracker();
25
- /** Blind 4xx retries stop here, loudly. 5xx/network retry indefinitely. */
26
- const MAX_SYNC_JOB_ATTEMPTS = 5;
27
+ /** Bounded 4xx retries stop here, loudly. 5xx/network retry indefinitely. */
28
+ const MAX_SYNC_JOB_ATTEMPTS = 8;
27
29
  function isOnline() {
28
30
  const s = getConnectivityState();
29
31
  return s.isOnline && s.isCloudReachable !== "unreachable";
@@ -65,6 +67,36 @@ function clearRetry(businessId) {
65
67
  }
66
68
  backoffTracker.reset(businessId);
67
69
  }
70
+ /** Best-effort device id for failure reports (never throws). */
71
+ function safeDeviceId() {
72
+ try {
73
+ return getAppInstanceId();
74
+ }
75
+ catch {
76
+ return undefined;
77
+ }
78
+ }
79
+ /** Forward a failed job attempt to the host's observability reporter. */
80
+ function reportJobFailure(businessId, row, extra) {
81
+ let userId;
82
+ try {
83
+ const p = JSON.parse(row.payload);
84
+ const candidate = p.user_id ?? p.user?.id;
85
+ if (typeof candidate === "string")
86
+ userId = candidate;
87
+ }
88
+ catch { }
89
+ reportSyncFailure({
90
+ phase: "sync_jobs",
91
+ business_id: businessId,
92
+ user_id: userId,
93
+ device_id: safeDeviceId(),
94
+ command_type: row.command_type,
95
+ command_id: row.id,
96
+ entity_id: row.entity_id,
97
+ ...extra,
98
+ });
99
+ }
68
100
  async function tryPhxPush(businessId, payload) {
69
101
  try {
70
102
  const socket = phoenixSocket.getChannel?.(`business:${businessId}`);
@@ -385,38 +417,13 @@ export async function drainOne(businessId, entityId) {
385
417
  syncLogger.debug("SyncJobs", "401 deferred until login", { businessId });
386
418
  return;
387
419
  }
388
- // Permanent 4xx: mark failed to stop the retry loop.
389
- // Retryable (bounded, 5 attempts): 401 (refresh owns it below), 408,
390
- // 409, 422, 429, any 404 (FIFO race: earlier command still in flight),
391
- // BUSINESS_CLOSED (pending-by-design). Everything else 4xx (400
392
- // validation, 402 plan, 403 forbidden) fails fast. Backend codes are
420
+ // Any 4xx is retryable with backoff (bounded); 5xx/network keeps retrying
421
+ // indefinitely. Every failed attempt is reported to observability so an
422
+ // operator can see the device/app/user and visit it. BUSINESS_CLOSED is a
423
+ // legitimate closed state (keep pending, re-pull hours). Backend codes are
393
424
  // exact (no message matching).
394
425
  const isBusinessClosed = rest.code === "BUSINESS_CLOSED" || rest.code === "HOURS_NOT_SYNCED";
395
- const is4xx = rest.status && rest.status >= 400 && rest.status < 500;
396
- const isRetryable4xx = rest.status === 401 || rest.status === 408 || rest.status === 409
397
- || rest.status === 422 || rest.status === 429 || rest.status === 404
398
- || isBusinessClosed;
399
- if (is4xx && !isRetryable4xx) {
400
- syncLogger.warn("SyncJobs", "Permanent 4xx — marking job failed to stop retry loop", { businessId, entityId, status: rest.status, code: rest.code, message: rest.message });
401
- try {
402
- await db.runAsync(`UPDATE sync_jobs SET status='failed', last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
403
- }
404
- catch { }
405
- clearRetry(businessId);
406
- try {
407
- const { notifyError } = await import("../adapters/notify");
408
- notifyError(`Server rejected ${row.command_type}: ${rest.message ?? rest.code ?? rest.status}`);
409
- }
410
- catch { }
411
- // Retain optimistic row — pull will converge for transitions, create stays for manual retry.
412
- if (row.command_type !== "order:create") {
413
- try {
414
- void pullRemoteOrders(businessId).catch(() => { });
415
- }
416
- catch { }
417
- }
418
- return;
419
- }
426
+ const is4xx = !!rest.status && rest.status >= 400 && rest.status < 500;
420
427
  // Already-applied absorb for transitions: the server already moved the
421
428
  // order (duplicate tap, earlier drain won the race) — complete the job
422
429
  // instead of failing it. Server truth only (hlc-guarded pull), never the
@@ -432,39 +439,43 @@ export async function drainOne(businessId, entityId) {
432
439
  }
433
440
  catch { }
434
441
  }
435
- // Bounded retries: blind 4xx retries stop after 5 attempts with a loud
436
- // toast (attempts only increments on the keep-pending path below).
437
- if (is4xx && isRetryable4xx && (row.attempts ?? 0) + 1 >= 5) {
438
- syncLogger.warn("SyncJobs", "Retryable 4xx exhausted — marking job failed", { businessId, entityId, status: rest.status, code: rest.code, attempts: row.attempts });
442
+ if (isBusinessClosed) {
443
+ syncLogger.warn("SyncJobs", "Business closed — keeping pending for retry after hours sync", { businessId, entityId, code: rest.code });
444
+ reportJobFailure(businessId, row, { status: rest.status, code: rest.code, message: rest.message, attempts: row.attempts, terminal: false });
439
445
  try {
440
- await db.runAsync(`UPDATE sync_jobs SET status='failed', last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""} (after ${MAX_SYNC_JOB_ATTEMPTS} attempts)`.trim(), new Date().toISOString(), row.id);
446
+ await db.runAsync(`UPDATE sync_jobs SET last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
441
447
  }
442
448
  catch { }
443
- clearRetry(businessId);
449
+ // Re-pull reference so hours merge converges, then retry with backoff.
444
450
  try {
445
- const { notifyError } = await import("../adapters/notify");
446
- notifyError(`Server rejected ${row.command_type}: ${rest.message ?? rest.code ?? rest.status} (after retries)`);
451
+ const { referenceSyncService } = await import("../reference/service");
452
+ void referenceSyncService.sync({ businessId, keys: ["operating-hours", "business:profile"] }).catch(() => { });
447
453
  }
448
454
  catch { }
455
+ scheduleRetry(businessId);
449
456
  return;
450
457
  }
451
- if (isBusinessClosed) {
452
- syncLogger.warn("SyncJobs", "Business closed — keeping pending for retry after hours sync", { businessId, entityId, code: rest.code });
458
+ const attempts = (row.attempts ?? 0) + 1;
459
+ // Bounded 4xx: after MAX_SYNC_JOB_ATTEMPTS, dead-letter (retained) with a
460
+ // loud toast + observability report so an operator can visit the device.
461
+ if (is4xx && attempts >= MAX_SYNC_JOB_ATTEMPTS) {
462
+ syncLogger.warn("SyncJobs", "4xx exhausted — marking job failed", { businessId, entityId, status: rest.status, code: rest.code, attempts: row.attempts });
453
463
  try {
454
- await db.runAsync(`UPDATE sync_jobs SET last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
464
+ await db.runAsync(`UPDATE sync_jobs SET status='failed', last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""} (after ${MAX_SYNC_JOB_ATTEMPTS} attempts)`.trim(), new Date().toISOString(), row.id);
455
465
  }
456
466
  catch { }
457
- // Re-pull reference so hours merge converges, then retry with backoff.
467
+ reportJobFailure(businessId, row, { status: rest.status, code: rest.code, message: rest.message, attempts, terminal: true });
468
+ clearRetry(businessId);
458
469
  try {
459
- const { referenceSyncService } = await import("../reference/service");
460
- void referenceSyncService.sync({ businessId, keys: ["operating-hours", "business:profile"] }).catch(() => { });
470
+ const { notifyError } = await import("../adapters/notify");
471
+ notifyError(`Server rejected ${row.command_type}: ${rest.message ?? rest.code ?? rest.status} (after retries)`);
461
472
  }
462
473
  catch { }
463
- scheduleRetry(businessId);
464
474
  return;
465
475
  }
466
- // Keep pending for retry (5xx / network) — auto retry with backoff
476
+ // 4xx (bounded) or 5xx/network (indefinite): keep pending and back off.
467
477
  syncLogger.debug("SyncJobs", "drainOne kept pending for retry", { businessId, entityId, status: rest.status, code: rest.code });
478
+ reportJobFailure(businessId, row, { status: rest.status, code: rest.code, message: rest.message, attempts, terminal: false });
468
479
  try {
469
480
  await db.runAsync(`UPDATE sync_jobs SET attempts = attempts + 1, last_error = ?, updated_at = ? WHERE id = ?;`, `${rest.status ?? "network"} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
470
481
  }
@@ -18,6 +18,13 @@ export declare const TRANSITION_ERROR_CODES: {
18
18
  readonly FORBIDDEN: "FORBIDDEN";
19
19
  readonly SUBSCRIPTION_REQUIRED: "SUBSCRIPTION_REQUIRED";
20
20
  readonly REPLAY_DEFERRED: "REPLAY_DEFERRED";
21
+ /**
22
+ * Replay found no usable transport at all (cloud + LAN both unavailable).
23
+ * Distinct from REPLAY_DEFERRED (per-command backoff/retryable-4xx) so the
24
+ * outbox can stop the whole pass instead of skipping just this entity:
25
+ * nothing can succeed until a transport returns.
26
+ */
27
+ readonly REPLAY_NO_TRANSPORT: "REPLAY_NO_TRANSPORT";
21
28
  readonly MISSING_RIDER: "MISSING_RIDER";
22
29
  readonly NO_DELIVERY: "NO_DELIVERY";
23
30
  };
@@ -30,3 +37,4 @@ export declare class TransitionError extends Error {
30
37
  export declare function errorCodeOf(error: unknown): string | undefined;
31
38
  export declare function isFinalGuardRejection(error: unknown): boolean;
32
39
  export declare function isReplayDeferred(error: unknown): boolean;
40
+ export declare function isReplayNoTransport(error: unknown): boolean;
@@ -18,6 +18,13 @@ export const TRANSITION_ERROR_CODES = {
18
18
  FORBIDDEN: "FORBIDDEN",
19
19
  SUBSCRIPTION_REQUIRED: "SUBSCRIPTION_REQUIRED",
20
20
  REPLAY_DEFERRED: "REPLAY_DEFERRED",
21
+ /**
22
+ * Replay found no usable transport at all (cloud + LAN both unavailable).
23
+ * Distinct from REPLAY_DEFERRED (per-command backoff/retryable-4xx) so the
24
+ * outbox can stop the whole pass instead of skipping just this entity:
25
+ * nothing can succeed until a transport returns.
26
+ */
27
+ REPLAY_NO_TRANSPORT: "REPLAY_NO_TRANSPORT",
21
28
  MISSING_RIDER: "MISSING_RIDER",
22
29
  NO_DELIVERY: "NO_DELIVERY",
23
30
  };
@@ -59,3 +66,6 @@ export function isFinalGuardRejection(error) {
59
66
  export function isReplayDeferred(error) {
60
67
  return errorCodeOf(error) === TRANSITION_ERROR_CODES.REPLAY_DEFERRED;
61
68
  }
69
+ export function isReplayNoTransport(error) {
70
+ return errorCodeOf(error) === TRANSITION_ERROR_CODES.REPLAY_NO_TRANSPORT;
71
+ }
@@ -95,6 +95,18 @@ export function createMemoryDriver() {
95
95
  tables.commands.push(row);
96
96
  return { lastInsertRowId: row.id, changes: 1 };
97
97
  }
98
+ // crash recovery: in_flight -> pending on open
99
+ if (s.includes("UPDATE commands SET status = 'pending'") && s.includes("WHERE status = 'in_flight'")) {
100
+ let changes = 0;
101
+ for (const row of tables.commands) {
102
+ if (row.status === "in_flight") {
103
+ row.status = "pending";
104
+ row.error = null;
105
+ changes += 1;
106
+ }
107
+ }
108
+ return { lastInsertRowId: 0, changes };
109
+ }
98
110
  // commands update status (4- or 5-placeholder variants — always take the
99
111
  // LAST arg as command_id; older code assumed 5).
100
112
  if (s.includes("UPDATE commands SET status = ?")) {
@@ -365,7 +377,12 @@ export function createMemoryDriver() {
365
377
  }
366
378
  if (s.includes("SELECT * FROM commands") && s.includes("WHERE status IN")) {
367
379
  const limit = args[args.length - 1];
368
- let rows = tables.commands.filter((r) => r.status === "pending" || r.status === "failed").sort((a, b) => (a.created_at < b.created_at ? -1 : 1));
380
+ // Mirror production: `failed` is terminal and only returned when the
381
+ // caller opts in (includeFailed -> SQL contains 'failed').
382
+ const includeFailed = s.includes("'failed'");
383
+ let rows = tables.commands
384
+ .filter((r) => r.status === "pending" || (includeFailed && r.status === "failed"))
385
+ .sort((a, b) => (a.created_at < b.created_at ? -1 : 1));
369
386
  if (limit)
370
387
  rows = rows.slice(0, limit);
371
388
  return rows;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@teincfood/core",
3
- "version": "0.9.14",
3
+ "version": "0.9.16",
4
4
  "description": "TeincFood shared offline-first core — types, sync engine, local DB, reference data, and repositories for mobile + desktop",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",