@teincfood/core 0.9.14 → 0.9.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/sync-failure.d.ts +33 -0
- package/dist/adapters/sync-failure.js +23 -0
- package/dist/db/connection.js +25 -0
- package/dist/db/operations.d.ts +1 -1
- package/dist/db/operations.js +7 -2
- package/dist/hooks/startup-readiness.hooks.d.ts +24 -0
- package/dist/hooks/startup-readiness.hooks.js +61 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +3 -0
- package/dist/sync/command-queue.service.d.ts +9 -2
- package/dist/sync/command-queue.service.js +117 -43
- package/dist/sync/engine.js +4 -2
- package/dist/sync/full-sync.d.ts +11 -3
- package/dist/sync/full-sync.js +23 -13
- package/dist/sync/startup-readiness.d.ts +44 -0
- package/dist/sync/startup-readiness.js +115 -0
- package/dist/sync/sync-jobs.service.js +59 -48
- package/dist/sync/transition-errors.d.ts +8 -0
- package/dist/sync/transition-errors.js +10 -0
- package/dist/test-helpers/memory-driver.js +18 -1
- package/package.json +1 -1
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sync-failure reporter adapter.
|
|
3
|
+
*
|
|
4
|
+
* Core never depends on the observability SDK. The host app registers a
|
|
5
|
+
* reporter at boot (built on `@teincfood/observability`) and core forwards
|
|
6
|
+
* every failed command attempt so operators can see which device/app/user is
|
|
7
|
+
* struggling and go fix it.
|
|
8
|
+
*/
|
|
9
|
+
export type SyncFailurePhase = "outbox" | "sync_jobs";
|
|
10
|
+
export interface SyncFailureReport {
|
|
11
|
+
/** Which queue produced the failure. */
|
|
12
|
+
phase: SyncFailurePhase;
|
|
13
|
+
business_id: string;
|
|
14
|
+
user_id?: string;
|
|
15
|
+
/** Stable per-install device id (matches the telemetry device). */
|
|
16
|
+
device_id?: string;
|
|
17
|
+
command_type: string;
|
|
18
|
+
command_id?: string;
|
|
19
|
+
entity_id?: string;
|
|
20
|
+
/** HTTP status when the failure came from a response. */
|
|
21
|
+
status?: number;
|
|
22
|
+
/** Backend/guard error code. */
|
|
23
|
+
code?: string;
|
|
24
|
+
message?: string;
|
|
25
|
+
attempts?: number;
|
|
26
|
+
/** True when the command was dead-lettered (no further auto-retry). */
|
|
27
|
+
terminal: boolean;
|
|
28
|
+
transport?: string;
|
|
29
|
+
}
|
|
30
|
+
export type SyncFailureReporter = (report: SyncFailureReport) => void;
|
|
31
|
+
export declare function setSyncFailureReporter(next: SyncFailureReporter | null): void;
|
|
32
|
+
/** Fire-and-forget; never throws, safe before a reporter is registered. */
|
|
33
|
+
export declare function reportSyncFailure(report: SyncFailureReport): void;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sync-failure reporter adapter.
|
|
3
|
+
*
|
|
4
|
+
* Core never depends on the observability SDK. The host app registers a
|
|
5
|
+
* reporter at boot (built on `@teincfood/observability`) and core forwards
|
|
6
|
+
* every failed command attempt so operators can see which device/app/user is
|
|
7
|
+
* struggling and go fix it.
|
|
8
|
+
*/
|
|
9
|
+
let reporter = null;
|
|
10
|
+
export function setSyncFailureReporter(next) {
|
|
11
|
+
reporter = next;
|
|
12
|
+
}
|
|
13
|
+
/** Fire-and-forget; never throws, safe before a reporter is registered. */
|
|
14
|
+
export function reportSyncFailure(report) {
|
|
15
|
+
if (!reporter)
|
|
16
|
+
return;
|
|
17
|
+
try {
|
|
18
|
+
reporter(report);
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
/* telemetry must never break sync */
|
|
22
|
+
}
|
|
23
|
+
}
|
package/dist/db/connection.js
CHANGED
|
@@ -19,6 +19,7 @@ export async function openSyncDatabase() {
|
|
|
19
19
|
const driver = getSqliteDriver();
|
|
20
20
|
initPromise = (async () => {
|
|
21
21
|
await runMigrations(driver);
|
|
22
|
+
await recoverInFlightCommands(driver);
|
|
22
23
|
dbInstance = driver;
|
|
23
24
|
syncLogger.info("SQLite", "Database opened and migrated", {
|
|
24
25
|
name: DB_NAME,
|
|
@@ -71,6 +72,30 @@ async function runMigrations(db) {
|
|
|
71
72
|
await doMigrate();
|
|
72
73
|
}
|
|
73
74
|
}
|
|
75
|
+
/**
|
|
76
|
+
* Crash recovery: a process kill between `markCommandInFlight` and the
|
|
77
|
+
* command's terminal update leaves rows stuck in `in_flight`. They are
|
|
78
|
+
* invisible to the outbox query (which selects pending only), so the order
|
|
79
|
+
* would silently never sync. Reset them to pending on open so the next replay
|
|
80
|
+
* picks them up. Idempotent and safe: a fresh process owns all in-flight work.
|
|
81
|
+
*/
|
|
82
|
+
async function recoverInFlightCommands(db) {
|
|
83
|
+
try {
|
|
84
|
+
const result = await db.runAsync("UPDATE commands SET status = 'pending', error = NULL, updated_at = ? WHERE status = 'in_flight';", new Date().toISOString());
|
|
85
|
+
const changes = result?.changes ?? 0;
|
|
86
|
+
if (changes > 0) {
|
|
87
|
+
syncLogger.warn("SQLite", "Recovered in-flight commands after restart", {
|
|
88
|
+
count: changes,
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
// Never block DB open on recovery — replay/backoff still works.
|
|
94
|
+
syncLogger.error("SQLite", "in-flight recovery failed", {
|
|
95
|
+
error: String(error),
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
74
99
|
export async function closeSyncDatabase() {
|
|
75
100
|
if (dbInstance) {
|
|
76
101
|
if (dbInstance.closeAsync)
|
package/dist/db/operations.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export interface InsertCommandInput {
|
|
|
8
8
|
status?: PersistedCommand["status"];
|
|
9
9
|
}
|
|
10
10
|
export declare function insertCommand(db: SQLiteDatabase, input: InsertCommandInput): Promise<PersistedCommand>;
|
|
11
|
-
export declare function getPendingCommands(db: SQLiteDatabase, limit?: number): Promise<PersistedCommand[]>;
|
|
11
|
+
export declare function getPendingCommands(db: SQLiteDatabase, limit?: number, includeFailed?: boolean): Promise<PersistedCommand[]>;
|
|
12
12
|
export declare function updateCommandStatus(db: SQLiteDatabase, commandId: string, status: PersistedCommand["status"], error?: string, incrementRetry?: boolean): Promise<void>;
|
|
13
13
|
export declare function markCommandInFlight(db: SQLiteDatabase, commandId: string): Promise<void>;
|
|
14
14
|
export declare function deleteCommand(db: SQLiteDatabase, commandId: string): Promise<void>;
|
package/dist/db/operations.js
CHANGED
|
@@ -24,10 +24,15 @@ export async function insertCommand(db, input) {
|
|
|
24
24
|
syncLogger.command(command, "queued");
|
|
25
25
|
return rowToPersistedCommand(row);
|
|
26
26
|
}
|
|
27
|
-
export async function getPendingCommands(db, limit = 100) {
|
|
27
|
+
export async function getPendingCommands(db, limit = 100, includeFailed = false) {
|
|
28
|
+
// `failed` rows are terminal (permanent rejection / retries exhausted) — they
|
|
29
|
+
// must NOT be re-fetched by the replay loop, or every failed command would be
|
|
30
|
+
// re-POSTed and re-toasted on each pass. Diagnostics opts in via
|
|
31
|
+
// includeFailed for the full outbox dump.
|
|
32
|
+
const statuses = includeFailed ? "('pending', 'failed')" : "('pending')";
|
|
28
33
|
const rows = await db.getAllAsync(`
|
|
29
34
|
SELECT * FROM commands
|
|
30
|
-
WHERE status IN
|
|
35
|
+
WHERE status IN ${statuses}
|
|
31
36
|
ORDER BY created_at ASC
|
|
32
37
|
LIMIT ?;
|
|
33
38
|
`, limit);
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* React hook for startup readiness (seller POS/order surfaces).
|
|
3
|
+
*
|
|
4
|
+
* Runs one bounded catch-up on mount (and whenever the cloud becomes
|
|
5
|
+
* reachable), then exposes the local cache readiness so the app can block the
|
|
6
|
+
* POS with an explicit "preparing offline data" state instead of surprising
|
|
7
|
+
* the seller with a guard failure at order-tap time.
|
|
8
|
+
*/
|
|
9
|
+
import { type StartupReadiness } from "../sync/startup-readiness";
|
|
10
|
+
export interface UseStartupReadinessOptions {
|
|
11
|
+
/** Bound the online catch-up attempt. Defaults to 15000ms. */
|
|
12
|
+
timeoutMs?: number;
|
|
13
|
+
/** Set false to skip automatic checking. */
|
|
14
|
+
enabled?: boolean;
|
|
15
|
+
}
|
|
16
|
+
export interface UseStartupReadinessResult {
|
|
17
|
+
businessId: string | undefined;
|
|
18
|
+
readiness: StartupReadiness | null;
|
|
19
|
+
/** True while the catch-up check is running (first load only). */
|
|
20
|
+
checking: boolean;
|
|
21
|
+
/** Re-run the readiness check on demand (e.g. a Retry button). */
|
|
22
|
+
recheck: () => Promise<void>;
|
|
23
|
+
}
|
|
24
|
+
export declare function useStartupReadiness(options?: UseStartupReadinessOptions): UseStartupReadinessResult;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* React hook for startup readiness (seller POS/order surfaces).
|
|
3
|
+
*
|
|
4
|
+
* Runs one bounded catch-up on mount (and whenever the cloud becomes
|
|
5
|
+
* reachable), then exposes the local cache readiness so the app can block the
|
|
6
|
+
* POS with an explicit "preparing offline data" state instead of surprising
|
|
7
|
+
* the seller with a guard failure at order-tap time.
|
|
8
|
+
*/
|
|
9
|
+
import { useCallback, useEffect, useRef, useState } from "react";
|
|
10
|
+
import { useSyncExternalStore } from "react";
|
|
11
|
+
import { getActiveBusiness, subscribeBusiness } from "../adapters/business";
|
|
12
|
+
import { getConnectivityState, subscribeConnectivity } from "../adapters/connectivity";
|
|
13
|
+
import { ensureStartupReady, } from "../sync/startup-readiness";
|
|
14
|
+
function useActiveBusinessId() {
|
|
15
|
+
const id = useSyncExternalStore(subscribeBusiness, () => getActiveBusiness()?.id ?? null, () => null);
|
|
16
|
+
return id ?? undefined;
|
|
17
|
+
}
|
|
18
|
+
function useCloudStatus() {
|
|
19
|
+
return useSyncExternalStore(subscribeConnectivity, () => getConnectivityState().isCloudReachable, () => "unknown");
|
|
20
|
+
}
|
|
21
|
+
export function useStartupReadiness(options = {}) {
|
|
22
|
+
const { timeoutMs, enabled = true } = options;
|
|
23
|
+
const businessId = useActiveBusinessId();
|
|
24
|
+
const cloudStatus = useCloudStatus();
|
|
25
|
+
const [readiness, setReadiness] = useState(null);
|
|
26
|
+
const [checking, setChecking] = useState(false);
|
|
27
|
+
const runRef = useRef(false);
|
|
28
|
+
const check = useCallback(async () => {
|
|
29
|
+
if (!businessId)
|
|
30
|
+
return;
|
|
31
|
+
if (runRef.current)
|
|
32
|
+
return;
|
|
33
|
+
runRef.current = true;
|
|
34
|
+
setChecking(true);
|
|
35
|
+
try {
|
|
36
|
+
const result = await ensureStartupReady(businessId, { timeoutMs });
|
|
37
|
+
setReadiness(result);
|
|
38
|
+
}
|
|
39
|
+
finally {
|
|
40
|
+
runRef.current = false;
|
|
41
|
+
setChecking(false);
|
|
42
|
+
}
|
|
43
|
+
}, [businessId, timeoutMs]);
|
|
44
|
+
// First check when a business is available.
|
|
45
|
+
useEffect(() => {
|
|
46
|
+
if (!enabled || !businessId)
|
|
47
|
+
return;
|
|
48
|
+
if (readiness)
|
|
49
|
+
return;
|
|
50
|
+
void check();
|
|
51
|
+
}, [enabled, businessId, readiness, check]);
|
|
52
|
+
// Re-check whenever cloud reachability flips to reachable (data may now sync).
|
|
53
|
+
useEffect(() => {
|
|
54
|
+
if (!enabled || !businessId)
|
|
55
|
+
return;
|
|
56
|
+
if (cloudStatus !== "reachable")
|
|
57
|
+
return;
|
|
58
|
+
void check();
|
|
59
|
+
}, [enabled, businessId, cloudStatus, check]);
|
|
60
|
+
return { businessId, readiness, checking, recheck: check };
|
|
61
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -29,6 +29,7 @@ export * from "./utils/customer-display";
|
|
|
29
29
|
export * from "./utils/order-filter";
|
|
30
30
|
export { useReferenceData } from "./reference/hooks";
|
|
31
31
|
export { setNotifyHandlers } from "./adapters/notify";
|
|
32
|
+
export { setSyncFailureReporter, reportSyncFailure, type SyncFailureReport, type SyncFailureReporter, type SyncFailurePhase, } from "./adapters/sync-failure";
|
|
32
33
|
export { walletService, riderService, fleetService, deliveryService, dashboardService, analyticsService, placesService } from "./adapters/services";
|
|
33
34
|
export { buildMenuItemEnvelope, buildMenuCategoryEnvelope } from "./reference/menu-record";
|
|
34
35
|
export { validateMenuItemInput, validateMenuCategoryInput, validateOrderCreateInput, validateInviteInput } from "./validation/offline";
|
|
@@ -54,6 +55,8 @@ export { buildCommand } from "./sync/command-builder";
|
|
|
54
55
|
export { generateLocalOrderNumber, generateLocalCalloutNumber, isLocalOrderNumber, generateOrderNumber, generateOrderNumberForBusiness, isOfflineOrderNumber, parseOrderNumber, resetCalloutForTests, } from "./sync/order-number";
|
|
55
56
|
export { deviceLeaseService, getDeviceNumber, isOnlineOnly, ensureLease } from "./sync/device-lease.service";
|
|
56
57
|
export { useDeviceLease, ensureDeviceLease } from "./sync/use-device-lease";
|
|
58
|
+
export { getStartupReadiness, ensureStartupReady, REQUIRED_ORDER_KEYS, type StartupReadiness, } from "./sync/startup-readiness";
|
|
59
|
+
export { useStartupReadiness, type UseStartupReadinessOptions, type UseStartupReadinessResult, } from "./hooks/startup-readiness.hooks";
|
|
57
60
|
export * from "./utils/hlc";
|
|
58
61
|
export * from "./reference/lww";
|
|
59
62
|
export { orderSnapshotsService } from "./sync/order-snapshots.service";
|
package/dist/index.js
CHANGED
|
@@ -30,6 +30,7 @@ export * from "./utils/customer-display";
|
|
|
30
30
|
export * from "./utils/order-filter";
|
|
31
31
|
export { useReferenceData } from "./reference/hooks";
|
|
32
32
|
export { setNotifyHandlers } from "./adapters/notify";
|
|
33
|
+
export { setSyncFailureReporter, reportSyncFailure, } from "./adapters/sync-failure";
|
|
33
34
|
export { walletService, riderService, fleetService, deliveryService, dashboardService, analyticsService, placesService } from "./adapters/services";
|
|
34
35
|
export { buildMenuItemEnvelope, buildMenuCategoryEnvelope } from "./reference/menu-record";
|
|
35
36
|
export { validateMenuItemInput, validateMenuCategoryInput, validateOrderCreateInput, validateInviteInput } from "./validation/offline";
|
|
@@ -57,6 +58,8 @@ export { buildCommand } from "./sync/command-builder";
|
|
|
57
58
|
export { generateLocalOrderNumber, generateLocalCalloutNumber, isLocalOrderNumber, generateOrderNumber, generateOrderNumberForBusiness, isOfflineOrderNumber, parseOrderNumber, resetCalloutForTests, } from "./sync/order-number";
|
|
58
59
|
export { deviceLeaseService, getDeviceNumber, isOnlineOnly, ensureLease } from "./sync/device-lease.service";
|
|
59
60
|
export { useDeviceLease, ensureDeviceLease } from "./sync/use-device-lease";
|
|
61
|
+
export { getStartupReadiness, ensureStartupReady, REQUIRED_ORDER_KEYS, } from "./sync/startup-readiness";
|
|
62
|
+
export { useStartupReadiness, } from "./hooks/startup-readiness.hooks";
|
|
60
63
|
export * from "./utils/hlc";
|
|
61
64
|
export * from "./reference/lww";
|
|
62
65
|
export { orderSnapshotsService } from "./sync/order-snapshots.service";
|
|
@@ -24,6 +24,8 @@ declare class CommandQueueService {
|
|
|
24
24
|
* Try to execute a single pending command through the current transport.
|
|
25
25
|
*/
|
|
26
26
|
processCommand(command: PersistedCommand): Promise<void>;
|
|
27
|
+
/** Forward a failed attempt to the host's observability reporter. */
|
|
28
|
+
private reportFailure;
|
|
27
29
|
/**
|
|
28
30
|
* Mark a command permanently failed: loud toast + optimistic revert, then
|
|
29
31
|
* replay continues to the next command. Shared by the exhausted-retry and
|
|
@@ -39,8 +41,13 @@ declare class CommandQueueService {
|
|
|
39
41
|
*/
|
|
40
42
|
private absorbIfApplied;
|
|
41
43
|
/**
|
|
42
|
-
* Replay
|
|
43
|
-
*
|
|
44
|
+
* Replay pending commands in FIFO order with entity-aware skip-and-continue.
|
|
45
|
+
*
|
|
46
|
+
* Ordering is preserved per entity: a failing command defers only its own
|
|
47
|
+
* entity_id, so that order's later commands stay queued behind it, while
|
|
48
|
+
* independent entities keep syncing in the same pass. The pass stops only
|
|
49
|
+
* when nothing can make progress: no transport at all (REPLAY_NO_TRANSPORT)
|
|
50
|
+
* or TRANSPORT_FAILURE_BREAKER consecutive network/5xx failures.
|
|
44
51
|
*/
|
|
45
52
|
replay(): Promise<void>;
|
|
46
53
|
/**
|
|
@@ -12,7 +12,7 @@ import { getSyncDatabase, openSyncDatabase } from "../db/connection";
|
|
|
12
12
|
import { insertCommand, getPendingCommands, markCommandInFlight, updateCommandStatus, deleteCommand, insertEvent, } from "../db/operations";
|
|
13
13
|
import { syncLogger } from "../utils/sync-logger";
|
|
14
14
|
import { recordAttempt } from "../utils/attempt-stats";
|
|
15
|
-
import { TransitionError, isReplayDeferred, TRANSITION_ERROR_CODES } from "./transition-errors";
|
|
15
|
+
import { TransitionError, isReplayDeferred, isReplayNoTransport, TRANSITION_ERROR_CODES } from "./transition-errors";
|
|
16
16
|
import { assertBusinessOpenForCreate } from "../utils/business-hours";
|
|
17
17
|
import { extractApiErrorCode } from "../utils/error";
|
|
18
18
|
import { generateEventId } from "../utils/uuid";
|
|
@@ -21,16 +21,18 @@ import { generateLocalCalloutNumber, generateOrderNumberForBusiness, generateOrd
|
|
|
21
21
|
import { getConnectivityState, subscribeConnectivity } from "../adapters/connectivity";
|
|
22
22
|
import { canBusinessSync } from "./sync-guard";
|
|
23
23
|
import { backoffDelayMs } from "./backoff";
|
|
24
|
-
|
|
24
|
+
import { reportSyncFailure } from "../adapters/sync-failure";
|
|
25
|
+
const MAX_ATTEMPTS = 8;
|
|
25
26
|
/**
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
|
|
27
|
+
* Stop a replay pass after this many consecutive transport-level failures
|
|
28
|
+
* (network/timeout/5xx) so a dead server is not hammered once per queued
|
|
29
|
+
* command. Any server response (2xx, permanent 4xx) or deferral resets the
|
|
30
|
+
* counter. The next trigger (connectivity/backoff) starts a fresh pass.
|
|
31
|
+
*/
|
|
32
|
+
const TRANSPORT_FAILURE_BREAKER = 3;
|
|
33
|
+
/**
|
|
34
|
+
* In-memory next-eligible time per command (shared 2s→5min curve).
|
|
31
35
|
*/
|
|
32
|
-
const RETRYABLE_4XX = new Set([404, 408, 409, 422, 429]);
|
|
33
|
-
/** In-memory next-eligible time per command (shared 2s→5min curve). */
|
|
34
36
|
const retryNotBefore = new Map();
|
|
35
37
|
const QUEUED_STATUS_BY_COMMAND = {
|
|
36
38
|
"order:accept": "accepted",
|
|
@@ -260,7 +262,7 @@ class CommandQueueService {
|
|
|
260
262
|
timestamp: command.timestamp,
|
|
261
263
|
payload: command.payload,
|
|
262
264
|
};
|
|
263
|
-
if (command.retry_count >=
|
|
265
|
+
if (command.retry_count >= MAX_ATTEMPTS) {
|
|
264
266
|
await updateCommandStatus(db, command.command_id, "failed", "Max retry count exceeded");
|
|
265
267
|
return;
|
|
266
268
|
}
|
|
@@ -294,10 +296,16 @@ class CommandQueueService {
|
|
|
294
296
|
// the next trigger (do not mark, do not count). Must reset the
|
|
295
297
|
// in_flight flag first — rows stuck in_flight are invisible to the
|
|
296
298
|
// outbox query and would wedge the queue forever.
|
|
297
|
-
if (
|
|
299
|
+
if (isReplayNoTransport(error)) {
|
|
298
300
|
await updateCommandStatus(db, command.command_id, "pending", "deferred:no-transport");
|
|
299
301
|
throw error;
|
|
300
302
|
}
|
|
303
|
+
// Per-command backoff (retry window or retryable-4xx backoff): leave the
|
|
304
|
+
// row pending and rethrow so the pass defers just this entity.
|
|
305
|
+
if (isReplayDeferred(error)) {
|
|
306
|
+
await updateCommandStatus(db, command.command_id, "pending", "deferred:backoff");
|
|
307
|
+
throw error;
|
|
308
|
+
}
|
|
301
309
|
const apiErr = error;
|
|
302
310
|
const message = error instanceof Error ? error.message : String(error);
|
|
303
311
|
// 409 ORDER_NUMBER_TAKEN is retryable once with new RRR (exact backend code, any envelope)
|
|
@@ -323,6 +331,13 @@ class CommandQueueService {
|
|
|
323
331
|
catch { }
|
|
324
332
|
}
|
|
325
333
|
await updateCommandStatus(db, command.command_id, "failed", `order_number_taken:${message}`);
|
|
334
|
+
this.reportFailure(command, {
|
|
335
|
+
status: 409,
|
|
336
|
+
code: "ORDER_NUMBER_TAKEN",
|
|
337
|
+
message,
|
|
338
|
+
attempts: command.retry_count + 1,
|
|
339
|
+
terminal: true,
|
|
340
|
+
});
|
|
326
341
|
throw error;
|
|
327
342
|
}
|
|
328
343
|
const status = typeof apiErr?.status === "number" ? apiErr.status : undefined;
|
|
@@ -339,19 +354,28 @@ class CommandQueueService {
|
|
|
339
354
|
}
|
|
340
355
|
catch { }
|
|
341
356
|
}
|
|
342
|
-
//
|
|
343
|
-
//
|
|
344
|
-
//
|
|
345
|
-
//
|
|
346
|
-
|
|
357
|
+
// Every other failure is retryable on the shared 2s→5min backoff.
|
|
358
|
+
//
|
|
359
|
+
// 4xx (validation/permission/plan races) is bounded: a device stuck in
|
|
360
|
+
// a bad local state keeps retrying, then dead-letters so an operator is
|
|
361
|
+
// alerted and can visit the device. 5xx/network is an outage, not a
|
|
362
|
+
// device problem — it keeps retrying indefinitely so a long outage can
|
|
363
|
+
// never lose good orders. Every attempt is reported to observability.
|
|
364
|
+
const is4xxResponse = status !== undefined && status >= 400 && status < 500;
|
|
365
|
+
if (is4xxResponse) {
|
|
347
366
|
const attempts = command.retry_count + 1;
|
|
348
|
-
if (attempts >=
|
|
349
|
-
await this.failPermanently(db, command, message, `
|
|
367
|
+
if (attempts >= MAX_ATTEMPTS) {
|
|
368
|
+
await this.failPermanently(db, command, message, `4xx exhausted after ${attempts} attempts`, {
|
|
369
|
+
status,
|
|
370
|
+
code,
|
|
371
|
+
attempts,
|
|
372
|
+
});
|
|
350
373
|
return;
|
|
351
374
|
}
|
|
375
|
+
this.reportFailure(command, { status, code, message, attempts, terminal: false });
|
|
352
376
|
retryNotBefore.set(command.command_id, Date.now() + backoffDelayMs(command.retry_count));
|
|
353
377
|
await updateCommandStatus(db, command.command_id, "pending", `retry_4xx:${message}`, true);
|
|
354
|
-
syncLogger.warn("CommandQueueService", "
|
|
378
|
+
syncLogger.warn("CommandQueueService", "4xx — backing off", {
|
|
355
379
|
command_id: command.command_id,
|
|
356
380
|
command_type: command.command_type,
|
|
357
381
|
status,
|
|
@@ -359,30 +383,37 @@ class CommandQueueService {
|
|
|
359
383
|
});
|
|
360
384
|
throw new TransitionError(TRANSITION_ERROR_CODES.REPLAY_DEFERRED, `Retryable ${status} on ${command.command_type}, backing off`);
|
|
361
385
|
}
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
// Do not throw — allow replay to continue to next command.
|
|
368
|
-
return;
|
|
369
|
-
}
|
|
370
|
-
// Network/timeout/5xx: NOT a rejection — keep the command retryable
|
|
371
|
-
// indefinitely (status back to pending, retry_count untouched) so a
|
|
372
|
-
// long outage never dead-letters the outbox via MAX_RETRIES. Replay
|
|
373
|
-
// stops here to preserve ordering; the next trigger retries.
|
|
386
|
+
// 5xx / network / timeout: keep the command retryable indefinitely
|
|
387
|
+
// (status back to pending, retry_count untouched) so a long outage never
|
|
388
|
+
// dead-letters the outbox. Replay defers this entity; the next trigger
|
|
389
|
+
// retries.
|
|
390
|
+
this.reportFailure(command, { status, code, message, attempts: command.retry_count, terminal: false });
|
|
374
391
|
await updateCommandStatus(db, command.command_id, "pending", `network:${message}`);
|
|
375
392
|
syncLogger.command(parsedCommand, "queued", { error: message, deferred: true });
|
|
376
393
|
throw error;
|
|
377
394
|
}
|
|
378
395
|
}
|
|
396
|
+
/** Forward a failed attempt to the host's observability reporter. */
|
|
397
|
+
reportFailure(command, extra) {
|
|
398
|
+
reportSyncFailure({
|
|
399
|
+
phase: "outbox",
|
|
400
|
+
business_id: command.business_id,
|
|
401
|
+
user_id: command.user_id,
|
|
402
|
+
device_id: command.device_id,
|
|
403
|
+
command_type: command.command_type,
|
|
404
|
+
command_id: command.command_id,
|
|
405
|
+
entity_id: command.entity_id,
|
|
406
|
+
...extra,
|
|
407
|
+
});
|
|
408
|
+
}
|
|
379
409
|
/**
|
|
380
410
|
* Mark a command permanently failed: loud toast + optimistic revert, then
|
|
381
411
|
* replay continues to the next command. Shared by the exhausted-retry and
|
|
382
412
|
* permanent-rejection paths.
|
|
383
413
|
*/
|
|
384
|
-
async failPermanently(db, command, message, reason) {
|
|
414
|
+
async failPermanently(db, command, message, reason, extra) {
|
|
385
415
|
await updateCommandStatus(db, command.command_id, "failed", `permanent:${message}`);
|
|
416
|
+
this.reportFailure(command, { ...extra, message, terminal: true });
|
|
386
417
|
syncLogger.warn("CommandQueueService", `Queued command failed (${reason})`, {
|
|
387
418
|
command_id: command.command_id,
|
|
388
419
|
command_type: command.command_type,
|
|
@@ -437,8 +468,13 @@ class CommandQueueService {
|
|
|
437
468
|
return false;
|
|
438
469
|
}
|
|
439
470
|
/**
|
|
440
|
-
* Replay
|
|
441
|
-
*
|
|
471
|
+
* Replay pending commands in FIFO order with entity-aware skip-and-continue.
|
|
472
|
+
*
|
|
473
|
+
* Ordering is preserved per entity: a failing command defers only its own
|
|
474
|
+
* entity_id, so that order's later commands stay queued behind it, while
|
|
475
|
+
* independent entities keep syncing in the same pass. The pass stops only
|
|
476
|
+
* when nothing can make progress: no transport at all (REPLAY_NO_TRANSPORT)
|
|
477
|
+
* or TRANSPORT_FAILURE_BREAKER consecutive network/5xx failures.
|
|
442
478
|
*/
|
|
443
479
|
async replay() {
|
|
444
480
|
if (!canBusinessSync())
|
|
@@ -469,21 +505,59 @@ class CommandQueueService {
|
|
|
469
505
|
syncLogger.info("CommandQueueService", "Replaying commands", {
|
|
470
506
|
count: pending.length,
|
|
471
507
|
});
|
|
508
|
+
// Entities deferred in THIS pass (backoff window or a transport failure):
|
|
509
|
+
// later commands for the same entity are skipped to preserve per-entity
|
|
510
|
+
// FIFO without blocking independent commands.
|
|
511
|
+
const deferredEntities = new Set();
|
|
512
|
+
let consecutiveTransportFailures = 0;
|
|
472
513
|
for (const command of pending) {
|
|
514
|
+
const entity = command.entity_id;
|
|
515
|
+
if (entity && deferredEntities.has(entity)) {
|
|
516
|
+
syncLogger.debug("CommandQueueService", "Skipping command for deferred entity", {
|
|
517
|
+
command_id: command.command_id,
|
|
518
|
+
entity_id: entity,
|
|
519
|
+
});
|
|
520
|
+
continue;
|
|
521
|
+
}
|
|
473
522
|
try {
|
|
474
523
|
await this.processCommand(command);
|
|
524
|
+
consecutiveTransportFailures = 0;
|
|
475
525
|
}
|
|
476
526
|
catch (e) {
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
527
|
+
// No transport at all: nothing in this pass can succeed — stop.
|
|
528
|
+
if (isReplayNoTransport(e)) {
|
|
529
|
+
syncLogger.info("CommandQueueService", "Replay stopped: no transport available", {
|
|
530
|
+
command_id: command.command_id,
|
|
531
|
+
});
|
|
532
|
+
break;
|
|
533
|
+
}
|
|
534
|
+
// Per-command deferral (backoff window / retryable-4xx backoff):
|
|
535
|
+
// defer this entity only and keep draining the rest.
|
|
536
|
+
if (isReplayDeferred(e)) {
|
|
537
|
+
if (entity)
|
|
538
|
+
deferredEntities.add(entity);
|
|
539
|
+
consecutiveTransportFailures = 0;
|
|
483
540
|
continue;
|
|
484
541
|
}
|
|
485
|
-
|
|
486
|
-
|
|
542
|
+
const status = e?.status;
|
|
543
|
+
const is4xx = typeof status === "number" && status >= 400 && status < 500;
|
|
544
|
+
if (is4xx) {
|
|
545
|
+
// Permanent/retryable-exhausted 4xx: processCommand already marked
|
|
546
|
+
// the row failed (or absorbed it) — continue to the next command.
|
|
547
|
+
consecutiveTransportFailures = 0;
|
|
548
|
+
continue;
|
|
549
|
+
}
|
|
550
|
+
// Network/timeout/5xx: defer this entity, continue, but trip the
|
|
551
|
+
// breaker if the transport looks dead so we stop hammering.
|
|
552
|
+
if (entity)
|
|
553
|
+
deferredEntities.add(entity);
|
|
554
|
+
consecutiveTransportFailures += 1;
|
|
555
|
+
if (consecutiveTransportFailures >= TRANSPORT_FAILURE_BREAKER) {
|
|
556
|
+
syncLogger.warn("CommandQueueService", "Replay stopped: transport failure breaker tripped", {
|
|
557
|
+
consecutiveTransportFailures,
|
|
558
|
+
});
|
|
559
|
+
break;
|
|
560
|
+
}
|
|
487
561
|
}
|
|
488
562
|
}
|
|
489
563
|
}
|
|
@@ -520,7 +594,7 @@ class CommandQueueService {
|
|
|
520
594
|
async debugDumpCommands(businessId) {
|
|
521
595
|
try {
|
|
522
596
|
const db = getSyncDatabase();
|
|
523
|
-
const rows = await getPendingCommands(db, Number.MAX_SAFE_INTEGER);
|
|
597
|
+
const rows = await getPendingCommands(db, Number.MAX_SAFE_INTEGER, true);
|
|
524
598
|
return rows
|
|
525
599
|
.filter((c) => !businessId || c.business_id === businessId)
|
|
526
600
|
.map((c) => ({
|
package/dist/sync/engine.js
CHANGED
|
@@ -151,9 +151,11 @@ export class SyncEngine {
|
|
|
151
151
|
if (attempt === "queue") {
|
|
152
152
|
// Replays must never re-enqueue-then-delete (the command would eat
|
|
153
153
|
// itself without executing). If no real transport is available, leave
|
|
154
|
-
// the row untouched for a later trigger.
|
|
154
|
+
// the row untouched for a later trigger. Distinct code so the outbox
|
|
155
|
+
// stops the whole replay pass (nothing can succeed) rather than
|
|
156
|
+
// skipping just this entity and hammering the rest.
|
|
155
157
|
if (opts?.forReplay) {
|
|
156
|
-
throw new TransitionError(TRANSITION_ERROR_CODES.
|
|
158
|
+
throw new TransitionError(TRANSITION_ERROR_CODES.REPLAY_NO_TRANSPORT, "No transport available, keeping command queued");
|
|
157
159
|
}
|
|
158
160
|
const event = await commandQueueService.enqueue(command);
|
|
159
161
|
if (queueDespiteEntitlement) {
|
package/dist/sync/full-sync.d.ts
CHANGED
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Full sync: push → pull → push.
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* Full sync: reference → push → pull → push.
|
|
3
|
+
*
|
|
4
|
+
* Reference (menu/hours/tax/capabilities) is pulled FIRST and non-blockingly:
|
|
5
|
+
* every phase is independently swallowed, so a reference failure never
|
|
6
|
+
* prevents queued orders from flushing. Getting reference in before the order
|
|
7
|
+
* push means newly-launched terminals can satisfy order-creation prerequisites
|
|
8
|
+
* (hour/tax/capability guards) in the same cycle.
|
|
9
|
+
*
|
|
10
|
+
* Orders still push before the order pull, so local edits (already HLC-bumped)
|
|
11
|
+
* reach the server before the pull merge could overwrite them. The final
|
|
12
|
+
* drain flushes anything newly queued or retried after the pull.
|
|
5
13
|
*/
|
|
6
14
|
export declare function performFullSync(businessId?: string): Promise<void>;
|
package/dist/sync/full-sync.js
CHANGED
|
@@ -1,7 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Full sync: push → pull → push.
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* Full sync: reference → push → pull → push.
|
|
3
|
+
*
|
|
4
|
+
* Reference (menu/hours/tax/capabilities) is pulled FIRST and non-blockingly:
|
|
5
|
+
* every phase is independently swallowed, so a reference failure never
|
|
6
|
+
* prevents queued orders from flushing. Getting reference in before the order
|
|
7
|
+
* push means newly-launched terminals can satisfy order-creation prerequisites
|
|
8
|
+
* (hour/tax/capability guards) in the same cycle.
|
|
9
|
+
*
|
|
10
|
+
* Orders still push before the order pull, so local edits (already HLC-bumped)
|
|
11
|
+
* reach the server before the pull merge could overwrite them. The final
|
|
12
|
+
* drain flushes anything newly queued or retried after the pull.
|
|
5
13
|
*/
|
|
6
14
|
import { getConnectivityState } from "../adapters/connectivity";
|
|
7
15
|
import { getActiveBusiness } from "../adapters/business";
|
|
@@ -26,28 +34,30 @@ export async function performFullSync(businessId) {
|
|
|
26
34
|
}
|
|
27
35
|
const p = (async () => {
|
|
28
36
|
try {
|
|
29
|
-
syncLogger.info("FullSync", "Starting push→pull→push", { businessId: bid });
|
|
30
|
-
// 1.
|
|
37
|
+
syncLogger.info("FullSync", "Starting reference→push→pull→push", { businessId: bid });
|
|
38
|
+
// 1. Reference first (non-blocking): menu/hours/tax/caps so order
|
|
39
|
+
// creation prerequisites are present before we flush orders.
|
|
31
40
|
try {
|
|
32
|
-
await
|
|
41
|
+
await referenceSyncService.sync({ businessId: bid });
|
|
33
42
|
}
|
|
34
43
|
catch (e) {
|
|
35
|
-
syncLogger.warn("FullSync", "
|
|
44
|
+
syncLogger.warn("FullSync", "reference sync failed", { err: String(e) });
|
|
36
45
|
}
|
|
37
|
-
// 2.
|
|
46
|
+
// 2. Push pending FIFO — HLC-bumped local edits outrank stale server
|
|
38
47
|
try {
|
|
39
|
-
await
|
|
48
|
+
await drainPending(bid);
|
|
40
49
|
}
|
|
41
50
|
catch (e) {
|
|
42
|
-
syncLogger.warn("FullSync", "
|
|
51
|
+
syncLogger.warn("FullSync", "drainPending(pre) failed", { err: String(e) });
|
|
43
52
|
}
|
|
53
|
+
// 3. Pull remote orders (HLC-guarded)
|
|
44
54
|
try {
|
|
45
|
-
await
|
|
55
|
+
await pullRemoteOrders(bid);
|
|
46
56
|
}
|
|
47
57
|
catch (e) {
|
|
48
|
-
syncLogger.warn("FullSync", "
|
|
58
|
+
syncLogger.warn("FullSync", "pullRemoteOrders failed", { err: String(e) });
|
|
49
59
|
}
|
|
50
|
-
//
|
|
60
|
+
// 4. Final drain — anything newly queued or retried after pull
|
|
51
61
|
try {
|
|
52
62
|
await drainPending(bid);
|
|
53
63
|
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Startup readiness for the POS/order surfaces.
|
|
3
|
+
*
|
|
4
|
+
* The offline-first guards intentionally fail closed (HOURS_NOT_SYNCED,
|
|
5
|
+
* SUBSCRIPTION_REQUIRED, "no device leased"), which is correct — but if the
|
|
6
|
+
* required reference data was never loaded, those guards fire at order-tap
|
|
7
|
+
* time as a surprise. This module lets the app discover the same conditions at
|
|
8
|
+
* launch: it evaluates the local cache (no network) and, when online, performs
|
|
9
|
+
* one bounded catch-up sync so a freshly-launched terminal can satisfy the
|
|
10
|
+
* guards before the seller starts taking orders.
|
|
11
|
+
*
|
|
12
|
+
* Seller-only. Fleet/rider surfaces are fully online and never gate here.
|
|
13
|
+
*/
|
|
14
|
+
import type { ReferenceDataKey } from "../reference/types";
|
|
15
|
+
/**
|
|
16
|
+
* Reference keys required before the app presents itself as ready to take
|
|
17
|
+
* orders: menu to sell, taxes to price, hours to open, profile/capabilities
|
|
18
|
+
* for routing + guards, and the signed entitlement proof for gated commands.
|
|
19
|
+
*/
|
|
20
|
+
export declare const REQUIRED_ORDER_KEYS: readonly ReferenceDataKey[];
|
|
21
|
+
export interface StartupReadiness {
|
|
22
|
+
/** True when every required key is cached and a device number is leased. */
|
|
23
|
+
ready: boolean;
|
|
24
|
+
/** Required keys with no cached payload. */
|
|
25
|
+
missingKeys: ReferenceDataKey[];
|
|
26
|
+
/** True when no DD-RRR-CCC device number is available (needs one online lease). */
|
|
27
|
+
needsLease: boolean;
|
|
28
|
+
/**
|
|
29
|
+
* True when an entitlement envelope is absent or does not evaluate as
|
|
30
|
+
* entitled. Informational: cash orders still queue offline, so this does not
|
|
31
|
+
* by itself block readiness.
|
|
32
|
+
*/
|
|
33
|
+
entitlementUnverified: boolean;
|
|
34
|
+
}
|
|
35
|
+
/** Evaluate the local cache only (no network). */
|
|
36
|
+
export declare function getStartupReadiness(businessId: string): Promise<StartupReadiness>;
|
|
37
|
+
/**
|
|
38
|
+
* One bounded catch-up attempt, then evaluate the cache. When offline (or the
|
|
39
|
+
* cloud is unreachable) this performs no network work and simply reports what
|
|
40
|
+
* is cached. Never throws.
|
|
41
|
+
*/
|
|
42
|
+
export declare function ensureStartupReady(businessId: string, opts?: {
|
|
43
|
+
timeoutMs?: number;
|
|
44
|
+
}): Promise<StartupReadiness>;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Startup readiness for the POS/order surfaces.
|
|
3
|
+
*
|
|
4
|
+
* The offline-first guards intentionally fail closed (HOURS_NOT_SYNCED,
|
|
5
|
+
* SUBSCRIPTION_REQUIRED, "no device leased"), which is correct — but if the
|
|
6
|
+
* required reference data was never loaded, those guards fire at order-tap
|
|
7
|
+
* time as a surprise. This module lets the app discover the same conditions at
|
|
8
|
+
* launch: it evaluates the local cache (no network) and, when online, performs
|
|
9
|
+
* one bounded catch-up sync so a freshly-launched terminal can satisfy the
|
|
10
|
+
* guards before the seller starts taking orders.
|
|
11
|
+
*
|
|
12
|
+
* Seller-only. Fleet/rider surfaces are fully online and never gate here.
|
|
13
|
+
*/
|
|
14
|
+
import { openSyncDatabase } from "../db/connection";
|
|
15
|
+
import { getReferenceData } from "../reference/operations";
|
|
16
|
+
import { referenceSyncService } from "../reference/service";
|
|
17
|
+
import { evaluateBusinessEntitlements } from "../reference/entitlement-guard";
|
|
18
|
+
import { ensureLease } from "./device-lease.service";
|
|
19
|
+
import { getConnectivityState } from "../adapters/connectivity";
|
|
20
|
+
import { syncLogger } from "../utils/sync-logger";
|
|
21
|
+
/**
|
|
22
|
+
* Reference keys required before the app presents itself as ready to take
|
|
23
|
+
* orders: menu to sell, taxes to price, hours to open, profile/capabilities
|
|
24
|
+
* for routing + guards, and the signed entitlement proof for gated commands.
|
|
25
|
+
*/
|
|
26
|
+
export const REQUIRED_ORDER_KEYS = [
|
|
27
|
+
"subscription:entitlements",
|
|
28
|
+
"capabilities",
|
|
29
|
+
"menu:categories",
|
|
30
|
+
"menu:items",
|
|
31
|
+
"tax-rules",
|
|
32
|
+
"business:profile",
|
|
33
|
+
"operating-hours",
|
|
34
|
+
];
|
|
35
|
+
/** Evaluate the local cache only (no network). */
|
|
36
|
+
export async function getStartupReadiness(businessId) {
|
|
37
|
+
const db = await openSyncDatabase();
|
|
38
|
+
const records = await Promise.all(REQUIRED_ORDER_KEYS.map(async (key) => ({
|
|
39
|
+
key,
|
|
40
|
+
rec: await getReferenceData(db, businessId, key),
|
|
41
|
+
})));
|
|
42
|
+
const missingKeys = records
|
|
43
|
+
.filter((r) => !r.rec || r.rec.payload == null)
|
|
44
|
+
.map((r) => r.key);
|
|
45
|
+
// ensureLease() hydrates the sync KV from async KV and leases online when
|
|
46
|
+
// possible; offline with no cache it returns null (caller blocks POS).
|
|
47
|
+
let deviceNumber = null;
|
|
48
|
+
try {
|
|
49
|
+
deviceNumber = await ensureLease(businessId);
|
|
50
|
+
}
|
|
51
|
+
catch (e) {
|
|
52
|
+
syncLogger.warn("StartupReadiness", "lease check failed", { err: String(e) });
|
|
53
|
+
}
|
|
54
|
+
const needsLease = !deviceNumber;
|
|
55
|
+
let entitlementUnverified = missingKeys.includes("subscription:entitlements");
|
|
56
|
+
if (!entitlementUnverified) {
|
|
57
|
+
try {
|
|
58
|
+
const { evaluation } = await evaluateBusinessEntitlements(businessId);
|
|
59
|
+
entitlementUnverified = !evaluation.entitled;
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
entitlementUnverified = true;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
ready: missingKeys.length === 0 && !needsLease,
|
|
67
|
+
missingKeys,
|
|
68
|
+
needsLease,
|
|
69
|
+
entitlementUnverified,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* One bounded catch-up attempt, then evaluate the cache. When offline (or the
|
|
74
|
+
* cloud is unreachable) this performs no network work and simply reports what
|
|
75
|
+
* is cached. Never throws.
|
|
76
|
+
*/
|
|
77
|
+
export async function ensureStartupReady(businessId, opts) {
|
|
78
|
+
const timeoutMs = opts?.timeoutMs ?? 15000;
|
|
79
|
+
const { isOnline, isCloudReachable } = getConnectivityState();
|
|
80
|
+
if (isOnline && isCloudReachable !== "unreachable") {
|
|
81
|
+
const catchUp = (async () => {
|
|
82
|
+
try {
|
|
83
|
+
await referenceSyncService.sync({ businessId, keys: [...REQUIRED_ORDER_KEYS] });
|
|
84
|
+
}
|
|
85
|
+
catch (e) {
|
|
86
|
+
syncLogger.warn("StartupReadiness", "reference catch-up failed", { err: String(e) });
|
|
87
|
+
}
|
|
88
|
+
try {
|
|
89
|
+
await ensureLease(businessId);
|
|
90
|
+
}
|
|
91
|
+
catch (e) {
|
|
92
|
+
syncLogger.warn("StartupReadiness", "lease catch-up failed", { err: String(e) });
|
|
93
|
+
}
|
|
94
|
+
})();
|
|
95
|
+
await raceWithTimeout(catchUp, timeoutMs);
|
|
96
|
+
}
|
|
97
|
+
return getStartupReadiness(businessId);
|
|
98
|
+
}
|
|
99
|
+
/** Resolve after either the promise settles or the timeout elapses. */
|
|
100
|
+
async function raceWithTimeout(promise, ms) {
|
|
101
|
+
let timer;
|
|
102
|
+
const timeout = new Promise((resolve) => {
|
|
103
|
+
timer = setTimeout(resolve, ms);
|
|
104
|
+
});
|
|
105
|
+
try {
|
|
106
|
+
await Promise.race([
|
|
107
|
+
promise.then(() => undefined, () => undefined),
|
|
108
|
+
timeout,
|
|
109
|
+
]);
|
|
110
|
+
}
|
|
111
|
+
finally {
|
|
112
|
+
if (timer)
|
|
113
|
+
clearTimeout(timer);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
@@ -19,11 +19,13 @@ import { getTransitionPath, DELIVERY_COMMANDS } from "./transition-endpoints";
|
|
|
19
19
|
import { getQueuedStatus } from "../utils/order-transitions";
|
|
20
20
|
import { extractApiErrorCode } from "../utils/error";
|
|
21
21
|
import { canBusinessSync, isAuthenticated } from "./sync-guard";
|
|
22
|
+
import { reportSyncFailure } from "../adapters/sync-failure";
|
|
23
|
+
import { getAppInstanceId } from "../adapters/devices";
|
|
22
24
|
const inflight = new Map();
|
|
23
25
|
const retryTimers = new Map();
|
|
24
26
|
const backoffTracker = new BackoffTracker();
|
|
25
|
-
/**
|
|
26
|
-
const MAX_SYNC_JOB_ATTEMPTS =
|
|
27
|
+
/** Bounded 4xx retries stop here, loudly. 5xx/network retry indefinitely. */
|
|
28
|
+
const MAX_SYNC_JOB_ATTEMPTS = 8;
|
|
27
29
|
function isOnline() {
|
|
28
30
|
const s = getConnectivityState();
|
|
29
31
|
return s.isOnline && s.isCloudReachable !== "unreachable";
|
|
@@ -65,6 +67,36 @@ function clearRetry(businessId) {
|
|
|
65
67
|
}
|
|
66
68
|
backoffTracker.reset(businessId);
|
|
67
69
|
}
|
|
70
|
+
/** Best-effort device id for failure reports (never throws). */
|
|
71
|
+
function safeDeviceId() {
|
|
72
|
+
try {
|
|
73
|
+
return getAppInstanceId();
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/** Forward a failed job attempt to the host's observability reporter. */
|
|
80
|
+
function reportJobFailure(businessId, row, extra) {
|
|
81
|
+
let userId;
|
|
82
|
+
try {
|
|
83
|
+
const p = JSON.parse(row.payload);
|
|
84
|
+
const candidate = p.user_id ?? p.user?.id;
|
|
85
|
+
if (typeof candidate === "string")
|
|
86
|
+
userId = candidate;
|
|
87
|
+
}
|
|
88
|
+
catch { }
|
|
89
|
+
reportSyncFailure({
|
|
90
|
+
phase: "sync_jobs",
|
|
91
|
+
business_id: businessId,
|
|
92
|
+
user_id: userId,
|
|
93
|
+
device_id: safeDeviceId(),
|
|
94
|
+
command_type: row.command_type,
|
|
95
|
+
command_id: row.id,
|
|
96
|
+
entity_id: row.entity_id,
|
|
97
|
+
...extra,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
68
100
|
async function tryPhxPush(businessId, payload) {
|
|
69
101
|
try {
|
|
70
102
|
const socket = phoenixSocket.getChannel?.(`business:${businessId}`);
|
|
@@ -385,38 +417,13 @@ export async function drainOne(businessId, entityId) {
|
|
|
385
417
|
syncLogger.debug("SyncJobs", "401 deferred until login", { businessId });
|
|
386
418
|
return;
|
|
387
419
|
}
|
|
388
|
-
//
|
|
389
|
-
//
|
|
390
|
-
//
|
|
391
|
-
//
|
|
392
|
-
// validation, 402 plan, 403 forbidden) fails fast. Backend codes are
|
|
420
|
+
// Any 4xx is retryable with backoff (bounded); 5xx/network keeps retrying
|
|
421
|
+
// indefinitely. Every failed attempt is reported to observability so an
|
|
422
|
+
// operator can see the device/app/user and visit it. BUSINESS_CLOSED is a
|
|
423
|
+
// legitimate closed state (keep pending, re-pull hours). Backend codes are
|
|
393
424
|
// exact (no message matching).
|
|
394
425
|
const isBusinessClosed = rest.code === "BUSINESS_CLOSED" || rest.code === "HOURS_NOT_SYNCED";
|
|
395
|
-
const is4xx = rest.status && rest.status >= 400 && rest.status < 500;
|
|
396
|
-
const isRetryable4xx = rest.status === 401 || rest.status === 408 || rest.status === 409
|
|
397
|
-
|| rest.status === 422 || rest.status === 429 || rest.status === 404
|
|
398
|
-
|| isBusinessClosed;
|
|
399
|
-
if (is4xx && !isRetryable4xx) {
|
|
400
|
-
syncLogger.warn("SyncJobs", "Permanent 4xx — marking job failed to stop retry loop", { businessId, entityId, status: rest.status, code: rest.code, message: rest.message });
|
|
401
|
-
try {
|
|
402
|
-
await db.runAsync(`UPDATE sync_jobs SET status='failed', last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
|
|
403
|
-
}
|
|
404
|
-
catch { }
|
|
405
|
-
clearRetry(businessId);
|
|
406
|
-
try {
|
|
407
|
-
const { notifyError } = await import("../adapters/notify");
|
|
408
|
-
notifyError(`Server rejected ${row.command_type}: ${rest.message ?? rest.code ?? rest.status}`);
|
|
409
|
-
}
|
|
410
|
-
catch { }
|
|
411
|
-
// Retain optimistic row — pull will converge for transitions, create stays for manual retry.
|
|
412
|
-
if (row.command_type !== "order:create") {
|
|
413
|
-
try {
|
|
414
|
-
void pullRemoteOrders(businessId).catch(() => { });
|
|
415
|
-
}
|
|
416
|
-
catch { }
|
|
417
|
-
}
|
|
418
|
-
return;
|
|
419
|
-
}
|
|
426
|
+
const is4xx = !!rest.status && rest.status >= 400 && rest.status < 500;
|
|
420
427
|
// Already-applied absorb for transitions: the server already moved the
|
|
421
428
|
// order (duplicate tap, earlier drain won the race) — complete the job
|
|
422
429
|
// instead of failing it. Server truth only (hlc-guarded pull), never the
|
|
@@ -432,39 +439,43 @@ export async function drainOne(businessId, entityId) {
|
|
|
432
439
|
}
|
|
433
440
|
catch { }
|
|
434
441
|
}
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
syncLogger.warn("SyncJobs", "Retryable 4xx exhausted — marking job failed", { businessId, entityId, status: rest.status, code: rest.code, attempts: row.attempts });
|
|
442
|
+
if (isBusinessClosed) {
|
|
443
|
+
syncLogger.warn("SyncJobs", "Business closed — keeping pending for retry after hours sync", { businessId, entityId, code: rest.code });
|
|
444
|
+
reportJobFailure(businessId, row, { status: rest.status, code: rest.code, message: rest.message, attempts: row.attempts, terminal: false });
|
|
439
445
|
try {
|
|
440
|
-
await db.runAsync(`UPDATE sync_jobs SET
|
|
446
|
+
await db.runAsync(`UPDATE sync_jobs SET last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
|
|
441
447
|
}
|
|
442
448
|
catch { }
|
|
443
|
-
|
|
449
|
+
// Re-pull reference so hours merge converges, then retry with backoff.
|
|
444
450
|
try {
|
|
445
|
-
const {
|
|
446
|
-
|
|
451
|
+
const { referenceSyncService } = await import("../reference/service");
|
|
452
|
+
void referenceSyncService.sync({ businessId, keys: ["operating-hours", "business:profile"] }).catch(() => { });
|
|
447
453
|
}
|
|
448
454
|
catch { }
|
|
455
|
+
scheduleRetry(businessId);
|
|
449
456
|
return;
|
|
450
457
|
}
|
|
451
|
-
|
|
452
|
-
|
|
458
|
+
const attempts = (row.attempts ?? 0) + 1;
|
|
459
|
+
// Bounded 4xx: after MAX_SYNC_JOB_ATTEMPTS, dead-letter (retained) with a
|
|
460
|
+
// loud toast + observability report so an operator can visit the device.
|
|
461
|
+
if (is4xx && attempts >= MAX_SYNC_JOB_ATTEMPTS) {
|
|
462
|
+
syncLogger.warn("SyncJobs", "4xx exhausted — marking job failed", { businessId, entityId, status: rest.status, code: rest.code, attempts: row.attempts });
|
|
453
463
|
try {
|
|
454
|
-
await db.runAsync(`UPDATE sync_jobs SET last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
|
|
464
|
+
await db.runAsync(`UPDATE sync_jobs SET status='failed', last_error=?, updated_at=? WHERE id=?;`, `${rest.status} ${rest.code ?? ""} ${rest.message ?? ""} (after ${MAX_SYNC_JOB_ATTEMPTS} attempts)`.trim(), new Date().toISOString(), row.id);
|
|
455
465
|
}
|
|
456
466
|
catch { }
|
|
457
|
-
|
|
467
|
+
reportJobFailure(businessId, row, { status: rest.status, code: rest.code, message: rest.message, attempts, terminal: true });
|
|
468
|
+
clearRetry(businessId);
|
|
458
469
|
try {
|
|
459
|
-
const {
|
|
460
|
-
|
|
470
|
+
const { notifyError } = await import("../adapters/notify");
|
|
471
|
+
notifyError(`Server rejected ${row.command_type}: ${rest.message ?? rest.code ?? rest.status} (after retries)`);
|
|
461
472
|
}
|
|
462
473
|
catch { }
|
|
463
|
-
scheduleRetry(businessId);
|
|
464
474
|
return;
|
|
465
475
|
}
|
|
466
|
-
//
|
|
476
|
+
// 4xx (bounded) or 5xx/network (indefinite): keep pending and back off.
|
|
467
477
|
syncLogger.debug("SyncJobs", "drainOne kept pending for retry", { businessId, entityId, status: rest.status, code: rest.code });
|
|
478
|
+
reportJobFailure(businessId, row, { status: rest.status, code: rest.code, message: rest.message, attempts, terminal: false });
|
|
468
479
|
try {
|
|
469
480
|
await db.runAsync(`UPDATE sync_jobs SET attempts = attempts + 1, last_error = ?, updated_at = ? WHERE id = ?;`, `${rest.status ?? "network"} ${rest.code ?? ""} ${rest.message ?? ""}`.trim(), new Date().toISOString(), row.id);
|
|
470
481
|
}
|
|
@@ -18,6 +18,13 @@ export declare const TRANSITION_ERROR_CODES: {
|
|
|
18
18
|
readonly FORBIDDEN: "FORBIDDEN";
|
|
19
19
|
readonly SUBSCRIPTION_REQUIRED: "SUBSCRIPTION_REQUIRED";
|
|
20
20
|
readonly REPLAY_DEFERRED: "REPLAY_DEFERRED";
|
|
21
|
+
/**
|
|
22
|
+
* Replay found no usable transport at all (cloud + LAN both unavailable).
|
|
23
|
+
* Distinct from REPLAY_DEFERRED (per-command backoff/retryable-4xx) so the
|
|
24
|
+
* outbox can stop the whole pass instead of skipping just this entity:
|
|
25
|
+
* nothing can succeed until a transport returns.
|
|
26
|
+
*/
|
|
27
|
+
readonly REPLAY_NO_TRANSPORT: "REPLAY_NO_TRANSPORT";
|
|
21
28
|
readonly MISSING_RIDER: "MISSING_RIDER";
|
|
22
29
|
readonly NO_DELIVERY: "NO_DELIVERY";
|
|
23
30
|
};
|
|
@@ -30,3 +37,4 @@ export declare class TransitionError extends Error {
|
|
|
30
37
|
export declare function errorCodeOf(error: unknown): string | undefined;
|
|
31
38
|
export declare function isFinalGuardRejection(error: unknown): boolean;
|
|
32
39
|
export declare function isReplayDeferred(error: unknown): boolean;
|
|
40
|
+
export declare function isReplayNoTransport(error: unknown): boolean;
|
|
@@ -18,6 +18,13 @@ export const TRANSITION_ERROR_CODES = {
|
|
|
18
18
|
FORBIDDEN: "FORBIDDEN",
|
|
19
19
|
SUBSCRIPTION_REQUIRED: "SUBSCRIPTION_REQUIRED",
|
|
20
20
|
REPLAY_DEFERRED: "REPLAY_DEFERRED",
|
|
21
|
+
/**
|
|
22
|
+
* Replay found no usable transport at all (cloud + LAN both unavailable).
|
|
23
|
+
* Distinct from REPLAY_DEFERRED (per-command backoff/retryable-4xx) so the
|
|
24
|
+
* outbox can stop the whole pass instead of skipping just this entity:
|
|
25
|
+
* nothing can succeed until a transport returns.
|
|
26
|
+
*/
|
|
27
|
+
REPLAY_NO_TRANSPORT: "REPLAY_NO_TRANSPORT",
|
|
21
28
|
MISSING_RIDER: "MISSING_RIDER",
|
|
22
29
|
NO_DELIVERY: "NO_DELIVERY",
|
|
23
30
|
};
|
|
@@ -59,3 +66,6 @@ export function isFinalGuardRejection(error) {
|
|
|
59
66
|
export function isReplayDeferred(error) {
|
|
60
67
|
return errorCodeOf(error) === TRANSITION_ERROR_CODES.REPLAY_DEFERRED;
|
|
61
68
|
}
|
|
69
|
+
export function isReplayNoTransport(error) {
|
|
70
|
+
return errorCodeOf(error) === TRANSITION_ERROR_CODES.REPLAY_NO_TRANSPORT;
|
|
71
|
+
}
|
|
@@ -95,6 +95,18 @@ export function createMemoryDriver() {
|
|
|
95
95
|
tables.commands.push(row);
|
|
96
96
|
return { lastInsertRowId: row.id, changes: 1 };
|
|
97
97
|
}
|
|
98
|
+
// crash recovery: in_flight -> pending on open
|
|
99
|
+
if (s.includes("UPDATE commands SET status = 'pending'") && s.includes("WHERE status = 'in_flight'")) {
|
|
100
|
+
let changes = 0;
|
|
101
|
+
for (const row of tables.commands) {
|
|
102
|
+
if (row.status === "in_flight") {
|
|
103
|
+
row.status = "pending";
|
|
104
|
+
row.error = null;
|
|
105
|
+
changes += 1;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return { lastInsertRowId: 0, changes };
|
|
109
|
+
}
|
|
98
110
|
// commands update status (4- or 5-placeholder variants — always take the
|
|
99
111
|
// LAST arg as command_id; older code assumed 5).
|
|
100
112
|
if (s.includes("UPDATE commands SET status = ?")) {
|
|
@@ -365,7 +377,12 @@ export function createMemoryDriver() {
|
|
|
365
377
|
}
|
|
366
378
|
if (s.includes("SELECT * FROM commands") && s.includes("WHERE status IN")) {
|
|
367
379
|
const limit = args[args.length - 1];
|
|
368
|
-
|
|
380
|
+
// Mirror production: `failed` is terminal and only returned when the
|
|
381
|
+
// caller opts in (includeFailed -> SQL contains 'failed').
|
|
382
|
+
const includeFailed = s.includes("'failed'");
|
|
383
|
+
let rows = tables.commands
|
|
384
|
+
.filter((r) => r.status === "pending" || (includeFailed && r.status === "failed"))
|
|
385
|
+
.sort((a, b) => (a.created_at < b.created_at ? -1 : 1));
|
|
369
386
|
if (limit)
|
|
370
387
|
rows = rows.slice(0, limit);
|
|
371
388
|
return rows;
|
package/package.json
CHANGED