on-zero 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/createZeroClient.cjs +91 -12
- package/dist/cjs/createZeroClient.connection.test.cjs +163 -0
- package/dist/cjs/createZeroClient.connection.test.native.js +242 -0
- package/dist/cjs/createZeroClient.connection.test.native.js.map +1 -0
- package/dist/cjs/createZeroClient.native.js +99 -12
- package/dist/cjs/createZeroClient.native.js.map +1 -1
- package/dist/cjs/createZeroClient.recovery.test.cjs +52 -0
- package/dist/cjs/createZeroClient.recovery.test.native.js +52 -0
- package/dist/cjs/createZeroClient.recovery.test.native.js.map +1 -1
- package/dist/cjs/helpers/recoverZeroClient.cjs +197 -14
- package/dist/cjs/helpers/recoverZeroClient.native.js +220 -24
- package/dist/cjs/helpers/recoverZeroClient.native.js.map +1 -1
- package/dist/cjs/helpers/recoverZeroClient.test.cjs +241 -0
- package/dist/cjs/helpers/recoverZeroClient.test.native.js +269 -0
- package/dist/cjs/helpers/recoverZeroClient.test.native.js.map +1 -1
- package/dist/cjs/index.cjs +2 -0
- package/dist/cjs/index.native.js +2 -0
- package/dist/cjs/index.native.js.map +1 -1
- package/dist/cjs/multi.cjs +13 -0
- package/dist/cjs/multi.native.js +35 -0
- package/dist/cjs/multi.native.js.map +1 -1
- package/dist/cjs/multiPartition.test.cjs +42 -0
- package/dist/cjs/multiPartition.test.native.js +51 -0
- package/dist/cjs/multiPartition.test.native.js.map +1 -0
- package/dist/esm/createZeroClient.connection.test.mjs +164 -0
- package/dist/esm/createZeroClient.connection.test.mjs.map +1 -0
- package/dist/esm/createZeroClient.connection.test.native.js +240 -0
- package/dist/esm/createZeroClient.connection.test.native.js.map +1 -0
- package/dist/esm/createZeroClient.mjs +92 -13
- package/dist/esm/createZeroClient.mjs.map +1 -1
- package/dist/esm/createZeroClient.native.js +100 -13
- package/dist/esm/createZeroClient.native.js.map +1 -1
- package/dist/esm/createZeroClient.recovery.test.mjs +52 -0
- package/dist/esm/createZeroClient.recovery.test.mjs.map +1 -1
- package/dist/esm/createZeroClient.recovery.test.native.js +52 -0
- package/dist/esm/createZeroClient.recovery.test.native.js.map +1 -1
- package/dist/esm/helpers/recoverZeroClient.mjs +196 -15
- package/dist/esm/helpers/recoverZeroClient.mjs.map +1 -1
- package/dist/esm/helpers/recoverZeroClient.native.js +219 -25
- package/dist/esm/helpers/recoverZeroClient.native.js.map +1 -1
- package/dist/esm/helpers/recoverZeroClient.test.mjs +242 -1
- package/dist/esm/helpers/recoverZeroClient.test.mjs.map +1 -1
- package/dist/esm/helpers/recoverZeroClient.test.native.js +270 -1
- package/dist/esm/helpers/recoverZeroClient.test.native.js.map +1 -1
- package/dist/esm/index.js +2 -2
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/index.mjs +2 -2
- package/dist/esm/index.mjs.map +1 -1
- package/dist/esm/index.native.js +2 -2
- package/dist/esm/index.native.js.map +1 -1
- package/dist/esm/multi.mjs +13 -1
- package/dist/esm/multi.mjs.map +1 -1
- package/dist/esm/multi.native.js +35 -1
- package/dist/esm/multi.native.js.map +1 -1
- package/dist/esm/multiPartition.test.mjs +43 -0
- package/dist/esm/multiPartition.test.mjs.map +1 -0
- package/dist/esm/multiPartition.test.native.js +49 -0
- package/dist/esm/multiPartition.test.native.js.map +1 -0
- package/lint/ssr-guards.js +164 -0
- package/package.json +2 -2
- package/readme.md +126 -0
- package/src/createZeroClient.connection.test.tsx +162 -0
- package/src/createZeroClient.recovery.test.tsx +68 -0
- package/src/createZeroClient.tsx +159 -13
- package/src/helpers/recoverZeroClient.test.ts +270 -1
- package/src/helpers/recoverZeroClient.ts +347 -50
- package/src/index.ts +6 -0
- package/src/multi.ts +33 -0
- package/src/multiPartition.test.ts +38 -0
- package/types/createZeroClient.connection.test.d.ts +5 -0
- package/types/createZeroClient.connection.test.d.ts.map +1 -0
- package/types/createZeroClient.d.ts +17 -0
- package/types/createZeroClient.d.ts.map +1 -1
- package/types/createZeroServer.d.ts +4 -4
- package/types/helpers/recoverZeroClient.d.ts +23 -1
- package/types/helpers/recoverZeroClient.d.ts.map +1 -1
- package/types/index.d.ts +1 -1
- package/types/index.d.ts.map +1 -1
- package/types/multi.d.ts +1 -0
- package/types/multi.d.ts.map +1 -1
- package/types/multiPartition.test.d.ts +2 -0
- package/types/multiPartition.test.d.ts.map +1 -0
|
@@ -7,9 +7,59 @@ import type { Context, LogLevel, LogSink } from '@rocicorp/logger'
|
|
|
7
7
|
// surfaced as fatal instead of reload-storming on something the reload can't fix.
|
|
8
8
|
const RECOVER_GUARD_MS = 60_000
|
|
9
9
|
|
|
10
|
+
// if the scheduled reload never actually takes the page down (a consumer's
|
|
11
|
+
// scheduleReload deferred it, a native reload no-oped, or reload() threw) the
|
|
12
|
+
// module latch would stay set and kill EVERY later recovery for the page's
|
|
13
|
+
// life. time it out so recovery can try again — a real reload tears down the
|
|
14
|
+
// context long before this fires, so it only matters when the reload didn't
|
|
15
|
+
// land.
|
|
16
|
+
const RELOAD_LATCH_TIMEOUT_MS = 15_000
|
|
17
|
+
|
|
10
18
|
// the fatal local-store-loss signature Zero logs when the IndexedDB it expects
|
|
11
19
|
// is gone (evicted, or deleted by another tab).
|
|
12
20
|
const LOCAL_STORE_LOST = 'Expected IndexedDB not found'
|
|
21
|
+
const SQLITE_ERROR_NAME = 'SqliteError'
|
|
22
|
+
const SQLITE_STATEMENT_FINALIZED = 'This statement has been finalized'
|
|
23
|
+
const STORE_CLOSED = 'Store is closed'
|
|
24
|
+
const STORE_CLOSED_REPEAT_MIN_MS = 2_000
|
|
25
|
+
const STORE_CLOSED_REPEAT_MAX_MS = 60_000
|
|
26
|
+
|
|
27
|
+
export type ZeroRecoveryLogReason =
|
|
28
|
+
| 'indexeddb-not-found'
|
|
29
|
+
| 'sqlite-statement-finalized'
|
|
30
|
+
| 'store-closed-repeat'
|
|
31
|
+
| 'mutation-desync'
|
|
32
|
+
| 'connection-cookie-invalid'
|
|
33
|
+
| 'client-not-found'
|
|
34
|
+
| 'connection-userid-mismatch'
|
|
35
|
+
|
|
36
|
+
export type ZeroRecoveryLogClassification = {
|
|
37
|
+
reason: ZeroRecoveryLogReason
|
|
38
|
+
reasonKey: string
|
|
39
|
+
message: string
|
|
40
|
+
dropLocalState: boolean
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// a minimal synchronous key/value store the cross-reload guard persists into.
|
|
44
|
+
// web defaults to sessionStorage; native (Hermes) has none, so a consumer can
|
|
45
|
+
// inject an MMKV/sqlite-backed store to get cross-reload loop protection — the
|
|
46
|
+
// in-memory guard already covers within-a-page-load.
|
|
47
|
+
export type RecoveryGuardStorage = {
|
|
48
|
+
getItem: (key: string) => string | null
|
|
49
|
+
setItem: (key: string, value: string) => void
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// what a scheduleReload consumer receives so it can gate/annotate the reload
|
|
53
|
+
// (soot: IDE-active gate + countdown toast + background-mutation fence; mobile:
|
|
54
|
+
// expo-updates reload) while still driving the SAME deletes-then-reload work.
|
|
55
|
+
export type ScheduleReloadContext = {
|
|
56
|
+
reason: string
|
|
57
|
+
reasonKey: string
|
|
58
|
+
dropLocalState: boolean
|
|
59
|
+
// deletes every affected instance's local store, awaits beforeReload, then
|
|
60
|
+
// reloads. idempotent — safe to call once the consumer decides to proceed.
|
|
61
|
+
performReload: () => Promise<void>
|
|
62
|
+
}
|
|
13
63
|
|
|
14
64
|
export type ZeroRecoveryDeps = {
|
|
15
65
|
// delete THIS client's local persistent state (its own scoped IDB). pass the
|
|
@@ -21,32 +71,140 @@ export type ZeroRecoveryDeps = {
|
|
|
21
71
|
// awaited before the reload — e.g. wait for the dev origin to come back so the
|
|
22
72
|
// reload doesn't land on a restarting server. optional.
|
|
23
73
|
beforeReload?: () => Promise<void>
|
|
74
|
+
// take over WHEN/HOW the recovery reload happens. default: immediate guarded
|
|
75
|
+
// reload. the consumer decides (defer until safe, show a countdown, reload
|
|
76
|
+
// natively) but still calls ctx.performReload to run the real work.
|
|
77
|
+
scheduleReload?: (ctx: ScheduleReloadContext) => void
|
|
78
|
+
// cross-reload guard backing store (defaults to sessionStorage on web). inject
|
|
79
|
+
// a native KV so Hermes gets real cross-reload loop protection.
|
|
80
|
+
guardStorage?: RecoveryGuardStorage
|
|
81
|
+
// called with the joined log text for a classified recovery signature; return
|
|
82
|
+
// true to treat it as benign and NOT recover (soot's cf cold-boot timeout).
|
|
83
|
+
benignLogFilter?: (message: string) => boolean
|
|
24
84
|
// injectable for tests; defaults to a real page reload.
|
|
25
85
|
reload?: () => void
|
|
26
86
|
}
|
|
27
87
|
|
|
28
88
|
// one reload per page-load. the local-state deletes that must precede it are
|
|
29
89
|
// collected here so EVERY affected instance (a combined client has several) drops
|
|
30
|
-
// its own store before we reload — not just whichever instance fired first.
|
|
90
|
+
// its own store before we reload — not just whichever instance fired first. they
|
|
91
|
+
// are thunks (not started promises) so a deferred reload can't leave the app
|
|
92
|
+
// running on an already-deleted store: the deletes run only when the reload
|
|
93
|
+
// actually proceeds.
|
|
31
94
|
let reloadScheduled = false
|
|
32
|
-
|
|
95
|
+
let reloadInProgress = false
|
|
96
|
+
let reloadLatchTimer: ReturnType<typeof setTimeout> | undefined
|
|
97
|
+
const pendingDeletes: Array<() => Promise<unknown>> = []
|
|
33
98
|
|
|
34
|
-
//
|
|
35
|
-
//
|
|
36
|
-
//
|
|
37
|
-
|
|
38
|
-
|
|
99
|
+
// within-a-page-load per-reason guard: real loop protection everywhere,
|
|
100
|
+
// including Hermes (no storage needed). a reload wipes it, which is why the
|
|
101
|
+
// injectable storage below survives across reloads.
|
|
102
|
+
const inMemoryGuard = new Map<string, number>()
|
|
103
|
+
|
|
104
|
+
function defaultGuardStorage(): RecoveryGuardStorage | undefined {
|
|
39
105
|
try {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
if (Date.now() - last < RECOVER_GUARD_MS) return false
|
|
43
|
-
window.sessionStorage.setItem(key, String(Date.now()))
|
|
106
|
+
if (typeof window === 'undefined' || !window.sessionStorage) return undefined
|
|
107
|
+
return window.sessionStorage
|
|
44
108
|
} catch {
|
|
45
|
-
//
|
|
109
|
+
// sessionStorage access throws in some sandboxes / Hermes — no web store.
|
|
110
|
+
return undefined
|
|
46
111
|
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// cross-page-load, per-reason guard: distinct keys mean a NewClientGroup recovery
|
|
115
|
+
// never suppresses a later SchemaVersionNotSupported one. the in-memory tier
|
|
116
|
+
// catches re-fires within a page-load (works on Hermes); the storage tier
|
|
117
|
+
// survives the reload, so a reason that reloaded then immediately re-fires is
|
|
118
|
+
// caught as a genuine fatal instead of reload-storming across loads.
|
|
119
|
+
function recoveryGuardOpen(
|
|
120
|
+
reasonKey: string,
|
|
121
|
+
guardStorage: RecoveryGuardStorage | undefined,
|
|
122
|
+
): boolean {
|
|
123
|
+
const key = `on-zero-recover-${reasonKey}`
|
|
124
|
+
const now = Date.now()
|
|
125
|
+
|
|
126
|
+
const memLast = inMemoryGuard.get(key) ?? 0
|
|
127
|
+
if (memLast > 0 && now - memLast < RECOVER_GUARD_MS) return false
|
|
128
|
+
|
|
129
|
+
const storage = guardStorage ?? defaultGuardStorage()
|
|
130
|
+
if (storage) {
|
|
131
|
+
try {
|
|
132
|
+
const rawLast = storage.getItem(key)
|
|
133
|
+
const last = rawLast ? Number(rawLast) : 0
|
|
134
|
+
if (last > 0 && now - last < RECOVER_GUARD_MS) return false
|
|
135
|
+
storage.setItem(key, String(now))
|
|
136
|
+
} catch {
|
|
137
|
+
// storage unavailable mid-flight — the in-memory tier is the floor.
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
inMemoryGuard.set(key, now)
|
|
47
142
|
return true
|
|
48
143
|
}
|
|
49
144
|
|
|
145
|
+
// "Store is closed" is noisy during intentional instance replacement. One log is
|
|
146
|
+
// teardown noise; repeated logs seconds apart mean a live client is stuck using a
|
|
147
|
+
// closed local store and needs fresh local state.
|
|
148
|
+
let lastStoreClosedAtMs = 0
|
|
149
|
+
|
|
150
|
+
function armReloadLatchTimeout() {
|
|
151
|
+
if (typeof setTimeout !== 'function') return
|
|
152
|
+
if (reloadLatchTimer) clearTimeout(reloadLatchTimer)
|
|
153
|
+
reloadLatchTimer = setTimeout(() => {
|
|
154
|
+
reloadLatchTimer = undefined
|
|
155
|
+
// re-open SCHEDULING only. a reload that never landed (still deferred behind
|
|
156
|
+
// a consumer's IDE gate / countdown, which can hold for minutes) must not
|
|
157
|
+
// kill future recovery — but the deferred performReload still owns the
|
|
158
|
+
// pending delete thunks and, if it is already mid-flight (a slow
|
|
159
|
+
// beforeReload can exceed this timeout), reloadInProgress must stay set so a
|
|
160
|
+
// second recovery can't double-drive the reload. so touch neither
|
|
161
|
+
// pendingDeletes nor reloadInProgress here.
|
|
162
|
+
reloadScheduled = false
|
|
163
|
+
}, RELOAD_LATCH_TIMEOUT_MS)
|
|
164
|
+
// don't hold the event loop open (node/native) waiting on this defense timer.
|
|
165
|
+
if (
|
|
166
|
+
reloadLatchTimer &&
|
|
167
|
+
typeof reloadLatchTimer === 'object' &&
|
|
168
|
+
'unref' in reloadLatchTimer
|
|
169
|
+
) {
|
|
170
|
+
reloadLatchTimer.unref()
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function disarmReloadLatchTimeout() {
|
|
175
|
+
if (reloadLatchTimer) {
|
|
176
|
+
clearTimeout(reloadLatchTimer)
|
|
177
|
+
reloadLatchTimer = undefined
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// deletes every collected instance's local store, awaits beforeReload, then
|
|
182
|
+
// reloads. deferred a microtask so sibling instances failing in the same tick
|
|
183
|
+
// enqueue their deletes first. idempotent: a consumer that calls performReload
|
|
184
|
+
// twice (or after the latch timeout) only runs one reload.
|
|
185
|
+
function performReload(deps: ZeroRecoveryDeps): Promise<void> {
|
|
186
|
+
if (reloadInProgress) return Promise.resolve()
|
|
187
|
+
reloadInProgress = true
|
|
188
|
+
// the consumer has committed to reloading, so the un-latch timeout has no job
|
|
189
|
+
// now — disarm it before the (possibly slow) delete/beforeReload chain so it
|
|
190
|
+
// can't fire mid-flight and re-open scheduling under an in-progress reload.
|
|
191
|
+
disarmReloadLatchTimeout()
|
|
192
|
+
// resolve the reload exactly as @rocicorp/zero does internally
|
|
193
|
+
// (`getBrowserGlobal('location')?.reload()`, zero.js): read location off the
|
|
194
|
+
// global and optional-chain through it. a non-DOM host with a `window` shim
|
|
195
|
+
// but no real `location` (the sootsim tenant render-worker hides `location`
|
|
196
|
+
// for isolation) then drops its stale IDB and no-ops the reload — letting the
|
|
197
|
+
// host remount — instead of throwing "reading 'reload'" on undefined.
|
|
198
|
+
const doReload = deps.reload ?? (() => globalThis.location?.reload?.())
|
|
199
|
+
return Promise.resolve()
|
|
200
|
+
.then(() => Promise.allSettled(pendingDeletes.splice(0).map((run) => run())))
|
|
201
|
+
.then(() => deps.beforeReload?.())
|
|
202
|
+
.catch(() => {})
|
|
203
|
+
.then(() => {
|
|
204
|
+
doReload()
|
|
205
|
+
})
|
|
206
|
+
}
|
|
207
|
+
|
|
50
208
|
function recover(
|
|
51
209
|
deps: ZeroRecoveryDeps,
|
|
52
210
|
reasonKey: string,
|
|
@@ -58,36 +216,31 @@ function recover(
|
|
|
58
216
|
// already queued — otherwise a sibling's store survives the reload and
|
|
59
217
|
// fatal-loops on the next boot. the single scheduled reload awaits these.
|
|
60
218
|
if (dropLocalState) {
|
|
61
|
-
pendingDeletes.push(
|
|
62
|
-
Promise.resolve()
|
|
63
|
-
.then(deps.deleteLocalState)
|
|
64
|
-
.catch(() => {}),
|
|
65
|
-
)
|
|
219
|
+
pendingDeletes.push(() => Promise.resolve().then(deps.deleteLocalState))
|
|
66
220
|
}
|
|
67
221
|
// only ONE reload per page-load; a later trigger just contributes its delete.
|
|
68
222
|
if (reloadScheduled) return
|
|
69
|
-
if (!recoveryGuardOpen(reasonKey)) {
|
|
223
|
+
if (!recoveryGuardOpen(reasonKey, deps.guardStorage)) {
|
|
70
224
|
console.error(`[on-zero] ${message} — already recovered once, not reloading`)
|
|
71
225
|
deps.zeroEvents.emit({ type: 'fatal', reason: message })
|
|
72
226
|
return
|
|
73
227
|
}
|
|
74
228
|
reloadScheduled = true
|
|
229
|
+
armReloadLatchTimeout()
|
|
75
230
|
console.warn(`[on-zero] ${message} — recovering`)
|
|
76
231
|
deps.zeroEvents.emit({ type: 'recovering', reason: message })
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
.catch(() => {})
|
|
90
|
-
.finally(doReload)
|
|
232
|
+
|
|
233
|
+
const runReload = () => performReload(deps)
|
|
234
|
+
if (deps.scheduleReload) {
|
|
235
|
+
deps.scheduleReload({
|
|
236
|
+
reason: message,
|
|
237
|
+
reasonKey,
|
|
238
|
+
dropLocalState,
|
|
239
|
+
performReload: runReload,
|
|
240
|
+
})
|
|
241
|
+
} else {
|
|
242
|
+
void runReload()
|
|
243
|
+
}
|
|
91
244
|
}
|
|
92
245
|
|
|
93
246
|
// passing our own onUpdateNeeded/onClientStateNotFound DISABLES Zero's built-in
|
|
@@ -137,10 +290,139 @@ function logToConsole(
|
|
|
137
290
|
console[method](...(prefix ? [prefix] : []), ...args)
|
|
138
291
|
}
|
|
139
292
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
293
|
+
function logArgText(arg: unknown): string {
|
|
294
|
+
if (typeof arg === 'string') return arg
|
|
295
|
+
if (arg instanceof Error) return `${arg.name} ${arg.message} ${arg.stack || ''}`
|
|
296
|
+
if (arg && typeof arg === 'object') {
|
|
297
|
+
const message = 'message' in arg ? (arg as { message?: unknown }).message : undefined
|
|
298
|
+
const name = 'name' in arg ? (arg as { name?: unknown }).name : undefined
|
|
299
|
+
if (typeof message === 'string')
|
|
300
|
+
return `${typeof name === 'string' ? name : ''} ${message}`
|
|
301
|
+
}
|
|
302
|
+
return ''
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function isBenignStoreClosedLog(text: string): boolean {
|
|
306
|
+
if (/Mutator\s+".*"\s+error on server/i.test(text)) return true
|
|
307
|
+
if (/Mutator\s+".*"\s+app error on client/i.test(text)) return true
|
|
308
|
+
return false
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// the mutation/connection desync class: the local client group is out of sync
|
|
312
|
+
// with the server's last-mutation-id / cookie / client record, so the store is
|
|
313
|
+
// unusable and must be dropped + recovered. these surface only through the error
|
|
314
|
+
// log, never the structured onClientStateNotFound callback. genuinely app-infra
|
|
315
|
+
// signatures (a consumer's synthesized ack-timeout marker, a cold-boot connect
|
|
316
|
+
// timeout) are NOT here — a consumer keeps those with benignLogFilter / its own
|
|
317
|
+
// handling.
|
|
318
|
+
function classifyMutationDesync(
|
|
319
|
+
text: string,
|
|
320
|
+
): { reason: ZeroRecoveryLogReason; reasonKey: string; message: string } | undefined {
|
|
321
|
+
if (text.includes('sent mutation ID') && text.includes('but expected')) {
|
|
322
|
+
return {
|
|
323
|
+
reason: 'mutation-desync',
|
|
324
|
+
reasonKey: 'mutation-desync',
|
|
325
|
+
message: 'mutation id desync',
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
if (
|
|
329
|
+
text.includes('oooMutation') ||
|
|
330
|
+
text.includes('Server reported an out-of-order mutation')
|
|
331
|
+
) {
|
|
332
|
+
return {
|
|
333
|
+
reason: 'mutation-desync',
|
|
334
|
+
reasonKey: 'mutation-desync',
|
|
335
|
+
message: 'out-of-order mutation',
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
if (text.includes('already processed')) {
|
|
339
|
+
return {
|
|
340
|
+
reason: 'mutation-desync',
|
|
341
|
+
reasonKey: 'mutation-desync',
|
|
342
|
+
message: 'mutation already processed',
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
if (text.includes('InvalidConnectionRequestLastMutationID')) {
|
|
346
|
+
return {
|
|
347
|
+
reason: 'mutation-desync',
|
|
348
|
+
reasonKey: 'mutation-desync',
|
|
349
|
+
message: 'invalid connection last mutation id',
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
if (text.includes('InvalidConnectionRequestBaseCookie')) {
|
|
353
|
+
return {
|
|
354
|
+
reason: 'connection-cookie-invalid',
|
|
355
|
+
reasonKey: 'connection-cookie-invalid',
|
|
356
|
+
message: 'invalid connection base cookie',
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
if (text.includes('ClientNotFound') || text.includes('Client not found')) {
|
|
360
|
+
return {
|
|
361
|
+
reason: 'client-not-found',
|
|
362
|
+
reasonKey: 'client-not-found',
|
|
363
|
+
message: 'client not found',
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
if (text.includes('connection userID mismatch')) {
|
|
367
|
+
return {
|
|
368
|
+
reason: 'connection-userid-mismatch',
|
|
369
|
+
reasonKey: 'connection-userid-mismatch',
|
|
370
|
+
message: 'connection user id mismatch',
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
return undefined
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
export function classifyZeroRecoveryLog(
|
|
377
|
+
level: LogLevel | string,
|
|
378
|
+
args: readonly unknown[],
|
|
379
|
+
nowMs = Date.now(),
|
|
380
|
+
): ZeroRecoveryLogClassification | undefined {
|
|
381
|
+
if (level !== 'error') return undefined
|
|
382
|
+
const text = args.map(logArgText).join(' ')
|
|
383
|
+
if (text.includes(LOCAL_STORE_LOST)) {
|
|
384
|
+
return {
|
|
385
|
+
reason: 'indexeddb-not-found',
|
|
386
|
+
reasonKey: 'local-store',
|
|
387
|
+
message: 'local store lost',
|
|
388
|
+
dropLocalState: true,
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
if (text.includes(SQLITE_ERROR_NAME) && text.includes(SQLITE_STATEMENT_FINALIZED)) {
|
|
392
|
+
return {
|
|
393
|
+
reason: 'sqlite-statement-finalized',
|
|
394
|
+
reasonKey: 'sqlite-statement-finalized',
|
|
395
|
+
message: 'sqlite statement finalized',
|
|
396
|
+
dropLocalState: true,
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
const desync = classifyMutationDesync(text)
|
|
400
|
+
if (desync) {
|
|
401
|
+
return { ...desync, dropLocalState: true }
|
|
402
|
+
}
|
|
403
|
+
if (text.includes(STORE_CLOSED) && !isBenignStoreClosedLog(text)) {
|
|
404
|
+
const prevMs = lastStoreClosedAtMs
|
|
405
|
+
lastStoreClosedAtMs = nowMs
|
|
406
|
+
if (
|
|
407
|
+
prevMs > 0 &&
|
|
408
|
+
nowMs - prevMs >= STORE_CLOSED_REPEAT_MIN_MS &&
|
|
409
|
+
nowMs - prevMs <= STORE_CLOSED_REPEAT_MAX_MS
|
|
410
|
+
) {
|
|
411
|
+
return {
|
|
412
|
+
reason: 'store-closed-repeat',
|
|
413
|
+
reasonKey: 'store-closed-repeat',
|
|
414
|
+
message: 'local store closed repeatedly',
|
|
415
|
+
dropLocalState: true,
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
return undefined
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// watch error-level logs for the local-store-lost / desync signatures (the ones
|
|
423
|
+
// Zero surfaces only through the log, not the structured callbacks) and trigger
|
|
424
|
+
// the same recovery. forwards to the consumer sink, or to the console when there
|
|
425
|
+
// is none. wire this ONLY when the consumer didn't pass their own logSink, so a
|
|
144
426
|
// consumer that owns log-based recovery isn't double-fired.
|
|
145
427
|
export function composeRecoveryLogSink(
|
|
146
428
|
deps: ZeroRecoveryDeps,
|
|
@@ -151,19 +433,13 @@ export function composeRecoveryLogSink(
|
|
|
151
433
|
log(level: LogLevel, context: Context | undefined, ...args: unknown[]): void {
|
|
152
434
|
if (consumerLogSink) consumerLogSink.log(level, context, ...args)
|
|
153
435
|
else logToConsole(level, context, ...args)
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
: arg && typeof arg === 'object' && 'message' in arg
|
|
160
|
-
? String((arg as { message: unknown }).message)
|
|
161
|
-
: '',
|
|
162
|
-
)
|
|
163
|
-
.join(' ')
|
|
164
|
-
if (text.includes(LOCAL_STORE_LOST)) {
|
|
165
|
-
recover(deps, 'local-store', 'local store lost', true)
|
|
436
|
+
const recovery = classifyZeroRecoveryLog(level, args)
|
|
437
|
+
if (!recovery) return
|
|
438
|
+
if (deps.benignLogFilter) {
|
|
439
|
+
const text = args.map(logArgText).join(' ')
|
|
440
|
+
if (deps.benignLogFilter(text)) return
|
|
166
441
|
}
|
|
442
|
+
recover(deps, recovery.reasonKey, recovery.message, recovery.dropLocalState)
|
|
167
443
|
},
|
|
168
444
|
// call through the consumer sink so a class-based sink keeps its `this`,
|
|
169
445
|
// rather than handing Zero a detached method reference.
|
|
@@ -171,9 +447,30 @@ export function composeRecoveryLogSink(
|
|
|
171
447
|
}
|
|
172
448
|
}
|
|
173
449
|
|
|
174
|
-
//
|
|
175
|
-
//
|
|
450
|
+
// generic Zero stale-poke / stale-cookie signatures: the client's view is behind
|
|
451
|
+
// the server's snapshot cookie, which a plain reconnect resolves — this is not a
|
|
452
|
+
// fatal store-loss, so ConnectionMonitor reconnects instead of recovering.
|
|
453
|
+
export function isRecoverableZeroStalePokeMessage(message: string): boolean {
|
|
454
|
+
return (
|
|
455
|
+
message.includes('Server returned unexpected base cookie during sync') ||
|
|
456
|
+
(message.includes('Received cookie') &&
|
|
457
|
+
message.includes('is < than last snapshot cookie') &&
|
|
458
|
+
message.includes('ignoring client view'))
|
|
459
|
+
)
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// test-only: the reload latch + pending deletes + in-memory guard are in-memory
|
|
463
|
+
// (a real page reload clears them); tests simulate that reset between successive
|
|
464
|
+
// "page loads". the injectable/sessionStorage guard is NOT cleared here — like a
|
|
465
|
+
// real reload, it survives, which is what catches an immediate re-fire.
|
|
176
466
|
export function resetRecoveryStateForTests() {
|
|
177
467
|
reloadScheduled = false
|
|
468
|
+
reloadInProgress = false
|
|
469
|
+
if (reloadLatchTimer) {
|
|
470
|
+
clearTimeout(reloadLatchTimer)
|
|
471
|
+
reloadLatchTimer = undefined
|
|
472
|
+
}
|
|
178
473
|
pendingDeletes.length = 0
|
|
474
|
+
lastStoreClosedAtMs = 0
|
|
475
|
+
inMemoryGuard.clear()
|
|
179
476
|
}
|
package/src/index.ts
CHANGED
|
@@ -42,7 +42,13 @@ export {
|
|
|
42
42
|
type ZeroClientErrorInfo,
|
|
43
43
|
} from './helpers/showZeroClientError'
|
|
44
44
|
export {
|
|
45
|
+
classifyZeroRecoveryLog,
|
|
45
46
|
makeZeroRecovery,
|
|
46
47
|
composeRecoveryLogSink,
|
|
48
|
+
isRecoverableZeroStalePokeMessage,
|
|
49
|
+
type ZeroRecoveryLogClassification,
|
|
50
|
+
type ZeroRecoveryLogReason,
|
|
47
51
|
type ZeroRecoveryDeps,
|
|
52
|
+
type ScheduleReloadContext,
|
|
53
|
+
type RecoveryGuardStorage,
|
|
48
54
|
} from './helpers/recoverZeroClient'
|
package/src/multi.ts
CHANGED
|
@@ -22,3 +22,36 @@ export function createZeroClientWithDirectQueries<
|
|
|
22
22
|
createDirectUseQuery,
|
|
23
23
|
})
|
|
24
24
|
}
|
|
25
|
+
|
|
26
|
+
// fail loud when a generated namespace drifts out of a multi-instance partition.
|
|
27
|
+
// multi-instance consumers (control + project, control + server, …) hand-split
|
|
28
|
+
// the generated query/model namespaces across instances; a namespace that ends
|
|
29
|
+
// up in NEITHER partition silently un-registers its queries (its useQuery throws
|
|
30
|
+
// "query not registered" at runtime, error-boundarying a whole screen — the soot
|
|
31
|
+
// `planGrant` regression). run this at module-eval over each partitioned group so
|
|
32
|
+
// the drift is a build/boot throw instead. also catches a namespace listed in
|
|
33
|
+
// MORE than one partition (an ambiguous owner), which the hand-rolled loops did
|
|
34
|
+
// not. `partitions` maps a label (e.g. 'control'/'project') to that instance's
|
|
35
|
+
// namespace table; every key in `entries` must appear in exactly one.
|
|
36
|
+
export function assertZeroInstancePartition(
|
|
37
|
+
kind: string,
|
|
38
|
+
entries: Record<string, unknown>,
|
|
39
|
+
partitions: Record<string, Record<string, unknown>>,
|
|
40
|
+
): void {
|
|
41
|
+
const labels = Object.keys(partitions)
|
|
42
|
+
for (const name of Object.keys(entries)) {
|
|
43
|
+
const owners = labels.filter((label) => name in partitions[label])
|
|
44
|
+
if (owners.length === 0) {
|
|
45
|
+
throw new Error(
|
|
46
|
+
`[on-zero] generated ${kind} "${name}" is missing from the instance partition ` +
|
|
47
|
+
`(${labels.join(' / ')}) — add it to exactly one partition table.`,
|
|
48
|
+
)
|
|
49
|
+
}
|
|
50
|
+
if (owners.length > 1) {
|
|
51
|
+
throw new Error(
|
|
52
|
+
`[on-zero] generated ${kind} "${name}" is claimed by more than one instance partition ` +
|
|
53
|
+
`(${owners.join(', ')}) — it must belong to exactly one.`,
|
|
54
|
+
)
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { describe, expect, test } from 'vitest'
|
|
2
|
+
|
|
3
|
+
import { assertZeroInstancePartition } from './multi'
|
|
4
|
+
|
|
5
|
+
describe('assertZeroInstancePartition', () => {
|
|
6
|
+
const control = { user: 1, workspace: 1 }
|
|
7
|
+
const project = { message: 1, thread: 1 }
|
|
8
|
+
|
|
9
|
+
test('passes when every namespace belongs to exactly one partition', () => {
|
|
10
|
+
expect(() =>
|
|
11
|
+
assertZeroInstancePartition(
|
|
12
|
+
'query namespace',
|
|
13
|
+
{ user: {}, workspace: {}, message: {}, thread: {} },
|
|
14
|
+
{ control, project },
|
|
15
|
+
),
|
|
16
|
+
).not.toThrow()
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
test('throws on a namespace missing from every partition (the planGrant drift)', () => {
|
|
20
|
+
expect(() =>
|
|
21
|
+
assertZeroInstancePartition(
|
|
22
|
+
'query namespace',
|
|
23
|
+
{ user: {}, planGrant: {} },
|
|
24
|
+
{ control, project },
|
|
25
|
+
),
|
|
26
|
+
).toThrow(/planGrant.*missing from the instance partition/)
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
test('throws on a namespace claimed by more than one partition', () => {
|
|
30
|
+
expect(() =>
|
|
31
|
+
assertZeroInstancePartition(
|
|
32
|
+
'query namespace',
|
|
33
|
+
{ user: {} },
|
|
34
|
+
{ control, project: { user: 1 } },
|
|
35
|
+
),
|
|
36
|
+
).toThrow(/user.*more than one instance partition/)
|
|
37
|
+
})
|
|
38
|
+
})
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createZeroClient.connection.test.d.ts","sourceRoot":"","sources":["../src/createZeroClient.connection.test.tsx"],"names":[],"mappings":"AAuDA,OAAO,CAAC,MAAM,CAAC;IAEb,IAAI,wBAAwB,EAAE,OAAO,GAAG,SAAS,CAAA;CAClD"}
|
|
@@ -2,6 +2,7 @@ import { Zero as ZeroClient } from '@rocicorp/zero';
|
|
|
2
2
|
import { type Emitter } from '@take-out/helpers';
|
|
3
3
|
import { type Context, type ReactNode } from 'react';
|
|
4
4
|
import { type QueryControlMode, type UseQueryHook } from './createUseQuery';
|
|
5
|
+
import { type RecoveryGuardStorage, type ScheduleReloadContext } from './helpers/recoverZeroClient';
|
|
5
6
|
import { resolveQuery, type PlainQueryFn } from './resolveQuery';
|
|
6
7
|
import type { AuthData, GenericModels, GetZeroMutators, ZeroEvent } from './types';
|
|
7
8
|
import type { AnyQueryRegistry, Query, Row, ZeroOptions, Schema as ZeroSchema } from '@rocicorp/zero';
|
|
@@ -33,6 +34,11 @@ export declare function createZeroClient<Schema extends ZeroSchema, Models exten
|
|
|
33
34
|
transport?: "http-pull";
|
|
34
35
|
pullIntervalMs?: number;
|
|
35
36
|
beforeReload?: (() => Promise<void>) | undefined;
|
|
37
|
+
scheduleReload?: ((ctx: ScheduleReloadContext) => void) | undefined;
|
|
38
|
+
guardStorage?: RecoveryGuardStorage;
|
|
39
|
+
benignLogFilter?: ((message: string) => boolean) | undefined;
|
|
40
|
+
refreshAuth?: (() => Promise<string | undefined>) | undefined;
|
|
41
|
+
connectionDataset?: boolean;
|
|
36
42
|
}) => import("react/jsx-runtime").JSX.Element;
|
|
37
43
|
ControlQueries: ({ children, action, whenDisabled, }: {
|
|
38
44
|
children: ReactNode;
|
|
@@ -58,6 +64,9 @@ export declare function createZeroClient<Schema extends ZeroSchema, Models exten
|
|
|
58
64
|
<TArg, TTable extends keyof Schema["tables"] & string, TReturn>(fn: PlainQueryFn<TArg, Query<TTable, Schema, TReturn>>, params: TArg): ReturnType<typeof resolveQuery<Schema>>;
|
|
59
65
|
<TTable extends keyof Schema["tables"] & string, TReturn>(fn: PlainQueryFn<void, Query<TTable, Schema, TReturn>>): ReturnType<typeof resolveQuery<Schema>>;
|
|
60
66
|
};
|
|
67
|
+
remint: (opts?: {
|
|
68
|
+
dropLocalState?: boolean;
|
|
69
|
+
}) => Promise<boolean>;
|
|
61
70
|
};
|
|
62
71
|
export declare function createZeroClientInternal<Schema extends ZeroSchema, Models extends GenericModels>({ schema, models, groupedQueries, permissionStrategy, instanceName, createDirectUseQuery, }: CreateZeroClientOptions<Schema, Models> & {
|
|
63
72
|
createDirectUseQuery?: DirectQueryAdapter<Schema>;
|
|
@@ -71,6 +80,11 @@ export declare function createZeroClientInternal<Schema extends ZeroSchema, Mode
|
|
|
71
80
|
transport?: "http-pull";
|
|
72
81
|
pullIntervalMs?: number;
|
|
73
82
|
beforeReload?: () => Promise<void>;
|
|
83
|
+
scheduleReload?: (ctx: ScheduleReloadContext) => void;
|
|
84
|
+
guardStorage?: RecoveryGuardStorage;
|
|
85
|
+
benignLogFilter?: (message: string) => boolean;
|
|
86
|
+
refreshAuth?: () => Promise<string | undefined>;
|
|
87
|
+
connectionDataset?: boolean;
|
|
74
88
|
}) => import("react/jsx-runtime").JSX.Element;
|
|
75
89
|
ControlQueries: ({ children, action, whenDisabled, }: {
|
|
76
90
|
children: ReactNode;
|
|
@@ -96,6 +110,9 @@ export declare function createZeroClientInternal<Schema extends ZeroSchema, Mode
|
|
|
96
110
|
<TArg, TTable extends keyof Schema["tables"] & string, TReturn>(fn: PlainQueryFn<TArg, Query<TTable, Schema, TReturn>>, params: TArg): ReturnType<typeof resolveQuery<Schema>>;
|
|
97
111
|
<TTable extends keyof Schema["tables"] & string, TReturn>(fn: PlainQueryFn<void, Query<TTable, Schema, TReturn>>): ReturnType<typeof resolveQuery<Schema>>;
|
|
98
112
|
};
|
|
113
|
+
remint: (opts?: {
|
|
114
|
+
dropLocalState?: boolean;
|
|
115
|
+
}) => Promise<boolean>;
|
|
99
116
|
};
|
|
100
117
|
export {};
|
|
101
118
|
//# sourceMappingURL=createZeroClient.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createZeroClient.d.ts","sourceRoot":"","sources":["../src/createZeroClient.tsx"],"names":[],"mappings":"AAAA,OAAO,EAA8B,IAAI,IAAI,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAO/E,OAAO,EAAoC,KAAK,OAAO,EAAE,MAAM,mBAAmB,CAAA;AAClF,OAAO,EAQL,KAAK,OAAO,EACZ,KAAK,SAAS,EACf,MAAM,OAAO,CAAA;AAId,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,YAAY,EAClB,MAAM,kBAAkB,CAAA;
|
|
1
|
+
{"version":3,"file":"createZeroClient.d.ts","sourceRoot":"","sources":["../src/createZeroClient.tsx"],"names":[],"mappings":"AAAA,OAAO,EAA8B,IAAI,IAAI,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAO/E,OAAO,EAAoC,KAAK,OAAO,EAAE,MAAM,mBAAmB,CAAA;AAClF,OAAO,EAQL,KAAK,OAAO,EACZ,KAAK,SAAS,EACf,MAAM,OAAO,CAAA;AAId,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,YAAY,EAClB,MAAM,kBAAkB,CAAA;AAGzB,OAAO,EAIL,KAAK,oBAAoB,EACzB,KAAK,qBAAqB,EAE3B,MAAM,6BAA6B,CAAA;AAIpC,OAAO,EAAE,YAAY,EAAE,KAAK,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAOhE,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AAClF,OAAO,KAAK,EACV,gBAAgB,EAChB,KAAK,EACL,GAAG,EAEH,WAAW,EACX,MAAM,IAAI,UAAU,EACrB,MAAM,gBAAgB,CAAA;AAEvB,KAAK,cAAc,GAAG;IAAE,GAAG,CAAC,EAAE,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,CAAA;CAAE,CAAA;AAEvE,MAAM,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;AAMpF,MAAM,MAAM,kBAAkB,GAAG,YAAY,GAAG,iBAAiB,GAAG,kBAAkB,CAAA;AAEtF,MAAM,MAAM,uBAAuB,CACjC,MAAM,SAAS,UAAU,EACzB,MAAM,SAAS,aAAa,IAC1B;IACF,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,cAAc,EAAE,cAAc,CAAA;IAC9B,kBAAkB,CAAC,EAAE,kBAAkB,CAAA;IAIvC,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB,CAAA;AAED,MAAM,MAAM,kBAAkB,CAAC,MAAM,SAAS,UAAU,IAAI,CAAC,KAAK,EAAE;IAClE,eAAe,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAA;IAC1C,aAAa,EAAE,gBAAgB,CAAA;IAC/B,OAAO,EAAE,MAAM,GAAG,CAAA;IAClB,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;CAC7B,KAAK,YAAY,CAAC,MAAM,CAAC,CAAA;AA+B1B,wBAAgB,gBAAgB,CAAC,MAAM,SAAS,UAAU,EAAE,MAAM,SAAS,aAAa,EACtF,OAAO,EAAE,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC;;;;kBA+WpC,SAAS;;kBAUT,OAAO;oBAIL,WAAW;yBAGN,MAAM;8BAGF,OAAO,CAAC,IAAI,CAAC;gCAIX,qBAAqB,KAAK,IAAI;uBAGtC,oBAAoB;qCAGP,MAAM,KAAK,OAAO;6BAG1B,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;4BAI3B,OAAO;;;kBAgYjB,SAAS;iBACV,QAAQ,GAAG,SAAS;uBACd,OAAO,GAAG,YAAY;;;;uFAjlBxB,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,YACrC,OAAO,UACT,OAAO,KACZ,OAAO,GAAG,IAAI;6FAHR,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,YACrC,OAAO,UACT,OAAO,KACZ,OAAO,GAAG,IAAI;;;SAqiBN,IAAI,EAAE,MAAM,0CAA0C,OAAO,kFAGlE,cAAc;2BACN,IAAI;sBAAY,OAAO,CAAC,IAAI,CAAC;;SAChC,MAAM,0CAA0C,OAAO,oEAE5D,cAAc;2BACN,IAAI;sBAAY,OAAO,CAAC,IAAI,CAAC;;;;SAe/B,IAAI,EAAE,MAAM,0CAA0C,OAAO,yEAG5E,UAAU,CAAC,OAAO,YAAY,QAAQ,CAAC;SACxB,MAAM,0CAA0C,OAAO,2DAEtE,UAAU,CAAC,OAAO,YAAY,QAAQ,CAAC;;;yBAldK,OAAO;UAAU,OAAO,CAAC,OAAO,CAAC;EArTjF;AAED,wBAAgB,wBAAwB,CACtC,MAAM,SAAS,UAAU,EACzB,MAAM,SAAS,aAAa,EAC5B,EACA,MAAM,EACN,MAAM,EACN,cAAc,EACd,kBAAiC,EACjC,YAAwB,EACxB,oBAAoB,GACrB,EAAE,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG;IAC3C,oBAAoB,CAAC,EAAE,kBAAkB,CAAC,MAAM,CAAC,CAAA;CAClD;;;;kBA8Va,SAAS;mBACR,QAAQ,GAAG,IAAI;kBAShB,OAAO;oBAIL,WAAW;yBAGN,MAAM;uBAGR,MAAM,OAAO,CAAC,IAAI,CAAC;yBAIjB,CAAC,GAAG,EAAE,qBAAqB,KAAK,IAAI;uBAGtC,oBAAoB;0BAGjB,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO;sBAGhC,MAAM,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;4BAI3B,OAAO;;0DA+X1B;QACD,QAAQ,EAAE,SAAS,CAAA;QACnB,MAAM,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAA;QAC7B,YAAY,CAAC,EAAE,OAAO,GAAG,YAAY,CAAA;KACtC;;;2BAnlBY,oCAAY,CAAC,MAAM,GAAG,EAAE,CAAC,WACvB,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,YACrC,OAAO,UACT,OAAO,KACZ,OAAO,GAAG,IAAI;iCAJV,oCAAY,CAAC,MAAM,GAAG,EAAE,CAAC,WACvB,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,YACrC,OAAO,UACT,OAAO,KACZ,OAAO,GAAG,IAAI;;;SAqiBN,IAAI,EAAE,MAAM,SAAS,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE,OAAO,MACxE,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,UAC9C,IAAI,YACF,cAAc,GACvB;YAAE,OAAO,EAAE,MAAM,IAAI,CAAC;YAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;SAAE;SAClC,MAAM,SAAS,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE,OAAO,MAClE,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,YAC5C,cAAc,GACvB;YAAE,OAAO,EAAE,MAAM,IAAI,CAAC;YAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;SAAE;;;SAejC,IAAI,EAAE,MAAM,SAAS,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE,OAAO,MACzE,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,UAC9C,IAAI,GACX,UAAU,CAAC,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;SACxB,MAAM,SAAS,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE,OAAO,MACnE,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,GACrD,UAAU,CAAC,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;;oBAldd;QAAE,cAAc,CAAC,EAAE,OAAO,CAAA;KAAE,KAAQ,OAAO,CAAC,OAAO,CAAC;EAkfjF"}
|