@filen/sync 0.2.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.node-version +1 -1
- package/dist/ignorer.d.ts +6 -0
- package/dist/ignorer.js +43 -24
- package/dist/ignorer.js.map +1 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/deltas.d.ts +58 -2
- package/dist/lib/deltas.js +693 -108
- package/dist/lib/deltas.js.map +1 -1
- package/dist/lib/environment.d.ts +47 -0
- package/dist/lib/environment.js +71 -0
- package/dist/lib/environment.js.map +1 -0
- package/dist/lib/filesystems/dirTree.d.ts +70 -0
- package/dist/lib/filesystems/dirTree.js +157 -0
- package/dist/lib/filesystems/dirTree.js.map +1 -0
- package/dist/lib/filesystems/local.d.ts +18 -8
- package/dist/lib/filesystems/local.js +166 -160
- package/dist/lib/filesystems/local.js.map +1 -1
- package/dist/lib/filesystems/remote.d.ts +12 -5
- package/dist/lib/filesystems/remote.js +226 -172
- package/dist/lib/filesystems/remote.js.map +1 -1
- package/dist/lib/ipc.js +1 -2
- package/dist/lib/ipc.js.map +1 -1
- package/dist/lib/lock.js +19 -12
- package/dist/lib/lock.js.map +1 -1
- package/dist/lib/logger.js +9 -7
- package/dist/lib/logger.js.map +1 -1
- package/dist/lib/state.js +159 -63
- package/dist/lib/state.js.map +1 -1
- package/dist/lib/sync.d.ts +18 -0
- package/dist/lib/sync.js +165 -96
- package/dist/lib/sync.js.map +1 -1
- package/dist/lib/tasks.d.ts +7 -8
- package/dist/lib/tasks.js +38 -45
- package/dist/lib/tasks.js.map +1 -1
- package/dist/semaphore.d.ts +1 -0
- package/dist/semaphore.js +22 -5
- package/dist/semaphore.js.map +1 -1
- package/dist/utils.js +51 -35
- package/dist/utils.js.map +1 -1
- package/eslint.config.mjs +36 -0
- package/package.json +19 -15
- package/tests/bench/collapse.bench.ts +114 -0
- package/tests/bench/cycle.bench.ts +111 -0
- package/tests/bench/deltas.bench.ts +151 -0
- package/tests/bench/harness/fake-sync.ts +32 -0
- package/tests/bench/harness/measure.ts +276 -0
- package/tests/bench/harness/scale-world.ts +160 -0
- package/tests/bench/harness/trees.ts +275 -0
- package/tests/bench/local-scan.bench.ts +74 -0
- package/tests/bench/longrun.bench.ts +130 -0
- package/tests/bench/profile-incremental.ts +90 -0
- package/tests/bench/remote-build.bench.ts +104 -0
- package/tests/bench/render.ts +14 -0
- package/tests/bench/semaphore.bench.ts +79 -0
- package/tests/bench/state.bench.ts +85 -0
- package/tests/bench/tasks-dispatch.bench.ts +156 -0
- package/tests/conformance/virtual-fs.test.ts +213 -0
- package/tests/e2e/backup.e2e.test.ts +130 -0
- package/tests/e2e/confirm.e2e.test.ts +191 -0
- package/tests/e2e/conflict.e2e.test.ts +261 -0
- package/tests/e2e/edge.e2e.test.ts +339 -0
- package/tests/e2e/harness/account.ts +104 -0
- package/tests/e2e/harness/assert.ts +127 -0
- package/tests/e2e/harness/drive.ts +88 -0
- package/tests/e2e/harness/mutations.ts +249 -0
- package/tests/e2e/harness/world.ts +222 -0
- package/tests/e2e/ignore.e2e.test.ts +123 -0
- package/tests/e2e/lifecycle.e2e.test.ts +290 -0
- package/tests/e2e/modes.e2e.test.ts +215 -0
- package/tests/e2e/platform.e2e.test.ts +157 -0
- package/tests/e2e/property.e2e.test.ts +163 -0
- package/tests/e2e/races.e2e.test.ts +90 -0
- package/tests/e2e/regressions.e2e.test.ts +212 -0
- package/tests/e2e/resilience.e2e.test.ts +231 -0
- package/tests/e2e/special.e2e.test.ts +185 -0
- package/tests/e2e/state.e2e.test.ts +229 -0
- package/tests/e2e/sync.e2e.test.ts +222 -0
- package/tests/fakes/fake-cloud.test.ts +267 -0
- package/tests/fakes/fake-cloud.ts +1094 -0
- package/tests/fakes/virtual-fs.ts +354 -0
- package/tests/harness/known-bug.ts +17 -0
- package/tests/harness/mutations.ts +65 -0
- package/tests/harness/runner.ts +141 -0
- package/tests/harness/snapshot.ts +113 -0
- package/tests/harness/world.ts +187 -0
- package/tests/scenarios/a-baseline.test.ts +107 -0
- package/tests/scenarios/aa-races.test.ts +258 -0
- package/tests/scenarios/ab-mode-property.test.ts +189 -0
- package/tests/scenarios/ac-platform.test.ts +320 -0
- package/tests/scenarios/ad-unicode-normalization.test.ts +67 -0
- package/tests/scenarios/b-additions.test.ts +160 -0
- package/tests/scenarios/c-modifications.test.ts +194 -0
- package/tests/scenarios/d-deletions.test.ts +259 -0
- package/tests/scenarios/e-rename-move.test.ts +288 -0
- package/tests/scenarios/f-ignore-filter.test.ts +346 -0
- package/tests/scenarios/g-large-deletion.test.ts +277 -0
- package/tests/scenarios/h-resilience.test.ts +167 -0
- package/tests/scenarios/i-lifecycle.test.ts +353 -0
- package/tests/scenarios/j-state-cache.test.ts +264 -0
- package/tests/scenarios/k-scale.test.ts +202 -0
- package/tests/scenarios/l-property.test.ts +145 -0
- package/tests/scenarios/m-golden.test.ts +452 -0
- package/tests/scenarios/o-task-errors.test.ts +497 -0
- package/tests/scenarios/p-remote-originated.test.ts +306 -0
- package/tests/scenarios/q-cycle-lifecycle.test.ts +234 -0
- package/tests/scenarios/r-rename-stress.test.ts +208 -0
- package/tests/scenarios/s-upgrade-transition.test.ts +171 -0
- package/tests/scenarios/t-type-change.test.ts +144 -0
- package/tests/scenarios/u-mode-local-to-cloud.test.ts +347 -0
- package/tests/scenarios/v-mode-local-backup.test.ts +201 -0
- package/tests/scenarios/w-mode-cloud-to-local.test.ts +304 -0
- package/tests/scenarios/x-mode-cloud-backup.test.ts +201 -0
- package/tests/scenarios/y-conflict-matrix.test.ts +292 -0
- package/tests/scenarios/z-cross-ops.test.ts +285 -0
- package/tests/scenarios/zb-dir-rename-cross.test.ts +296 -0
- package/tests/scenarios/zc-crash-recovery.test.ts +189 -0
- package/tests/scenarios/zd-inode-reuse.test.ts +118 -0
- package/tests/scenarios/ze-move-into-new-dir.test.ts +130 -0
- package/tests/scenarios/zf-remote-change-unchanged-local.test.ts +81 -0
- package/tests/scenarios/zg-edit-during-scan.test.ts +68 -0
- package/tests/scenarios/zh-dir-delete-vs-child.test.ts +104 -0
- package/tests/scenarios/zi-smoke-test-outage.test.ts +78 -0
- package/tests/scenarios/zj-trash-cleanup.test.ts +133 -0
- package/tests/scenarios/zk-ignore-asymmetry.test.ts +150 -0
- package/tests/scenarios/zl-mode-atomicity.test.ts +104 -0
- package/tests/scenarios/zm-scan-concurrency.test.ts +78 -0
- package/tests/scenarios/zn-delta-ordering.test.ts +130 -0
- package/tests/scenarios/zo-download-temp-cleanup.test.ts +65 -0
- package/tests/unit/collapse-deltas.test.ts +276 -0
- package/tests/unit/dir-tree.test.ts +159 -0
- package/tests/unit/icloud.test.ts +115 -0
- package/tests/unit/ignorer-cache-regression.test.ts +70 -0
- package/tests/unit/ignorer.test.ts +63 -0
- package/tests/unit/ipc-lock.test.ts +438 -0
- package/tests/unit/lock.test.ts +135 -0
- package/tests/unit/n-unit.test.ts +632 -0
- package/tests/unit/remote-tree-unordered-regression.test.ts +101 -0
- package/tests/unit/semaphore-regression.test.ts +140 -0
- package/tests/unit/state-refencode-regression.test.ts +224 -0
- package/tests/unit/state.test.ts +809 -0
- package/tests/unit/tasks-dispatch-order-regression.test.ts +53 -0
- package/tests/unit/worker-api.test.ts +379 -0
- package/tsconfig.json +10 -1
- package/tsconfig.test.json +12 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/vitest.bench.config.ts +32 -0
- package/vitest.config.ts +27 -0
- package/vitest.e2e.config.ts +68 -0
- package/.eslintrc +0 -16
- package/jest.config.js +0 -5
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from "vitest"
|
|
2
|
+
import pathModule from "path"
|
|
3
|
+
import { PauseSignal } from "@filen/sdk"
|
|
4
|
+
import { SYNC_INTERVAL } from "../../src/constants"
|
|
5
|
+
import SyncWorker from "../../src/index"
|
|
6
|
+
import { type SyncMessage } from "../../src/types"
|
|
7
|
+
import { type SyncEnvironment } from "../../src/lib/environment"
|
|
8
|
+
import { DEVICE_ID_VERSION } from "../../src/lib/filesystems/remote"
|
|
9
|
+
import { createWorld, BASE_TIME, DB_ROOT, type CreateWorldOptions, type World } from "../harness/world"
|
|
10
|
+
import { snapshotLocal, snapshotRemote, messagesOfType, countMessages, transferKinds, allOps } from "../harness/snapshot"
|
|
11
|
+
import { writeLocal, rmLocal } from "../harness/mutations"
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Category I — lifecycle / control / transfers (behavioral spec §I, §8). These exercise the
|
|
15
|
+
* SyncWorker control surface that drives a pair's lifecycle: `runOnce` (one cycle then cleanup),
|
|
16
|
+
* `updatePaused` (short-circuit / resume), `updateRemoved` (abort transfers + delete db files),
|
|
17
|
+
* the per-transfer `pauseTransfer`/`resumeTransfer`/`stopTransfer` signals keyed `${type}:${path}`,
|
|
18
|
+
* `updateMode` (mode change takes effect next cycle), and `updateSyncPairs` (runtime registration).
|
|
19
|
+
*
|
|
20
|
+
* Like Category G these cycles are driven MANUALLY (not via runScenario): fake timers are installed,
|
|
21
|
+
* the clock is pinned to BASE_TIME, and {@link plainCycle} pumps the sync interval before each
|
|
22
|
+
* {@link Sync.runCycle}. This gives the deterministic control the lifecycle assertions need.
|
|
23
|
+
*/
|
|
24
|
+
const FAKE_TIMERS = ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"] as const
|
|
25
|
+
|
|
26
|
+
async function withWorld(options: CreateWorldOptions, body: (world: World) => Promise<void>): Promise<void> {
|
|
27
|
+
vi.useFakeTimers({ toFake: [...FAKE_TIMERS] })
|
|
28
|
+
vi.setSystemTime(BASE_TIME)
|
|
29
|
+
|
|
30
|
+
try {
|
|
31
|
+
const world = await createWorld(options)
|
|
32
|
+
|
|
33
|
+
await body(world)
|
|
34
|
+
} finally {
|
|
35
|
+
vi.useRealTimers()
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Drive exactly one cycle (advance the interval so the local-change wait passes, then run). */
|
|
40
|
+
async function plainCycle(world: World): Promise<void> {
|
|
41
|
+
await vi.advanceTimersByTimeAsync(SYNC_INTERVAL + 1)
|
|
42
|
+
|
|
43
|
+
await world.sync.runCycle()
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** The messages appended to the world's stream while `body` runs (a per-cycle slice). */
|
|
47
|
+
async function capture(world: World, body: () => Promise<void>): Promise<SyncMessage[]> {
|
|
48
|
+
const mark = world.messages.length
|
|
49
|
+
|
|
50
|
+
await body()
|
|
51
|
+
|
|
52
|
+
return world.messages.slice(mark)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
describe("Category I — lifecycle / control / transfers", () => {
|
|
56
|
+
it("I1: a runOnce worker performs a single cycle, syncs, then cleans up and exits without rescheduling", async () => {
|
|
57
|
+
await withWorld(
|
|
58
|
+
{
|
|
59
|
+
mode: "twoWay",
|
|
60
|
+
initialLocal: { "/local/a.txt": "hello" }
|
|
61
|
+
},
|
|
62
|
+
async world => {
|
|
63
|
+
// A runOnce pair is a WORKER-level constructor option, so build a second worker over the
|
|
64
|
+
// same virtual fs + cloud + pair. Its messages land on a private stream (process.onMessage
|
|
65
|
+
// is replaced when the worker is constructed), so assert against that stream directly.
|
|
66
|
+
const messages: SyncMessage[] = []
|
|
67
|
+
const environment: SyncEnvironment = {
|
|
68
|
+
fs: world.vfs.fs,
|
|
69
|
+
globFs: world.vfs.globFs,
|
|
70
|
+
writeFileAtomic: async (filename, data, writeOptions): Promise<void> => {
|
|
71
|
+
await world.vfs.fs.writeFile(filename, data, writeOptions)
|
|
72
|
+
},
|
|
73
|
+
createWatcher: async (_path, _onChange) => ({
|
|
74
|
+
close: async (): Promise<void> => {}
|
|
75
|
+
}),
|
|
76
|
+
fetchDirTree: (_sdk, request) => world.cloud.sdk.api(3).dir().tree(request)
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const worker = new SyncWorker({
|
|
80
|
+
syncPairs: [world.syncPair],
|
|
81
|
+
dbPath: DB_ROOT,
|
|
82
|
+
sdk: world.cloud.sdk,
|
|
83
|
+
onMessage: message => {
|
|
84
|
+
messages.push(message)
|
|
85
|
+
},
|
|
86
|
+
disableLogging: true,
|
|
87
|
+
environment,
|
|
88
|
+
runOnce: true
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
// initialize() floats the self-driving run() loop; for runOnce that loop runs one cycle
|
|
92
|
+
// then calls cleanup() (which emits cycleExited) instead of rescheduling. Pump until the
|
|
93
|
+
// terminal message arrives — the cycle never blocks, so this settles in a few iterations.
|
|
94
|
+
await worker.initialize()
|
|
95
|
+
|
|
96
|
+
for (let tick = 0; tick < 200 && !messages.some(message => message.type === "cycleExited"); tick++) {
|
|
97
|
+
await vi.advanceTimersByTimeAsync(100)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// The one cycle ran: the pending local file was uploaded and the worlds converged.
|
|
101
|
+
expect(transferKinds(messages)).toContain("upload")
|
|
102
|
+
expect(snapshotRemote(world)["/a.txt"]).toMatchObject({ type: "file" })
|
|
103
|
+
expect(snapshotLocal(world)).toEqual(snapshotRemote(world))
|
|
104
|
+
|
|
105
|
+
// Terminal state: it cleaned up (cycleExited) and did NOT schedule another cycle.
|
|
106
|
+
expect(messages.some(message => message.type === "cycleExited")).toBe(true)
|
|
107
|
+
expect(countMessages(messages, "cycleRestarting")).toBe(0)
|
|
108
|
+
expect(worker.syncs[world.syncPair.uuid]!.removed).toBe(true)
|
|
109
|
+
}
|
|
110
|
+
)
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
it("I2: a paused pair short-circuits its cycle and resuming lets the pending change sync", async () => {
|
|
114
|
+
await withWorld(
|
|
115
|
+
{
|
|
116
|
+
mode: "twoWay",
|
|
117
|
+
paused: true
|
|
118
|
+
},
|
|
119
|
+
async world => {
|
|
120
|
+
const uuid = world.syncPair.uuid
|
|
121
|
+
|
|
122
|
+
// A change is pending while the pair is paused.
|
|
123
|
+
writeLocal(world, "a.txt", "v1")
|
|
124
|
+
world.triggerWatcher()
|
|
125
|
+
|
|
126
|
+
const pausedCycle = await capture(world, () => plainCycle(world))
|
|
127
|
+
|
|
128
|
+
// The cycle short-circuits before doing any work: cyclePaused is emitted, nothing starts.
|
|
129
|
+
expect(messagesOfType(pausedCycle, "cyclePaused").length).toBeGreaterThan(0)
|
|
130
|
+
expect(countMessages(pausedCycle, "cycleStarted")).toBe(0)
|
|
131
|
+
expect(allOps(pausedCycle)).toEqual([])
|
|
132
|
+
// No op happened: the pending addition was NOT uploaded.
|
|
133
|
+
expect(snapshotRemote(world)).toEqual({})
|
|
134
|
+
|
|
135
|
+
// Resume via the worker control method, then run a cycle.
|
|
136
|
+
world.worker.updatePaused(uuid, false)
|
|
137
|
+
world.triggerWatcher()
|
|
138
|
+
|
|
139
|
+
const resumedCycle = await capture(world, () => plainCycle(world))
|
|
140
|
+
|
|
141
|
+
// Now the pending change syncs and the worlds converge.
|
|
142
|
+
expect(messagesOfType(resumedCycle, "cyclePaused").length).toBe(0)
|
|
143
|
+
expect(transferKinds(resumedCycle)).toContain("upload")
|
|
144
|
+
expect(snapshotRemote(world)["/a.txt"]).toMatchObject({ type: "file" })
|
|
145
|
+
expect(snapshotLocal(world)).toEqual(snapshotRemote(world))
|
|
146
|
+
}
|
|
147
|
+
)
|
|
148
|
+
})
|
|
149
|
+
|
|
150
|
+
it("I3: removing a pair aborts its in-flight transfers, deletes its db files, and exits", async () => {
|
|
151
|
+
await withWorld(
|
|
152
|
+
{
|
|
153
|
+
mode: "twoWay",
|
|
154
|
+
initialLocal: { "/local/a.txt": "a" }
|
|
155
|
+
},
|
|
156
|
+
async world => {
|
|
157
|
+
const uuid = world.syncPair.uuid
|
|
158
|
+
|
|
159
|
+
// First cycle uploads and persists state + a deviceId, so there are db files to delete.
|
|
160
|
+
await plainCycle(world)
|
|
161
|
+
|
|
162
|
+
const stateFiles = [
|
|
163
|
+
world.sync.state.previousLocalTreePath,
|
|
164
|
+
world.sync.state.previousRemoteTreePath,
|
|
165
|
+
world.sync.state.localFileHashesPath
|
|
166
|
+
]
|
|
167
|
+
const deviceIdFile = pathModule.posix.join(DB_ROOT, "deviceId", `v${DEVICE_ID_VERSION}`, uuid)
|
|
168
|
+
|
|
169
|
+
for (const file of stateFiles) {
|
|
170
|
+
expect(world.vfs.controls.exists(file)).toBe(true)
|
|
171
|
+
}
|
|
172
|
+
expect(world.vfs.controls.exists(deviceIdFile)).toBe(true)
|
|
173
|
+
|
|
174
|
+
// Stand in for an in-flight transfer: a registered, not-yet-aborted controller.
|
|
175
|
+
world.sync.abortControllers["upload:/a.txt"] = new AbortController()
|
|
176
|
+
|
|
177
|
+
const removalMessages = await capture(world, () => world.worker.updateRemoved(uuid, true))
|
|
178
|
+
|
|
179
|
+
// The transfer was aborted, the pair is marked removed, and it exited (cleanup -> cycleExited).
|
|
180
|
+
expect(world.sync.abortControllers["upload:/a.txt"]!.signal.aborted).toBe(true)
|
|
181
|
+
expect(world.sync.removed).toBe(true)
|
|
182
|
+
expect(messagesOfType(removalMessages, "cycleExited").length).toBeGreaterThan(0)
|
|
183
|
+
|
|
184
|
+
// The persisted state + deviceId files for this pair are gone.
|
|
185
|
+
for (const file of stateFiles) {
|
|
186
|
+
expect(world.vfs.controls.exists(file)).toBe(false)
|
|
187
|
+
}
|
|
188
|
+
expect(world.vfs.controls.exists(deviceIdFile)).toBe(false)
|
|
189
|
+
}
|
|
190
|
+
)
|
|
191
|
+
})
|
|
192
|
+
|
|
193
|
+
it("I4: pauseTransfer and resumeTransfer toggle a specific transfer signal and the pair still syncs", async () => {
|
|
194
|
+
await withWorld(
|
|
195
|
+
{
|
|
196
|
+
mode: "twoWay"
|
|
197
|
+
},
|
|
198
|
+
async world => {
|
|
199
|
+
const uuid = world.syncPair.uuid
|
|
200
|
+
|
|
201
|
+
writeLocal(world, "a.txt", "v1")
|
|
202
|
+
world.triggerWatcher()
|
|
203
|
+
|
|
204
|
+
// Stand in for an in-flight upload: a registered pause signal keyed `${type}:${path}`.
|
|
205
|
+
const signalKey = "upload:/a.txt"
|
|
206
|
+
world.sync.pauseSignals[signalKey] = new PauseSignal()
|
|
207
|
+
|
|
208
|
+
// pauseTransfer / resumeTransfer route by the same key and flip the signal state.
|
|
209
|
+
world.worker.pauseTransfer(uuid, "upload", "/a.txt")
|
|
210
|
+
expect(world.sync.pauseSignals[signalKey]!.isPaused()).toBe(true)
|
|
211
|
+
|
|
212
|
+
world.worker.resumeTransfer(uuid, "upload", "/a.txt")
|
|
213
|
+
expect(world.sync.pauseSignals[signalKey]!.isPaused()).toBe(false)
|
|
214
|
+
|
|
215
|
+
// An unknown key is a safe no-op: no throw and no signal conjured.
|
|
216
|
+
world.worker.pauseTransfer(uuid, "download", "/does-not-exist.txt")
|
|
217
|
+
expect(world.sync.pauseSignals["download:/does-not-exist.txt"]).toBeUndefined()
|
|
218
|
+
|
|
219
|
+
// With the signal resumed, the cycle still converges. (NOTE: the fake cloud does not honor
|
|
220
|
+
// pauseSignal mid-transfer, so end-to-end blocking is not observable here — only the
|
|
221
|
+
// worker's signal routing is. The transfer consumes/clears the signal when it completes.)
|
|
222
|
+
await plainCycle(world)
|
|
223
|
+
|
|
224
|
+
expect(snapshotRemote(world)["/a.txt"]).toMatchObject({ type: "file" })
|
|
225
|
+
expect(snapshotLocal(world)).toEqual(snapshotRemote(world))
|
|
226
|
+
}
|
|
227
|
+
)
|
|
228
|
+
})
|
|
229
|
+
|
|
230
|
+
it("I5: stopping a specific transfer surfaces an error, and the transfer is retried until it converges", async () => {
|
|
231
|
+
await withWorld(
|
|
232
|
+
{
|
|
233
|
+
mode: "twoWay"
|
|
234
|
+
},
|
|
235
|
+
async world => {
|
|
236
|
+
const uuid = world.syncPair.uuid
|
|
237
|
+
|
|
238
|
+
writeLocal(world, "a.txt", "v1")
|
|
239
|
+
world.triggerWatcher()
|
|
240
|
+
|
|
241
|
+
// Pre-register the transfer's abort controller (as if it were in flight) and stop it via the
|
|
242
|
+
// worker. The upload then runs against an already-aborted signal — the closest deterministic
|
|
243
|
+
// stand-in for a mid-transfer abort, since the harness completes transfers synchronously.
|
|
244
|
+
const signalKey = "upload:/a.txt"
|
|
245
|
+
world.sync.abortControllers[signalKey] = new AbortController()
|
|
246
|
+
world.worker.stopTransfer(uuid, "upload", "/a.txt")
|
|
247
|
+
expect(world.sync.abortControllers[signalKey]!.signal.aborted).toBe(true)
|
|
248
|
+
|
|
249
|
+
const abortedCycle = await capture(world, () => plainCycle(world))
|
|
250
|
+
|
|
251
|
+
// The abort is surfaced as an upload transfer error, and nothing was uploaded.
|
|
252
|
+
const uploadErrors = messagesOfType(abortedCycle, "transfer").filter(
|
|
253
|
+
message => message.data.of === "upload" && message.data.type === "error"
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
expect(uploadErrors.length).toBeGreaterThan(0)
|
|
257
|
+
expect(snapshotRemote(world)).toEqual({})
|
|
258
|
+
|
|
259
|
+
// Retry on a later cycle: clear the recorded task error (it gates the next cycle) and rerun.
|
|
260
|
+
// The controller was disposed by the failed transfer, so a fresh one is created and succeeds.
|
|
261
|
+
world.worker.resetTaskErrors(uuid)
|
|
262
|
+
world.triggerWatcher()
|
|
263
|
+
|
|
264
|
+
const retryCycle = await capture(world, () => plainCycle(world))
|
|
265
|
+
|
|
266
|
+
expect(transferKinds(retryCycle)).toContain("upload")
|
|
267
|
+
expect(snapshotRemote(world)["/a.txt"]).toMatchObject({ type: "file" })
|
|
268
|
+
expect(snapshotLocal(world)).toEqual(snapshotRemote(world))
|
|
269
|
+
|
|
270
|
+
// Idempotence: a further cycle with no external change performs no transfer.
|
|
271
|
+
world.triggerWatcher()
|
|
272
|
+
|
|
273
|
+
const settledCycle = await capture(world, () => plainCycle(world))
|
|
274
|
+
|
|
275
|
+
expect(allOps(settledCycle)).toEqual([])
|
|
276
|
+
}
|
|
277
|
+
)
|
|
278
|
+
})
|
|
279
|
+
|
|
280
|
+
it("I6: updateMode mid-run switches behavior on the next cycle (twoWay -> localBackup stops deletions)", async () => {
|
|
281
|
+
await withWorld(
|
|
282
|
+
{
|
|
283
|
+
mode: "twoWay",
|
|
284
|
+
initialLocal: { "/local/a.txt": "a", "/local/b.txt": "b" }
|
|
285
|
+
},
|
|
286
|
+
async world => {
|
|
287
|
+
const uuid = world.syncPair.uuid
|
|
288
|
+
|
|
289
|
+
// First cycle (twoWay): both files upload and the worlds converge.
|
|
290
|
+
const firstCycle = await capture(world, () => plainCycle(world))
|
|
291
|
+
|
|
292
|
+
expect(transferKinds(firstCycle)).toContain("upload")
|
|
293
|
+
expect(snapshotLocal(world)).toEqual(snapshotRemote(world))
|
|
294
|
+
|
|
295
|
+
// Switch to a backup mode; it must take effect on the NEXT cycle.
|
|
296
|
+
world.worker.updateMode(uuid, "localBackup")
|
|
297
|
+
|
|
298
|
+
// Delete one file (a deletion that twoWay WOULD propagate) and add another.
|
|
299
|
+
rmLocal(world, "a.txt")
|
|
300
|
+
writeLocal(world, "c.txt", "c")
|
|
301
|
+
world.triggerWatcher()
|
|
302
|
+
|
|
303
|
+
const backupCycle = await capture(world, () => plainCycle(world))
|
|
304
|
+
|
|
305
|
+
// localBackup propagates additions but NOT deletions, proving the new mode is active:
|
|
306
|
+
// the add reached the cloud, the deletion did not.
|
|
307
|
+
expect(transferKinds(backupCycle)).toContain("upload")
|
|
308
|
+
expect(transferKinds(backupCycle)).not.toContain("deleteRemoteFile")
|
|
309
|
+
expect(snapshotRemote(world)["/c.txt"]).toMatchObject({ type: "file" })
|
|
310
|
+
// The local deletion was NOT mirrored to the cloud (backup keeps the remote copy)…
|
|
311
|
+
expect(snapshotRemote(world)["/a.txt"]).toMatchObject({ type: "file" })
|
|
312
|
+
// …and it was not pulled back down either (backup is one-way local -> cloud).
|
|
313
|
+
expect(snapshotLocal(world)["/a.txt"]).toBeUndefined()
|
|
314
|
+
}
|
|
315
|
+
)
|
|
316
|
+
})
|
|
317
|
+
|
|
318
|
+
it("I7: updateSyncPairs is a safe idempotent no-op for empty input and an already-registered pair", async () => {
|
|
319
|
+
await withWorld(
|
|
320
|
+
{
|
|
321
|
+
mode: "twoWay",
|
|
322
|
+
initialLocal: { "/local/a.txt": "a" }
|
|
323
|
+
},
|
|
324
|
+
async world => {
|
|
325
|
+
const uuid = world.syncPair.uuid
|
|
326
|
+
|
|
327
|
+
await plainCycle(world)
|
|
328
|
+
|
|
329
|
+
const existingSync = world.worker.syncs[uuid]
|
|
330
|
+
|
|
331
|
+
// Empty input returns early; re-registering the existing pair must not recreate its Sync
|
|
332
|
+
// (the worker guards on syncs[uuid]). Neither call throws or disturbs internal state.
|
|
333
|
+
await world.worker.updateSyncPairs([])
|
|
334
|
+
await world.worker.updateSyncPairs([world.syncPair])
|
|
335
|
+
|
|
336
|
+
expect(world.worker.syncs[uuid]).toBe(existingSync)
|
|
337
|
+
expect(Object.keys(world.worker.syncs)).toEqual([uuid])
|
|
338
|
+
|
|
339
|
+
// The worker remains healthy after the no-op registrations: a new change still converges.
|
|
340
|
+
// (NOTE: registering a brand-new pair triggers Sync.initialize() -> the self-scheduling
|
|
341
|
+
// run() loop, which this direct-runCycle harness does not host; runtime-add of a NEW pair
|
|
342
|
+
// is therefore covered structurally — method present + idempotent — not end-to-end.)
|
|
343
|
+
writeLocal(world, "b.txt", "b")
|
|
344
|
+
world.triggerWatcher()
|
|
345
|
+
|
|
346
|
+
await plainCycle(world)
|
|
347
|
+
|
|
348
|
+
expect(snapshotRemote(world)["/b.txt"]).toMatchObject({ type: "file" })
|
|
349
|
+
expect(snapshotLocal(world)).toEqual(snapshotRemote(world))
|
|
350
|
+
}
|
|
351
|
+
)
|
|
352
|
+
})
|
|
353
|
+
})
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest"
|
|
2
|
+
import { runScenario, runCycle, restart, control, localMutate } from "../harness/runner"
|
|
3
|
+
import { transferOps, countMessages, snapshotLocal, snapshotRemote } from "../harness/snapshot"
|
|
4
|
+
import { rmLocal } from "../harness/mutations"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Category J — state persistence / cache (behavioral spec §J, §9).
|
|
8
|
+
*
|
|
9
|
+
* These pin the durability layer that lets the engine stay quiet across restarts and remote-tree
|
|
10
|
+
* cache hits:
|
|
11
|
+
*
|
|
12
|
+
* - The persisted state lives under `state/v2/<uuid>/` as five line-delimited-JSON files
|
|
13
|
+
* (`previousLocalTree`, `previousLocalINodes`, `previousRemoteTree`, `previousRemoteUUIDs`,
|
|
14
|
+
* `localFileHashes`); `restart()` rebuilds the engine over the same vfs+cloud so that state is
|
|
15
|
+
* reloaded from disk (see {@link restartSync}).
|
|
16
|
+
* - The `deviceId/v1/<uuid>` file drives the SDK `dir().tree()` cache: once a deviceId has fetched at
|
|
17
|
+
* a given revision, the next fetch returns an EMPTY `{files:[],folders:[]}` "unchanged" response,
|
|
18
|
+
* which {@link RemoteFileSystem.getDirectoryTree} treats as "data did not change, use cache" — NOT
|
|
19
|
+
* as "the remote was emptied".
|
|
20
|
+
* - `worker.resetCache(uuid)` drops both in-memory tree caches (and the deviceId-cache timestamp) so
|
|
21
|
+
* the next cycle performs a fresh full re-scan.
|
|
22
|
+
* - `localFileHashes` is pruned during the local tree scan: any stored path absent from the freshly
|
|
23
|
+
* scanned tree is dropped.
|
|
24
|
+
* - Corrupt/partial state files degrade to an empty previous state (the line reader swallows
|
|
25
|
+
* unparseable lines) rather than throwing, so the engine re-converges instead of crashing.
|
|
26
|
+
*/
|
|
27
|
+
describe("Category J — state persistence / cache", () => {
|
|
28
|
+
it("J1: persisted previous-trees + hashes round-trip through a restart → next cycle is a no-op", async () => {
|
|
29
|
+
let preLocalTree: Record<string, unknown> = {}
|
|
30
|
+
let preRemoteTree: Record<string, unknown> = {}
|
|
31
|
+
let preHashes: Record<string, string> = {}
|
|
32
|
+
let postLocalTree: Record<string, unknown> = {}
|
|
33
|
+
let postRemoteTree: Record<string, unknown> = {}
|
|
34
|
+
let postHashes: Record<string, string> = {}
|
|
35
|
+
|
|
36
|
+
const result = await runScenario({
|
|
37
|
+
name: "J1",
|
|
38
|
+
mode: "twoWay",
|
|
39
|
+
initialLocal: { "/local/a.txt": "alpha", "/local/dir/b.txt": "beta" },
|
|
40
|
+
steps: [
|
|
41
|
+
runCycle(),
|
|
42
|
+
runCycle(),
|
|
43
|
+
runCycle(),
|
|
44
|
+
// Capture the in-memory persisted state of the converged engine just before the restart.
|
|
45
|
+
control(world => {
|
|
46
|
+
preLocalTree = structuredClone(world.sync.previousLocalTree.tree)
|
|
47
|
+
preRemoteTree = structuredClone(world.sync.previousRemoteTree.tree)
|
|
48
|
+
preHashes = structuredClone(world.sync.localFileHashes)
|
|
49
|
+
}),
|
|
50
|
+
restart(),
|
|
51
|
+
// After the restart the NEW engine has loaded its previous state straight from disk.
|
|
52
|
+
control(world => {
|
|
53
|
+
postLocalTree = structuredClone(world.sync.previousLocalTree.tree)
|
|
54
|
+
postRemoteTree = structuredClone(world.sync.previousRemoteTree.tree)
|
|
55
|
+
postHashes = structuredClone(world.sync.localFileHashes)
|
|
56
|
+
}),
|
|
57
|
+
runCycle(),
|
|
58
|
+
runCycle()
|
|
59
|
+
]
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
// The reloaded state is non-empty and exactly equals what was persisted (a faithful round-trip).
|
|
63
|
+
expect(Object.keys(postLocalTree).length).toBeGreaterThan(0)
|
|
64
|
+
expect(postLocalTree).toEqual(preLocalTree)
|
|
65
|
+
expect(postRemoteTree).toEqual(preRemoteTree)
|
|
66
|
+
expect(postHashes).toEqual(preHashes)
|
|
67
|
+
expect(postLocalTree).toHaveProperty("/a.txt")
|
|
68
|
+
expect(postRemoteTree).toHaveProperty("/dir/b.txt")
|
|
69
|
+
expect(postHashes).toHaveProperty("/a.txt")
|
|
70
|
+
|
|
71
|
+
// The two cycles after the restart (indices 3 and 4) transfer nothing: state proves convergence.
|
|
72
|
+
expect(transferOps(result.cycles[3]!.messages)).toEqual([])
|
|
73
|
+
expect(transferOps(result.cycles[4]!.messages)).toEqual([])
|
|
74
|
+
expect(countMessages(result.messages, "cycleError")).toBe(0)
|
|
75
|
+
expect(countMessages(result.messages, "cycleNoChanges")).toBeGreaterThanOrEqual(1)
|
|
76
|
+
|
|
77
|
+
// The worlds remain converged and intact across the restart (no data loss).
|
|
78
|
+
expect(result.finalLocal["/a.txt"]).toMatchObject({ type: "file", size: 5 })
|
|
79
|
+
expect(result.finalLocal["/dir/b.txt"]).toMatchObject({ type: "file", size: 4 })
|
|
80
|
+
expect(result.finalLocal).toEqual(result.finalRemote)
|
|
81
|
+
})
|
|
82
|
+
|
|
83
|
+
it("J2: an empty deviceId cache-hit tree is read as 'unchanged', not 'remote emptied'", async () => {
|
|
84
|
+
let revisionBefore = -1
|
|
85
|
+
let revisionAfter = -1
|
|
86
|
+
|
|
87
|
+
const result = await runScenario({
|
|
88
|
+
name: "J2",
|
|
89
|
+
mode: "twoWay",
|
|
90
|
+
initialLocal: { "/local/one.txt": "one", "/local/sub/two.txt": "two" },
|
|
91
|
+
steps: [
|
|
92
|
+
runCycle(),
|
|
93
|
+
runCycle(),
|
|
94
|
+
runCycle(),
|
|
95
|
+
// Record the cloud's mutation counter just before a steady-state cycle...
|
|
96
|
+
control(world => {
|
|
97
|
+
revisionBefore = world.cloud.controls.revision()
|
|
98
|
+
}),
|
|
99
|
+
runCycle(),
|
|
100
|
+
// ...and again right after it.
|
|
101
|
+
control(world => {
|
|
102
|
+
revisionAfter = world.cloud.controls.revision()
|
|
103
|
+
})
|
|
104
|
+
]
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
// The steady-state cycle drove the remote tree fetch into its cache-hit (empty-tree) path. The
|
|
108
|
+
// engine used its cache: no cloud mutation happened (the revision counter is stable) and the
|
|
109
|
+
// cycle performed no file transfers.
|
|
110
|
+
expect(revisionBefore).toBeGreaterThanOrEqual(0)
|
|
111
|
+
expect(revisionAfter).toBe(revisionBefore)
|
|
112
|
+
expect(transferOps(result.cycles[3]!.messages)).toEqual([])
|
|
113
|
+
expect(countMessages(result.messages, "cycleNoChanges")).toBeGreaterThanOrEqual(1)
|
|
114
|
+
|
|
115
|
+
// Crucially: the empty cache-hit response did NOT delete everything locally — all files survive
|
|
116
|
+
// on both sides and the worlds stay converged.
|
|
117
|
+
expect(result.finalLocal["/one.txt"]).toMatchObject({ type: "file", size: 3 })
|
|
118
|
+
expect(result.finalLocal["/sub/two.txt"]).toMatchObject({ type: "file", size: 3 })
|
|
119
|
+
expect(result.finalLocal).toEqual(result.finalRemote)
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
it("J3: resetCache drops the tree caches and forces a fresh full re-scan that still converges", async () => {
|
|
123
|
+
let timestampAfterReset = -1
|
|
124
|
+
let treeKeysAfterReset: string[] = []
|
|
125
|
+
let timestampAfterRescan = -1
|
|
126
|
+
let treeKeysAfterRescan: string[] = []
|
|
127
|
+
let convergedRemote: Record<string, unknown> = {}
|
|
128
|
+
|
|
129
|
+
const result = await runScenario({
|
|
130
|
+
name: "J3",
|
|
131
|
+
mode: "twoWay",
|
|
132
|
+
initialLocal: { "/local/keep.txt": "keep-me", "/local/nested/deep.txt": "deep-content" },
|
|
133
|
+
steps: [
|
|
134
|
+
runCycle(),
|
|
135
|
+
runCycle(),
|
|
136
|
+
runCycle(),
|
|
137
|
+
// Force a full cache reset on the converged engine, then observe the now-empty remote cache.
|
|
138
|
+
control(world => {
|
|
139
|
+
convergedRemote = snapshotRemote(world)
|
|
140
|
+
|
|
141
|
+
world.worker.resetCache(world.syncPair.uuid)
|
|
142
|
+
|
|
143
|
+
timestampAfterReset = world.sync.remoteFileSystem.getDirectoryTreeCache.timestamp
|
|
144
|
+
treeKeysAfterReset = Object.keys(world.sync.remoteFileSystem.getDirectoryTreeCache.tree)
|
|
145
|
+
}),
|
|
146
|
+
runCycle(),
|
|
147
|
+
// The cycle had to re-fetch the full remote tree (cache timestamp was 0, forcing a
|
|
148
|
+
// skipCache fetch) — the cache is repopulated with every node.
|
|
149
|
+
control(world => {
|
|
150
|
+
timestampAfterRescan = world.sync.remoteFileSystem.getDirectoryTreeCache.timestamp
|
|
151
|
+
treeKeysAfterRescan = Object.keys(world.sync.remoteFileSystem.getDirectoryTreeCache.tree)
|
|
152
|
+
})
|
|
153
|
+
]
|
|
154
|
+
})
|
|
155
|
+
|
|
156
|
+
// resetCache cleared the remote tree cache outright.
|
|
157
|
+
expect(timestampAfterReset).toBe(0)
|
|
158
|
+
expect(treeKeysAfterReset).toEqual([])
|
|
159
|
+
|
|
160
|
+
// The next cycle did a fresh full re-scan: a new timestamp and the full tree are back.
|
|
161
|
+
expect(timestampAfterRescan).toBeGreaterThan(0)
|
|
162
|
+
expect(treeKeysAfterRescan).toContain("/keep.txt")
|
|
163
|
+
expect(treeKeysAfterRescan).toContain("/nested/deep.txt")
|
|
164
|
+
|
|
165
|
+
// A forced re-scan must not corrupt or lose data: no transfers, no deletions, same converged state.
|
|
166
|
+
expect(transferOps(result.cycles[3]!.messages)).toEqual([])
|
|
167
|
+
expect(countMessages(result.messages, "cycleError")).toBe(0)
|
|
168
|
+
expect(snapshotRemote(result.world)).toEqual(convergedRemote)
|
|
169
|
+
expect(result.finalLocal["/keep.txt"]).toMatchObject({ type: "file", size: 7 })
|
|
170
|
+
expect(result.finalLocal).toEqual(result.finalRemote)
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
it("J4: a deleted local path's stored hash is pruned while a surviving path's hash is kept", async () => {
|
|
174
|
+
let hashesBeforeDelete: Record<string, string> = {}
|
|
175
|
+
|
|
176
|
+
const result = await runScenario({
|
|
177
|
+
name: "J4",
|
|
178
|
+
mode: "twoWay",
|
|
179
|
+
initialLocal: { "/local/gone.txt": "to-be-deleted", "/local/stays.txt": "survivor" },
|
|
180
|
+
steps: [
|
|
181
|
+
runCycle(),
|
|
182
|
+
runCycle(),
|
|
183
|
+
// Both uploaded files have a stored md5 hash at this point.
|
|
184
|
+
control(world => {
|
|
185
|
+
hashesBeforeDelete = structuredClone(world.sync.localFileHashes)
|
|
186
|
+
}),
|
|
187
|
+
localMutate(world => rmLocal(world, "gone.txt")),
|
|
188
|
+
runCycle(),
|
|
189
|
+
runCycle()
|
|
190
|
+
]
|
|
191
|
+
})
|
|
192
|
+
|
|
193
|
+
// Pre-delete: both paths had a stored hash.
|
|
194
|
+
expect(hashesBeforeDelete["/gone.txt"]).toEqual(expect.any(String))
|
|
195
|
+
expect(hashesBeforeDelete["/stays.txt"]).toEqual(expect.any(String))
|
|
196
|
+
|
|
197
|
+
// Post-delete cycle: the local scan pruned the now-missing path's hash and kept the survivor's
|
|
198
|
+
// (still equal to the value it held before the deletion).
|
|
199
|
+
expect(result.world.sync.localFileHashes["/gone.txt"]).toBeUndefined()
|
|
200
|
+
expect(result.world.sync.localFileHashes["/stays.txt"]).toBe(hashesBeforeDelete["/stays.txt"])
|
|
201
|
+
|
|
202
|
+
// The deletion itself propagated correctly and the surviving file is intact.
|
|
203
|
+
expect(result.finalLocal["/gone.txt"]).toBeUndefined()
|
|
204
|
+
expect(result.finalRemote["/gone.txt"]).toBeUndefined()
|
|
205
|
+
expect(result.finalLocal["/stays.txt"]).toMatchObject({ type: "file", size: "survivor".length })
|
|
206
|
+
expect(result.finalLocal).toEqual(result.finalRemote)
|
|
207
|
+
})
|
|
208
|
+
|
|
209
|
+
it("J5: corrupt/partial state files degrade to an empty previous state and the engine re-converges", async () => {
|
|
210
|
+
let postRestartLocalKeys = -1
|
|
211
|
+
let postRestartRemoteKeys = -1
|
|
212
|
+
|
|
213
|
+
const result = await runScenario({
|
|
214
|
+
name: "J5",
|
|
215
|
+
mode: "twoWay",
|
|
216
|
+
initialLocal: { "/local/safe.txt": "safe-content", "/local/dir/child.txt": "child-content" },
|
|
217
|
+
steps: [
|
|
218
|
+
runCycle(),
|
|
219
|
+
runCycle(),
|
|
220
|
+
runCycle(),
|
|
221
|
+
// Overwrite the persisted tree records with unparseable garbage (the dir/files still exist,
|
|
222
|
+
// so the loader proceeds to read them rather than short-circuiting to "no saved state").
|
|
223
|
+
control(world => {
|
|
224
|
+
const state = world.sync.state
|
|
225
|
+
const garbage = "not-json\n}{ broken line\n<<< corrupt >>>\n"
|
|
226
|
+
|
|
227
|
+
for (const filePath of [
|
|
228
|
+
state.previousLocalTreePath,
|
|
229
|
+
state.previousLocalINodesPath,
|
|
230
|
+
state.previousRemoteTreePath,
|
|
231
|
+
state.previousRemoteUUIDsPath,
|
|
232
|
+
state.localFileHashesPath
|
|
233
|
+
]) {
|
|
234
|
+
world.vfs.ifs.writeFileSync(filePath, garbage)
|
|
235
|
+
}
|
|
236
|
+
}),
|
|
237
|
+
restart(),
|
|
238
|
+
// The corrupt lines were swallowed: the reloaded previous trees are empty (treated as a
|
|
239
|
+
// fresh, blank previous state) and the restart did not throw.
|
|
240
|
+
control(world => {
|
|
241
|
+
postRestartLocalKeys = Object.keys(world.sync.previousLocalTree.tree).length
|
|
242
|
+
postRestartRemoteKeys = Object.keys(world.sync.previousRemoteTree.tree).length
|
|
243
|
+
}),
|
|
244
|
+
runCycle(),
|
|
245
|
+
runCycle()
|
|
246
|
+
]
|
|
247
|
+
})
|
|
248
|
+
|
|
249
|
+
// The corruption was recovered as "empty previous state", not loaded as real data.
|
|
250
|
+
expect(postRestartLocalKeys).toBe(0)
|
|
251
|
+
expect(postRestartRemoteKeys).toBe(0)
|
|
252
|
+
|
|
253
|
+
// No crash, and because both sides already hold identical content the re-diff needs no transfers.
|
|
254
|
+
expect(countMessages(result.messages, "cycleError")).toBe(0)
|
|
255
|
+
expect(transferOps(result.cycles[3]!.messages)).toEqual([])
|
|
256
|
+
expect(transferOps(result.cycles[4]!.messages)).toEqual([])
|
|
257
|
+
|
|
258
|
+
// All originally-synced data survives intact on both sides and the worlds stay converged.
|
|
259
|
+
expect(snapshotLocal(result.world)["/safe.txt"]).toMatchObject({ type: "file", size: "safe-content".length })
|
|
260
|
+
expect(result.finalLocal["/safe.txt"]).toMatchObject({ type: "file" })
|
|
261
|
+
expect(result.finalLocal["/dir/child.txt"]).toMatchObject({ type: "file" })
|
|
262
|
+
expect(result.finalLocal).toEqual(result.finalRemote)
|
|
263
|
+
})
|
|
264
|
+
})
|