@filen/sync 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.node-version +1 -1
- package/dist/ignorer.d.ts +6 -0
- package/dist/ignorer.js +43 -24
- package/dist/ignorer.js.map +1 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/deltas.d.ts +58 -2
- package/dist/lib/deltas.js +693 -108
- package/dist/lib/deltas.js.map +1 -1
- package/dist/lib/environment.d.ts +47 -0
- package/dist/lib/environment.js +71 -0
- package/dist/lib/environment.js.map +1 -0
- package/dist/lib/filesystems/dirTree.d.ts +70 -0
- package/dist/lib/filesystems/dirTree.js +157 -0
- package/dist/lib/filesystems/dirTree.js.map +1 -0
- package/dist/lib/filesystems/local.d.ts +18 -8
- package/dist/lib/filesystems/local.js +166 -160
- package/dist/lib/filesystems/local.js.map +1 -1
- package/dist/lib/filesystems/remote.d.ts +12 -5
- package/dist/lib/filesystems/remote.js +226 -172
- package/dist/lib/filesystems/remote.js.map +1 -1
- package/dist/lib/ipc.js +1 -2
- package/dist/lib/ipc.js.map +1 -1
- package/dist/lib/lock.js +19 -12
- package/dist/lib/lock.js.map +1 -1
- package/dist/lib/logger.js +9 -7
- package/dist/lib/logger.js.map +1 -1
- package/dist/lib/state.js +159 -63
- package/dist/lib/state.js.map +1 -1
- package/dist/lib/sync.d.ts +18 -0
- package/dist/lib/sync.js +165 -96
- package/dist/lib/sync.js.map +1 -1
- package/dist/lib/tasks.d.ts +7 -8
- package/dist/lib/tasks.js +38 -45
- package/dist/lib/tasks.js.map +1 -1
- package/dist/semaphore.d.ts +1 -0
- package/dist/semaphore.js +22 -5
- package/dist/semaphore.js.map +1 -1
- package/dist/utils.js +51 -35
- package/dist/utils.js.map +1 -1
- package/eslint.config.mjs +36 -0
- package/package.json +19 -15
- package/tests/bench/collapse.bench.ts +114 -0
- package/tests/bench/cycle.bench.ts +111 -0
- package/tests/bench/deltas.bench.ts +151 -0
- package/tests/bench/harness/fake-sync.ts +32 -0
- package/tests/bench/harness/measure.ts +276 -0
- package/tests/bench/harness/scale-world.ts +160 -0
- package/tests/bench/harness/trees.ts +275 -0
- package/tests/bench/local-scan.bench.ts +74 -0
- package/tests/bench/longrun.bench.ts +130 -0
- package/tests/bench/profile-incremental.ts +90 -0
- package/tests/bench/remote-build.bench.ts +104 -0
- package/tests/bench/render.ts +14 -0
- package/tests/bench/semaphore.bench.ts +79 -0
- package/tests/bench/state.bench.ts +85 -0
- package/tests/bench/tasks-dispatch.bench.ts +156 -0
- package/tests/conformance/virtual-fs.test.ts +213 -0
- package/tests/e2e/backup.e2e.test.ts +130 -0
- package/tests/e2e/confirm.e2e.test.ts +191 -0
- package/tests/e2e/conflict.e2e.test.ts +261 -0
- package/tests/e2e/edge.e2e.test.ts +339 -0
- package/tests/e2e/harness/account.ts +104 -0
- package/tests/e2e/harness/assert.ts +127 -0
- package/tests/e2e/harness/drive.ts +88 -0
- package/tests/e2e/harness/mutations.ts +249 -0
- package/tests/e2e/harness/world.ts +222 -0
- package/tests/e2e/ignore.e2e.test.ts +123 -0
- package/tests/e2e/lifecycle.e2e.test.ts +290 -0
- package/tests/e2e/modes.e2e.test.ts +215 -0
- package/tests/e2e/platform.e2e.test.ts +157 -0
- package/tests/e2e/property.e2e.test.ts +163 -0
- package/tests/e2e/races.e2e.test.ts +90 -0
- package/tests/e2e/regressions.e2e.test.ts +212 -0
- package/tests/e2e/resilience.e2e.test.ts +231 -0
- package/tests/e2e/special.e2e.test.ts +185 -0
- package/tests/e2e/state.e2e.test.ts +229 -0
- package/tests/e2e/sync.e2e.test.ts +222 -0
- package/tests/fakes/fake-cloud.test.ts +267 -0
- package/tests/fakes/fake-cloud.ts +1094 -0
- package/tests/fakes/virtual-fs.ts +354 -0
- package/tests/harness/known-bug.ts +17 -0
- package/tests/harness/mutations.ts +65 -0
- package/tests/harness/runner.ts +141 -0
- package/tests/harness/snapshot.ts +113 -0
- package/tests/harness/world.ts +187 -0
- package/tests/scenarios/a-baseline.test.ts +107 -0
- package/tests/scenarios/aa-races.test.ts +258 -0
- package/tests/scenarios/ab-mode-property.test.ts +189 -0
- package/tests/scenarios/ac-platform.test.ts +320 -0
- package/tests/scenarios/ad-unicode-normalization.test.ts +67 -0
- package/tests/scenarios/b-additions.test.ts +160 -0
- package/tests/scenarios/c-modifications.test.ts +194 -0
- package/tests/scenarios/d-deletions.test.ts +259 -0
- package/tests/scenarios/e-rename-move.test.ts +288 -0
- package/tests/scenarios/f-ignore-filter.test.ts +346 -0
- package/tests/scenarios/g-large-deletion.test.ts +277 -0
- package/tests/scenarios/h-resilience.test.ts +167 -0
- package/tests/scenarios/i-lifecycle.test.ts +353 -0
- package/tests/scenarios/j-state-cache.test.ts +264 -0
- package/tests/scenarios/k-scale.test.ts +202 -0
- package/tests/scenarios/l-property.test.ts +145 -0
- package/tests/scenarios/m-golden.test.ts +452 -0
- package/tests/scenarios/o-task-errors.test.ts +497 -0
- package/tests/scenarios/p-remote-originated.test.ts +306 -0
- package/tests/scenarios/q-cycle-lifecycle.test.ts +234 -0
- package/tests/scenarios/r-rename-stress.test.ts +208 -0
- package/tests/scenarios/s-upgrade-transition.test.ts +171 -0
- package/tests/scenarios/t-type-change.test.ts +144 -0
- package/tests/scenarios/u-mode-local-to-cloud.test.ts +347 -0
- package/tests/scenarios/v-mode-local-backup.test.ts +201 -0
- package/tests/scenarios/w-mode-cloud-to-local.test.ts +304 -0
- package/tests/scenarios/x-mode-cloud-backup.test.ts +201 -0
- package/tests/scenarios/y-conflict-matrix.test.ts +292 -0
- package/tests/scenarios/z-cross-ops.test.ts +285 -0
- package/tests/scenarios/zb-dir-rename-cross.test.ts +296 -0
- package/tests/scenarios/zc-crash-recovery.test.ts +189 -0
- package/tests/scenarios/zd-inode-reuse.test.ts +118 -0
- package/tests/scenarios/ze-move-into-new-dir.test.ts +130 -0
- package/tests/scenarios/zf-remote-change-unchanged-local.test.ts +81 -0
- package/tests/scenarios/zg-edit-during-scan.test.ts +68 -0
- package/tests/scenarios/zh-dir-delete-vs-child.test.ts +104 -0
- package/tests/scenarios/zi-smoke-test-outage.test.ts +78 -0
- package/tests/scenarios/zj-trash-cleanup.test.ts +133 -0
- package/tests/scenarios/zk-ignore-asymmetry.test.ts +150 -0
- package/tests/scenarios/zl-mode-atomicity.test.ts +104 -0
- package/tests/scenarios/zm-scan-concurrency.test.ts +78 -0
- package/tests/scenarios/zn-delta-ordering.test.ts +130 -0
- package/tests/scenarios/zo-download-temp-cleanup.test.ts +65 -0
- package/tests/unit/collapse-deltas.test.ts +276 -0
- package/tests/unit/dir-tree.test.ts +159 -0
- package/tests/unit/icloud.test.ts +115 -0
- package/tests/unit/ignorer-cache-regression.test.ts +70 -0
- package/tests/unit/ignorer.test.ts +63 -0
- package/tests/unit/ipc-lock.test.ts +438 -0
- package/tests/unit/lock.test.ts +135 -0
- package/tests/unit/n-unit.test.ts +632 -0
- package/tests/unit/remote-tree-unordered-regression.test.ts +101 -0
- package/tests/unit/semaphore-regression.test.ts +140 -0
- package/tests/unit/state-refencode-regression.test.ts +224 -0
- package/tests/unit/state.test.ts +809 -0
- package/tests/unit/tasks-dispatch-order-regression.test.ts +53 -0
- package/tests/unit/worker-api.test.ts +379 -0
- package/tsconfig.json +10 -1
- package/tsconfig.test.json +12 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/vitest.bench.config.ts +32 -0
- package/vitest.config.ts +27 -0
- package/vitest.e2e.config.ts +68 -0
- package/.eslintrc +0 -16
- package/jest.config.js +0 -5
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import type FilenSDK from "@filen/sdk"
|
|
2
|
+
import Sync from "../../../src/lib/sync"
|
|
3
|
+
import SyncWorker from "../../../src/index"
|
|
4
|
+
import { type SyncPair, type SyncMessage, type SyncMode } from "../../../src/types"
|
|
5
|
+
import { type SyncEnvironment, defaultEnvironment } from "../../../src/lib/environment"
|
|
6
|
+
import os from "os"
|
|
7
|
+
import pathModule from "path"
|
|
8
|
+
import fs from "fs-extra"
|
|
9
|
+
import { v4 as uuidv4 } from "uuid"
|
|
10
|
+
|
|
11
|
+
export type E2EWorldOptions = {
|
|
12
|
+
sdk: FilenSDK
|
|
13
|
+
mode: SyncMode
|
|
14
|
+
/** Default true: local deletions are permanent (no `.filen.trash` subdir to reason about). */
|
|
15
|
+
localTrashDisabled?: boolean
|
|
16
|
+
excludeDotFiles?: boolean
|
|
17
|
+
requireConfirmationOnLargeDeletion?: boolean
|
|
18
|
+
/** `.filenignore` contents to seed at the local root before the first cycle. */
|
|
19
|
+
filenIgnore?: string
|
|
20
|
+
/**
|
|
21
|
+
* Default false: use a no-op watcher and drive cycles manually (deterministic). Set true only for a
|
|
22
|
+
* test that specifically exercises the production fs.watch watcher.
|
|
23
|
+
*/
|
|
24
|
+
useRealWatcher?: boolean
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export type E2EWorld = {
|
|
28
|
+
sdk: FilenSDK
|
|
29
|
+
worker: SyncWorker
|
|
30
|
+
sync: Sync
|
|
31
|
+
syncPair: SyncPair
|
|
32
|
+
messages: SyncMessage[]
|
|
33
|
+
/** The unique id for this world; used for both the local tmp dir name and the remote dir name. */
|
|
34
|
+
runId: string
|
|
35
|
+
/** UUID of the remote `/<runId>` directory that is this sync's remote root. */
|
|
36
|
+
remoteParentUUID: string
|
|
37
|
+
/** Absolute path of the local sync root (a fresh tmp dir). */
|
|
38
|
+
localRoot: string
|
|
39
|
+
/** Absolute path of the per-world state/db dir (a fresh tmp dir). */
|
|
40
|
+
dbPath: string
|
|
41
|
+
/** Absolute path of the tmp dir tree that holds localRoot + dbPath (removed on teardown). */
|
|
42
|
+
workRoot: string
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const noopWatcher: SyncEnvironment["createWatcher"] = async () => ({
|
|
46
|
+
close: async (): Promise<void> => {}
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Build a real, isolated sync world against the live account:
|
|
51
|
+
* - a remote directory `/<runId>` under the account's base folder is the sync's remote root,
|
|
52
|
+
* - a fresh local tmp dir is the sync's local root, with its state/db in a sibling tmp dir,
|
|
53
|
+
* - a real {@link SyncWorker}/{@link Sync} wired with the PRODUCTION environment (real fs, real
|
|
54
|
+
* write-file-atomic, the msgpack `fetchDirTree`) so the engine talks to the real backend exactly as
|
|
55
|
+
* it does in the desktop client. Only the directory watcher is a no-op by default — cycles are
|
|
56
|
+
* driven explicitly for determinism.
|
|
57
|
+
*
|
|
58
|
+
* Always pair with {@link destroyE2EWorld} (permanent remote delete + trash empty + local tmp removal).
|
|
59
|
+
*/
|
|
60
|
+
export async function createE2EWorld(options: E2EWorldOptions): Promise<E2EWorld> {
|
|
61
|
+
const { sdk } = options
|
|
62
|
+
const runId = uuidv4()
|
|
63
|
+
const baseFolderUUID = sdk.config.baseFolderUUID
|
|
64
|
+
|
|
65
|
+
if (!baseFolderUUID || baseFolderUUID.length === 0) {
|
|
66
|
+
throw new Error("SDK has no baseFolderUUID — login likely failed.")
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Remote root: a fresh /<runId> directory under the account base folder.
|
|
70
|
+
const remoteParentUUID = await sdk.cloud().createDirectory({
|
|
71
|
+
name: runId,
|
|
72
|
+
parent: baseFolderUUID
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
// Local roots: a fresh tmp tree, sync dir + db dir as siblings.
|
|
76
|
+
const workRoot = pathModule.join(os.tmpdir(), `filen-sync-e2e-${runId}`)
|
|
77
|
+
const localRoot = pathModule.join(workRoot, "local")
|
|
78
|
+
const dbPath = pathModule.join(workRoot, "db")
|
|
79
|
+
|
|
80
|
+
await fs.ensureDir(localRoot)
|
|
81
|
+
await fs.ensureDir(dbPath)
|
|
82
|
+
|
|
83
|
+
if (typeof options.filenIgnore === "string") {
|
|
84
|
+
await fs.writeFile(pathModule.join(localRoot, ".filenignore"), options.filenIgnore)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const messages: SyncMessage[] = []
|
|
88
|
+
const syncPair: SyncPair = {
|
|
89
|
+
name: `e2e-${runId}`,
|
|
90
|
+
uuid: runId,
|
|
91
|
+
localPath: localRoot,
|
|
92
|
+
remotePath: `/${runId}`,
|
|
93
|
+
remoteParentUUID,
|
|
94
|
+
mode: options.mode,
|
|
95
|
+
excludeDotFiles: options.excludeDotFiles ?? false,
|
|
96
|
+
paused: false,
|
|
97
|
+
localTrashDisabled: options.localTrashDisabled ?? true,
|
|
98
|
+
requireConfirmationOnLargeDeletion: options.requireConfirmationOnLargeDeletion ?? false
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const environment: SyncEnvironment = options.useRealWatcher
|
|
102
|
+
? defaultEnvironment()
|
|
103
|
+
: { ...defaultEnvironment(), createWatcher: noopWatcher }
|
|
104
|
+
|
|
105
|
+
const worker = new SyncWorker({
|
|
106
|
+
syncPairs: [syncPair],
|
|
107
|
+
dbPath,
|
|
108
|
+
sdk,
|
|
109
|
+
onMessage: message => {
|
|
110
|
+
messages.push(message)
|
|
111
|
+
},
|
|
112
|
+
disableLogging: true,
|
|
113
|
+
environment
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
const sync = new Sync({ syncPair, worker })
|
|
117
|
+
|
|
118
|
+
worker.syncs[syncPair.uuid] = sync
|
|
119
|
+
|
|
120
|
+
// Mirror the harness pattern: load persisted state directly and drive discrete cycles ourselves
|
|
121
|
+
// instead of letting initialize() start the self-scheduling loop.
|
|
122
|
+
await sync.state.initialize()
|
|
123
|
+
await sync.ignorer.initialize()
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
sdk,
|
|
127
|
+
worker,
|
|
128
|
+
sync,
|
|
129
|
+
syncPair,
|
|
130
|
+
messages,
|
|
131
|
+
runId,
|
|
132
|
+
remoteParentUUID,
|
|
133
|
+
localRoot,
|
|
134
|
+
dbPath,
|
|
135
|
+
workRoot
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Tear a world down completely and leave the account clean:
|
|
141
|
+
* 1. permanently delete the remote `/<runId>` directory (real delete, NOT trash),
|
|
142
|
+
* 2. empty the account trash (the engine trashes its own deletions during a test; a dedicated test
|
|
143
|
+
* account means this is safe and is what "no trash left behind" requires),
|
|
144
|
+
* 3. remove the local tmp tree.
|
|
145
|
+
* Every step is best-effort so one failure cannot strand the others.
|
|
146
|
+
*/
|
|
147
|
+
export async function destroyE2EWorld(world: E2EWorld): Promise<void> {
|
|
148
|
+
try {
|
|
149
|
+
await world.sync.cleanup({ deleteLocalDbFiles: false })
|
|
150
|
+
} catch {
|
|
151
|
+
// best effort
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
try {
|
|
155
|
+
await world.sdk.cloud().deleteDirectory({ uuid: world.remoteParentUUID })
|
|
156
|
+
} catch {
|
|
157
|
+
// already gone
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
try {
|
|
161
|
+
await world.sdk.cloud().emptyTrash()
|
|
162
|
+
} catch {
|
|
163
|
+
// best effort
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
try {
|
|
167
|
+
await fs.rm(world.workRoot, { force: true, recursive: true, maxRetries: 10, retryDelay: 100 })
|
|
168
|
+
} catch {
|
|
169
|
+
// best effort
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Simulate a process restart / client upgrade: rebuild the worker + sync over the SAME local dir, db
|
|
175
|
+
* dir, and remote root, so persisted state (previous trees, deviceId, hashes) is reloaded from disk.
|
|
176
|
+
* Mutates `world` in place and keeps appending to the same message stream.
|
|
177
|
+
*/
|
|
178
|
+
export async function restartE2EWorld(world: E2EWorld, options: { useRealWatcher?: boolean } = {}): Promise<void> {
|
|
179
|
+
try {
|
|
180
|
+
await world.sync.cleanup({ deleteLocalDbFiles: false })
|
|
181
|
+
} catch {
|
|
182
|
+
// best effort
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const environment: SyncEnvironment = options.useRealWatcher ? defaultEnvironment() : { ...defaultEnvironment(), createWatcher: noopWatcher }
|
|
186
|
+
|
|
187
|
+
const worker = new SyncWorker({
|
|
188
|
+
syncPairs: [world.syncPair],
|
|
189
|
+
dbPath: world.dbPath,
|
|
190
|
+
sdk: world.sdk,
|
|
191
|
+
onMessage: message => {
|
|
192
|
+
world.messages.push(message)
|
|
193
|
+
},
|
|
194
|
+
disableLogging: true,
|
|
195
|
+
environment
|
|
196
|
+
})
|
|
197
|
+
|
|
198
|
+
const sync = new Sync({ syncPair: world.syncPair, worker })
|
|
199
|
+
|
|
200
|
+
worker.syncs[world.syncPair.uuid] = sync
|
|
201
|
+
|
|
202
|
+
await sync.state.initialize()
|
|
203
|
+
await sync.ignorer.initialize()
|
|
204
|
+
|
|
205
|
+
world.worker = worker
|
|
206
|
+
world.sync = sync
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Create a world, run `body`, and ALWAYS tear it down (permanent remote delete + trash empty + local
|
|
211
|
+
* tmp removal) — even when the body throws. The standard way to write an e2e case so no account state
|
|
212
|
+
* leaks between tests or on failure.
|
|
213
|
+
*/
|
|
214
|
+
export async function withE2EWorld(options: E2EWorldOptions, body: (world: E2EWorld) => Promise<void>): Promise<void> {
|
|
215
|
+
const world = await createE2EWorld(options)
|
|
216
|
+
|
|
217
|
+
try {
|
|
218
|
+
await body(world)
|
|
219
|
+
} finally {
|
|
220
|
+
await destroyE2EWorld(world)
|
|
221
|
+
}
|
|
222
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { describe, it, expect, beforeAll, afterAll } from "vitest"
|
|
2
|
+
import type FilenSDK from "@filen/sdk"
|
|
3
|
+
import { E2E_ENABLED, loginTestSDK, teardownTestSDK } from "./harness/account"
|
|
4
|
+
import { withE2EWorld } from "./harness/world"
|
|
5
|
+
import { settle } from "./harness/drive"
|
|
6
|
+
import { snapshotRemoteReal } from "./harness/assert"
|
|
7
|
+
import { writeLocal, modifyLocal, existsLocal } from "./harness/mutations"
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Phase 3 e2e — .filenignore + dotfile filtering against the live backend. Ignored paths must never
|
|
11
|
+
* reach the remote; everything else must.
|
|
12
|
+
*/
|
|
13
|
+
describe.skipIf(!E2E_ENABLED)("E2E — ignore filtering", () => {
|
|
14
|
+
let sdk: FilenSDK
|
|
15
|
+
|
|
16
|
+
beforeAll(async () => {
|
|
17
|
+
sdk = await loginTestSDK()
|
|
18
|
+
}, 300_000)
|
|
19
|
+
|
|
20
|
+
afterAll(async () => {
|
|
21
|
+
await teardownTestSDK()
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
it("ignores a directory pattern", async () => {
|
|
25
|
+
await withE2EWorld({ sdk, mode: "twoWay", filenIgnore: "ignored/\n" }, async world => {
|
|
26
|
+
await writeLocal(world, "ignored/secret.txt", "nope")
|
|
27
|
+
await writeLocal(world, "visible.txt", "yes")
|
|
28
|
+
await settle(world)
|
|
29
|
+
|
|
30
|
+
const remote = await snapshotRemoteReal(world)
|
|
31
|
+
|
|
32
|
+
expect(remote["/visible.txt"]).toMatchObject({ type: "file" })
|
|
33
|
+
expect(remote["/ignored"]).toBeUndefined()
|
|
34
|
+
expect(remote["/ignored/secret.txt"]).toBeUndefined()
|
|
35
|
+
})
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
it("ignores a glob pattern", async () => {
|
|
39
|
+
await withE2EWorld({ sdk, mode: "twoWay", filenIgnore: "*.log\n" }, async world => {
|
|
40
|
+
await writeLocal(world, "app.log", "log")
|
|
41
|
+
await writeLocal(world, "app.txt", "txt")
|
|
42
|
+
await settle(world)
|
|
43
|
+
|
|
44
|
+
const remote = await snapshotRemoteReal(world)
|
|
45
|
+
|
|
46
|
+
expect(remote["/app.txt"]).toMatchObject({ type: "file" })
|
|
47
|
+
expect(remote["/app.log"]).toBeUndefined()
|
|
48
|
+
})
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
it("honors a negation pattern", async () => {
|
|
52
|
+
await withE2EWorld({ sdk, mode: "twoWay", filenIgnore: "*.log\n!keep.log\n" }, async world => {
|
|
53
|
+
await writeLocal(world, "drop.log", "drop")
|
|
54
|
+
await writeLocal(world, "keep.log", "keep")
|
|
55
|
+
await settle(world)
|
|
56
|
+
|
|
57
|
+
const remote = await snapshotRemoteReal(world)
|
|
58
|
+
|
|
59
|
+
expect(remote["/keep.log"]).toMatchObject({ type: "file" })
|
|
60
|
+
expect(remote["/drop.log"]).toBeUndefined()
|
|
61
|
+
})
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
it("ignores a nested glob pattern", async () => {
|
|
65
|
+
await withE2EWorld({ sdk, mode: "twoWay", filenIgnore: "**/*.tmp\n" }, async world => {
|
|
66
|
+
await writeLocal(world, "a/b/scratch.tmp", "tmp")
|
|
67
|
+
await writeLocal(world, "a/b/real.txt", "real")
|
|
68
|
+
await settle(world)
|
|
69
|
+
|
|
70
|
+
const remote = await snapshotRemoteReal(world)
|
|
71
|
+
|
|
72
|
+
expect(remote["/a/b/real.txt"]).toMatchObject({ type: "file" })
|
|
73
|
+
expect(remote["/a/b/scratch.tmp"]).toBeUndefined()
|
|
74
|
+
})
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
it("excludes dotfiles when excludeDotFiles is set", async () => {
|
|
78
|
+
await withE2EWorld({ sdk, mode: "twoWay", excludeDotFiles: true }, async world => {
|
|
79
|
+
await writeLocal(world, ".hidden", "secret")
|
|
80
|
+
await writeLocal(world, "shown.txt", "public")
|
|
81
|
+
await settle(world)
|
|
82
|
+
|
|
83
|
+
const remote = await snapshotRemoteReal(world)
|
|
84
|
+
|
|
85
|
+
expect(remote["/shown.txt"]).toMatchObject({ type: "file" })
|
|
86
|
+
expect(remote["/.hidden"]).toBeUndefined()
|
|
87
|
+
})
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
it("default OS-junk names (.DS_Store, Thumbs.db) are never uploaded; real files sync", async () => {
|
|
91
|
+
await withE2EWorld({ sdk, mode: "twoWay" }, async world => {
|
|
92
|
+
await writeLocal(world, ".DS_Store", "junk")
|
|
93
|
+
await writeLocal(world, "Thumbs.db", "junk")
|
|
94
|
+
await writeLocal(world, "real.txt", "real")
|
|
95
|
+
await settle(world)
|
|
96
|
+
|
|
97
|
+
const remote = await snapshotRemoteReal(world)
|
|
98
|
+
|
|
99
|
+
expect(remote["/.DS_Store"]).toBeUndefined()
|
|
100
|
+
expect(remote["/Thumbs.db"]).toBeUndefined()
|
|
101
|
+
expect(remote["/real.txt"]).toMatchObject({ type: "file" })
|
|
102
|
+
})
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
it("a file synced and THEN newly ignored is not deleted from the remote (ignore ≠ delete)", async () => {
|
|
106
|
+
await withE2EWorld({ sdk, mode: "twoWay" }, async world => {
|
|
107
|
+
await writeLocal(world, "keep-me.txt", "content")
|
|
108
|
+
await settle(world)
|
|
109
|
+
|
|
110
|
+
// It synced up.
|
|
111
|
+
expect((await snapshotRemoteReal(world))["/keep-me.txt"]).toMatchObject({ type: "file" })
|
|
112
|
+
|
|
113
|
+
// Now ignore the already-synced file and edit it; ignoring must not imply a remote deletion.
|
|
114
|
+
await world.worker.updateIgnorerContent(world.syncPair.uuid, "keep-me.txt")
|
|
115
|
+
await modifyLocal(world, "keep-me.txt", "content-edited-after-ignore")
|
|
116
|
+
await settle(world)
|
|
117
|
+
|
|
118
|
+
// The remote copy survives and the local file is untouched — ignore is not deletion.
|
|
119
|
+
expect((await snapshotRemoteReal(world))["/keep-me.txt"]).toMatchObject({ type: "file" })
|
|
120
|
+
expect(await existsLocal(world, "keep-me.txt")).toBe(true)
|
|
121
|
+
})
|
|
122
|
+
})
|
|
123
|
+
})
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
import { describe, it, expect, beforeAll, afterAll } from "vitest"
|
|
2
|
+
import FilenSDK, { PauseSignal } from "@filen/sdk"
|
|
3
|
+
import fs from "fs-extra"
|
|
4
|
+
import { E2E_ENABLED, loginTestSDK, teardownTestSDK } from "./harness/account"
|
|
5
|
+
import { withE2EWorld } from "./harness/world"
|
|
6
|
+
import { cycle, settle, expectConverged, messagesOfType, allOps } from "./harness/drive"
|
|
7
|
+
import { snapshotRemoteReal } from "./harness/assert"
|
|
8
|
+
import { writeLocal, rmLocal, uploadRemote, readLocal, existsLocal } from "./harness/mutations"
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Phase 3 e2e — the SyncWorker control surface against the live backend, the live counterpart of mocked
|
|
12
|
+
* Category I (+ the abort-driven error path of Q5). These exercise the REAL worker methods a host app
|
|
13
|
+
* drives at runtime — pause/resume (I2), runtime mode change (I6), pair removal + db cleanup (I3),
|
|
14
|
+
* idempotent registration (I7), per-transfer pause/stop routing (I4), and stop-then-retry for an upload
|
|
15
|
+
* (I5) — plus the task-error gate (Q5). A pre-registered abort controller is the deterministic stand-in
|
|
16
|
+
* for an in-flight transfer: the SDK honors the abort at the chunk-loop entry, so a pre-aborted upload
|
|
17
|
+
* fails reliably (verified by hammering). Stopping a DOWNLOAD is covered too — and surfaced a real bug now
|
|
18
|
+
* fixed in remote.ts: an aborted download RESOLVES with a 0-byte staged file, which the engine used to
|
|
19
|
+
* commit as synced (permanent divergence); the integrity guard now discards an incomplete download so the
|
|
20
|
+
* next cycle re-fetches it in full. Only runOnce (I1) stays mocked-only — a self-scheduling one-shot loop
|
|
21
|
+
* whose distinctive effect, cleanup-without-reschedule, is already exercised live by I3.
|
|
22
|
+
*/
|
|
23
|
+
describe.skipIf(!E2E_ENABLED)("E2E — lifecycle & control surface", () => {
|
|
24
|
+
let sdk: FilenSDK
|
|
25
|
+
|
|
26
|
+
beforeAll(async () => {
|
|
27
|
+
sdk = await loginTestSDK()
|
|
28
|
+
}, 300_000)
|
|
29
|
+
|
|
30
|
+
afterAll(async () => {
|
|
31
|
+
await teardownTestSDK()
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
it("a paused pair short-circuits its cycle; resuming syncs the pending change (I2)", async () => {
|
|
35
|
+
await withE2EWorld({ sdk, mode: "twoWay" }, async world => {
|
|
36
|
+
world.worker.updatePaused(world.syncPair.uuid, true)
|
|
37
|
+
|
|
38
|
+
// A change is pending while paused.
|
|
39
|
+
await writeLocal(world, "a.txt", "v1")
|
|
40
|
+
|
|
41
|
+
const pausedMessages = await cycle(world)
|
|
42
|
+
|
|
43
|
+
// The cycle short-circuits before doing any work: cyclePaused is emitted, nothing starts/transfers.
|
|
44
|
+
expect(messagesOfType(pausedMessages, "cyclePaused").length).toBeGreaterThan(0)
|
|
45
|
+
expect(messagesOfType(pausedMessages, "cycleStarted").length).toBe(0)
|
|
46
|
+
expect(allOps(pausedMessages)).toEqual([])
|
|
47
|
+
expect((await snapshotRemoteReal(world))["/a.txt"], "nothing uploaded while paused").toBeUndefined()
|
|
48
|
+
|
|
49
|
+
// Resume → the pending change now syncs to the real backend and the worlds converge.
|
|
50
|
+
world.worker.updatePaused(world.syncPair.uuid, false)
|
|
51
|
+
await settle(world)
|
|
52
|
+
await expectConverged(world)
|
|
53
|
+
|
|
54
|
+
expect((await snapshotRemoteReal(world))["/a.txt"]).toMatchObject({ type: "file" })
|
|
55
|
+
})
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
it("updateMode mid-run switches behavior on the next cycle: twoWay -> localBackup stops deletions (I6)", async () => {
|
|
59
|
+
await withE2EWorld({ sdk, mode: "twoWay" }, async world => {
|
|
60
|
+
await writeLocal(world, "a.txt", "a")
|
|
61
|
+
await writeLocal(world, "b.txt", "b")
|
|
62
|
+
await settle(world)
|
|
63
|
+
await expectConverged(world)
|
|
64
|
+
|
|
65
|
+
// Switch to a backup mode; it must take effect on the NEXT cycle.
|
|
66
|
+
world.worker.updateMode(world.syncPair.uuid, "localBackup")
|
|
67
|
+
|
|
68
|
+
// A deletion twoWay WOULD propagate, plus an addition.
|
|
69
|
+
await rmLocal(world, "a.txt")
|
|
70
|
+
await writeLocal(world, "c.txt", "c")
|
|
71
|
+
await settle(world)
|
|
72
|
+
|
|
73
|
+
const remote = await snapshotRemoteReal(world)
|
|
74
|
+
|
|
75
|
+
// localBackup propagates additions but NOT deletions, proving the new mode is active.
|
|
76
|
+
expect(remote["/c.txt"]).toMatchObject({ type: "file" })
|
|
77
|
+
expect(remote["/a.txt"], "the local deletion was NOT mirrored (backup keeps the remote copy)").toMatchObject({
|
|
78
|
+
type: "file"
|
|
79
|
+
})
|
|
80
|
+
})
|
|
81
|
+
})
|
|
82
|
+
|
|
83
|
+
it("removing a pair aborts its in-flight transfers, deletes its db files, and exits (I3)", async () => {
|
|
84
|
+
await withE2EWorld({ sdk, mode: "twoWay" }, async world => {
|
|
85
|
+
await writeLocal(world, "a.txt", "a")
|
|
86
|
+
// First settle persists state + a deviceId, so there are real db files to delete.
|
|
87
|
+
await settle(world)
|
|
88
|
+
|
|
89
|
+
const stateFiles = [
|
|
90
|
+
world.sync.state.previousLocalTreePath,
|
|
91
|
+
world.sync.state.previousRemoteTreePath,
|
|
92
|
+
world.sync.state.localFileHashesPath
|
|
93
|
+
]
|
|
94
|
+
|
|
95
|
+
for (const file of stateFiles) {
|
|
96
|
+
expect(await fs.pathExists(file)).toBe(true)
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Stand in for an in-flight transfer: a registered, not-yet-aborted controller at the real key.
|
|
100
|
+
world.sync.abortControllers["upload:/a.txt"] = new AbortController()
|
|
101
|
+
|
|
102
|
+
const mark = world.messages.length
|
|
103
|
+
|
|
104
|
+
await world.worker.updateRemoved(world.syncPair.uuid, true)
|
|
105
|
+
|
|
106
|
+
const removalMessages = world.messages.slice(mark)
|
|
107
|
+
|
|
108
|
+
// The transfer was aborted, the pair is marked removed, and it exited (cleanup -> cycleExited).
|
|
109
|
+
expect(world.sync.abortControllers["upload:/a.txt"]!.signal.aborted).toBe(true)
|
|
110
|
+
expect(world.sync.removed).toBe(true)
|
|
111
|
+
expect(messagesOfType(removalMessages, "cycleExited").length).toBeGreaterThan(0)
|
|
112
|
+
|
|
113
|
+
// The persisted state files for this pair are gone from the real filesystem.
|
|
114
|
+
for (const file of stateFiles) {
|
|
115
|
+
expect(await fs.pathExists(file)).toBe(false)
|
|
116
|
+
}
|
|
117
|
+
})
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
it("updateSyncPairs is an idempotent no-op for empty input and an already-registered pair (I7)", async () => {
|
|
121
|
+
await withE2EWorld({ sdk, mode: "twoWay" }, async world => {
|
|
122
|
+
await writeLocal(world, "a.txt", "a")
|
|
123
|
+
await settle(world)
|
|
124
|
+
await expectConverged(world)
|
|
125
|
+
|
|
126
|
+
const existingSync = world.worker.syncs[world.syncPair.uuid]
|
|
127
|
+
|
|
128
|
+
// Empty input returns early; re-registering the existing pair must not recreate its Sync.
|
|
129
|
+
await world.worker.updateSyncPairs([])
|
|
130
|
+
await world.worker.updateSyncPairs([world.syncPair])
|
|
131
|
+
|
|
132
|
+
expect(world.worker.syncs[world.syncPair.uuid]).toBe(existingSync)
|
|
133
|
+
|
|
134
|
+
// The worker stays healthy after the no-op registrations: a new change still converges live.
|
|
135
|
+
await writeLocal(world, "b.txt", "b")
|
|
136
|
+
await settle(world)
|
|
137
|
+
await expectConverged(world)
|
|
138
|
+
|
|
139
|
+
expect((await snapshotRemoteReal(world))["/b.txt"]).toMatchObject({ type: "file" })
|
|
140
|
+
})
|
|
141
|
+
})
|
|
142
|
+
|
|
143
|
+
it("pauseTransfer / resumeTransfer route to the per-transfer signal; unknown keys are safe no-ops (I4)", async () => {
|
|
144
|
+
await withE2EWorld({ sdk, mode: "twoWay" }, async world => {
|
|
145
|
+
const uuid = world.syncPair.uuid
|
|
146
|
+
|
|
147
|
+
// Stand in for an in-flight upload: a registered pause signal keyed `${type}:${path}`.
|
|
148
|
+
const signalKey = "upload:/a.txt"
|
|
149
|
+
world.sync.pauseSignals[signalKey] = new PauseSignal()
|
|
150
|
+
|
|
151
|
+
// pauseTransfer / resumeTransfer route by the same key and flip the real signal's state.
|
|
152
|
+
world.worker.pauseTransfer(uuid, "upload", "/a.txt")
|
|
153
|
+
expect(world.sync.pauseSignals[signalKey]!.isPaused()).toBe(true)
|
|
154
|
+
|
|
155
|
+
world.worker.resumeTransfer(uuid, "upload", "/a.txt")
|
|
156
|
+
expect(world.sync.pauseSignals[signalKey]!.isPaused()).toBe(false)
|
|
157
|
+
|
|
158
|
+
// An unknown key is a safe no-op: no throw and no signal conjured.
|
|
159
|
+
world.worker.pauseTransfer(uuid, "download", "/does-not-exist.txt")
|
|
160
|
+
expect(world.sync.pauseSignals["download:/does-not-exist.txt"]).toBeUndefined()
|
|
161
|
+
})
|
|
162
|
+
})
|
|
163
|
+
|
|
164
|
+
it("stopping a transfer surfaces an error, and a retry converges (I5)", async () => {
|
|
165
|
+
await withE2EWorld({ sdk, mode: "twoWay" }, async world => {
|
|
166
|
+
const uuid = world.syncPair.uuid
|
|
167
|
+
|
|
168
|
+
await writeLocal(world, "a.txt", "v1")
|
|
169
|
+
|
|
170
|
+
// Pre-register the transfer's abort controller (as if in flight) and stop it via the worker; the
|
|
171
|
+
// upload then runs against an already-aborted signal — the engine reuses the registered controller.
|
|
172
|
+
const signalKey = "upload:/a.txt"
|
|
173
|
+
world.sync.abortControllers[signalKey] = new AbortController()
|
|
174
|
+
world.worker.stopTransfer(uuid, "upload", "/a.txt")
|
|
175
|
+
|
|
176
|
+
expect(world.sync.abortControllers[signalKey]!.signal.aborted).toBe(true)
|
|
177
|
+
|
|
178
|
+
const abortedMessages = await cycle(world)
|
|
179
|
+
const uploadErrors = messagesOfType(abortedMessages, "transfer").filter(
|
|
180
|
+
message => message.data.of === "upload" && message.data.type === "error"
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
// The abort is surfaced as an upload transfer error and nothing was uploaded.
|
|
184
|
+
expect(uploadErrors.length).toBeGreaterThan(0)
|
|
185
|
+
expect((await snapshotRemoteReal(world))["/a.txt"]).toBeUndefined()
|
|
186
|
+
|
|
187
|
+
// Retry on a later cycle: clear the recorded task error (it gates the next cycle); a fresh
|
|
188
|
+
// controller is created and the upload succeeds against the real backend.
|
|
189
|
+
world.worker.resetTaskErrors(uuid)
|
|
190
|
+
await settle(world)
|
|
191
|
+
await expectConverged(world)
|
|
192
|
+
|
|
193
|
+
expect((await snapshotRemoteReal(world))["/a.txt"]).toMatchObject({ type: "file" })
|
|
194
|
+
})
|
|
195
|
+
})
|
|
196
|
+
|
|
197
|
+
it("a cycle starting with an unresolved task error re-reports it and gates without doing work (Q5)", async () => {
|
|
198
|
+
await withE2EWorld({ sdk, mode: "twoWay" }, async world => {
|
|
199
|
+
const uuid = world.syncPair.uuid
|
|
200
|
+
|
|
201
|
+
await writeLocal(world, "a.txt", "v1")
|
|
202
|
+
|
|
203
|
+
// Induce a REAL task error (no synthetic injection): stop the upload so the transfer fails and the
|
|
204
|
+
// engine records a task error for the pair.
|
|
205
|
+
world.sync.abortControllers["upload:/a.txt"] = new AbortController()
|
|
206
|
+
world.worker.stopTransfer(uuid, "upload", "/a.txt")
|
|
207
|
+
|
|
208
|
+
await cycle(world)
|
|
209
|
+
|
|
210
|
+
// The NEXT cycle is NOT error-reset, so it observes the pending task error at the very top and
|
|
211
|
+
// gates: it re-emits taskErrors + cycleRestarting and returns BEFORE cycleStarted (no work done).
|
|
212
|
+
const gated = await cycle(world)
|
|
213
|
+
|
|
214
|
+
expect(messagesOfType(gated, "taskErrors").length).toBeGreaterThan(0)
|
|
215
|
+
expect(messagesOfType(gated, "cycleRestarting").length).toBeGreaterThan(0)
|
|
216
|
+
expect(messagesOfType(gated, "cycleStarted").length).toBe(0)
|
|
217
|
+
expect((await snapshotRemoteReal(world))["/a.txt"], "still nothing uploaded while the error gates").toBeUndefined()
|
|
218
|
+
})
|
|
219
|
+
})
|
|
220
|
+
|
|
221
|
+
it("an incomplete (stopped) download is discarded, not committed as 0 bytes, and a retry converges", async () => {
|
|
222
|
+
await withE2EWorld({ sdk, mode: "twoWay" }, async world => {
|
|
223
|
+
const uuid = world.syncPair.uuid
|
|
224
|
+
|
|
225
|
+
// A remote-origin file the engine will pull down.
|
|
226
|
+
await uploadRemote(world, "r.txt", "remote-content")
|
|
227
|
+
|
|
228
|
+
// Pre-abort the download. The SDK resolves with a 0-byte staged file (the aborted stream ends
|
|
229
|
+
// cleanly), but the engine's integrity guard must DISCARD it on the size mismatch instead of
|
|
230
|
+
// moving a 0-byte file into place and caching it as synced — which would diverge permanently.
|
|
231
|
+
const signalKey = "download:/r.txt"
|
|
232
|
+
world.sync.abortControllers[signalKey] = new AbortController()
|
|
233
|
+
world.worker.stopTransfer(uuid, "download", "/r.txt")
|
|
234
|
+
|
|
235
|
+
const abortedMessages = await cycle(world)
|
|
236
|
+
|
|
237
|
+
expect(
|
|
238
|
+
messagesOfType(abortedMessages, "transfer").filter(message => message.data.of === "download" && message.data.type === "error")
|
|
239
|
+
.length
|
|
240
|
+
).toBeGreaterThan(0)
|
|
241
|
+
// The guard prevented a corrupt 0-byte file from being committed locally.
|
|
242
|
+
expect(await existsLocal(world, "r.txt"), "the incomplete download must not be committed").toBe(false)
|
|
243
|
+
|
|
244
|
+
// Retry: the FULL file downloads and the worlds converge (no lingering 0-byte divergence).
|
|
245
|
+
world.worker.resetTaskErrors(uuid)
|
|
246
|
+
await settle(world)
|
|
247
|
+
await expectConverged(world)
|
|
248
|
+
|
|
249
|
+
expect(await readLocal(world, "r.txt")).toBe("remote-content")
|
|
250
|
+
})
|
|
251
|
+
})
|
|
252
|
+
|
|
253
|
+
it("an upload failure on an already-synced, then EDITED file does not suppress the retry (F1)", async () => {
|
|
254
|
+
await withE2EWorld({ sdk, mode: "twoWay" }, async world => {
|
|
255
|
+
const uuid = world.syncPair.uuid
|
|
256
|
+
|
|
257
|
+
// First sync the file so it exists on BOTH sides and its md5 is recorded in localFileHashes.
|
|
258
|
+
await writeLocal(world, "a.txt", "v1-original")
|
|
259
|
+
await settle(world)
|
|
260
|
+
await expectConverged(world)
|
|
261
|
+
|
|
262
|
+
// Edit it (distinct size) → the modify branch, which consults the md5 dedup cache.
|
|
263
|
+
await writeLocal(world, "a.txt", "v2-edited-longer-content")
|
|
264
|
+
|
|
265
|
+
// Fail the upload of the edit by pre-aborting its transfer (no synthetic injection).
|
|
266
|
+
const signalKey = "upload:/a.txt"
|
|
267
|
+
world.sync.abortControllers[signalKey] = new AbortController()
|
|
268
|
+
world.worker.stopTransfer(uuid, "upload", "/a.txt")
|
|
269
|
+
|
|
270
|
+
const abortedMessages = await cycle(world)
|
|
271
|
+
|
|
272
|
+
expect(
|
|
273
|
+
messagesOfType(abortedMessages, "transfer").filter(message => message.data.of === "upload" && message.data.type === "error")
|
|
274
|
+
.length
|
|
275
|
+
).toBeGreaterThan(0)
|
|
276
|
+
// The edit has not landed remotely yet (still the original).
|
|
277
|
+
expect((await snapshotRemoteReal(world, { withContent: true }))["/a.txt"]).toMatchObject({ size: "v1-original".length })
|
|
278
|
+
|
|
279
|
+
// Recover. resetTaskErrors does NOT clear localFileHashes — if the failed upload had poisoned it with
|
|
280
|
+
// the edit's hash, the modify branch would now find the hash already cached and SUPPRESS the
|
|
281
|
+
// re-upload, stranding the edit locally forever. With the fix the hash is only written on success.
|
|
282
|
+
world.worker.resetTaskErrors(uuid)
|
|
283
|
+
await settle(world)
|
|
284
|
+
await expectConverged(world)
|
|
285
|
+
|
|
286
|
+
expect(await readLocal(world, "a.txt")).toBe("v2-edited-longer-content")
|
|
287
|
+
expect((await snapshotRemoteReal(world, { withContent: true }))["/a.txt"]).toMatchObject({ size: "v2-edited-longer-content".length })
|
|
288
|
+
})
|
|
289
|
+
})
|
|
290
|
+
})
|