@filen/sync 0.2.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.node-version +1 -1
- package/dist/ignorer.d.ts +6 -0
- package/dist/ignorer.js +43 -24
- package/dist/ignorer.js.map +1 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/deltas.d.ts +58 -2
- package/dist/lib/deltas.js +693 -108
- package/dist/lib/deltas.js.map +1 -1
- package/dist/lib/environment.d.ts +47 -0
- package/dist/lib/environment.js +71 -0
- package/dist/lib/environment.js.map +1 -0
- package/dist/lib/filesystems/dirTree.d.ts +70 -0
- package/dist/lib/filesystems/dirTree.js +157 -0
- package/dist/lib/filesystems/dirTree.js.map +1 -0
- package/dist/lib/filesystems/local.d.ts +18 -8
- package/dist/lib/filesystems/local.js +166 -160
- package/dist/lib/filesystems/local.js.map +1 -1
- package/dist/lib/filesystems/remote.d.ts +12 -5
- package/dist/lib/filesystems/remote.js +226 -172
- package/dist/lib/filesystems/remote.js.map +1 -1
- package/dist/lib/ipc.js +1 -2
- package/dist/lib/ipc.js.map +1 -1
- package/dist/lib/lock.js +19 -12
- package/dist/lib/lock.js.map +1 -1
- package/dist/lib/logger.js +9 -7
- package/dist/lib/logger.js.map +1 -1
- package/dist/lib/state.js +159 -63
- package/dist/lib/state.js.map +1 -1
- package/dist/lib/sync.d.ts +18 -0
- package/dist/lib/sync.js +165 -96
- package/dist/lib/sync.js.map +1 -1
- package/dist/lib/tasks.d.ts +7 -8
- package/dist/lib/tasks.js +38 -45
- package/dist/lib/tasks.js.map +1 -1
- package/dist/semaphore.d.ts +1 -0
- package/dist/semaphore.js +22 -5
- package/dist/semaphore.js.map +1 -1
- package/dist/utils.js +51 -35
- package/dist/utils.js.map +1 -1
- package/eslint.config.mjs +36 -0
- package/package.json +19 -15
- package/tests/bench/collapse.bench.ts +114 -0
- package/tests/bench/cycle.bench.ts +111 -0
- package/tests/bench/deltas.bench.ts +151 -0
- package/tests/bench/harness/fake-sync.ts +32 -0
- package/tests/bench/harness/measure.ts +276 -0
- package/tests/bench/harness/scale-world.ts +160 -0
- package/tests/bench/harness/trees.ts +275 -0
- package/tests/bench/local-scan.bench.ts +74 -0
- package/tests/bench/longrun.bench.ts +130 -0
- package/tests/bench/profile-incremental.ts +90 -0
- package/tests/bench/remote-build.bench.ts +104 -0
- package/tests/bench/render.ts +14 -0
- package/tests/bench/semaphore.bench.ts +79 -0
- package/tests/bench/state.bench.ts +85 -0
- package/tests/bench/tasks-dispatch.bench.ts +156 -0
- package/tests/conformance/virtual-fs.test.ts +213 -0
- package/tests/e2e/backup.e2e.test.ts +130 -0
- package/tests/e2e/confirm.e2e.test.ts +191 -0
- package/tests/e2e/conflict.e2e.test.ts +261 -0
- package/tests/e2e/edge.e2e.test.ts +339 -0
- package/tests/e2e/harness/account.ts +104 -0
- package/tests/e2e/harness/assert.ts +127 -0
- package/tests/e2e/harness/drive.ts +88 -0
- package/tests/e2e/harness/mutations.ts +249 -0
- package/tests/e2e/harness/world.ts +222 -0
- package/tests/e2e/ignore.e2e.test.ts +123 -0
- package/tests/e2e/lifecycle.e2e.test.ts +290 -0
- package/tests/e2e/modes.e2e.test.ts +215 -0
- package/tests/e2e/platform.e2e.test.ts +157 -0
- package/tests/e2e/property.e2e.test.ts +163 -0
- package/tests/e2e/races.e2e.test.ts +90 -0
- package/tests/e2e/regressions.e2e.test.ts +212 -0
- package/tests/e2e/resilience.e2e.test.ts +231 -0
- package/tests/e2e/special.e2e.test.ts +185 -0
- package/tests/e2e/state.e2e.test.ts +229 -0
- package/tests/e2e/sync.e2e.test.ts +222 -0
- package/tests/fakes/fake-cloud.test.ts +267 -0
- package/tests/fakes/fake-cloud.ts +1094 -0
- package/tests/fakes/virtual-fs.ts +354 -0
- package/tests/harness/known-bug.ts +17 -0
- package/tests/harness/mutations.ts +65 -0
- package/tests/harness/runner.ts +141 -0
- package/tests/harness/snapshot.ts +113 -0
- package/tests/harness/world.ts +187 -0
- package/tests/scenarios/a-baseline.test.ts +107 -0
- package/tests/scenarios/aa-races.test.ts +258 -0
- package/tests/scenarios/ab-mode-property.test.ts +189 -0
- package/tests/scenarios/ac-platform.test.ts +320 -0
- package/tests/scenarios/ad-unicode-normalization.test.ts +67 -0
- package/tests/scenarios/b-additions.test.ts +160 -0
- package/tests/scenarios/c-modifications.test.ts +194 -0
- package/tests/scenarios/d-deletions.test.ts +259 -0
- package/tests/scenarios/e-rename-move.test.ts +288 -0
- package/tests/scenarios/f-ignore-filter.test.ts +346 -0
- package/tests/scenarios/g-large-deletion.test.ts +277 -0
- package/tests/scenarios/h-resilience.test.ts +167 -0
- package/tests/scenarios/i-lifecycle.test.ts +353 -0
- package/tests/scenarios/j-state-cache.test.ts +264 -0
- package/tests/scenarios/k-scale.test.ts +202 -0
- package/tests/scenarios/l-property.test.ts +145 -0
- package/tests/scenarios/m-golden.test.ts +452 -0
- package/tests/scenarios/o-task-errors.test.ts +497 -0
- package/tests/scenarios/p-remote-originated.test.ts +306 -0
- package/tests/scenarios/q-cycle-lifecycle.test.ts +234 -0
- package/tests/scenarios/r-rename-stress.test.ts +208 -0
- package/tests/scenarios/s-upgrade-transition.test.ts +171 -0
- package/tests/scenarios/t-type-change.test.ts +144 -0
- package/tests/scenarios/u-mode-local-to-cloud.test.ts +347 -0
- package/tests/scenarios/v-mode-local-backup.test.ts +201 -0
- package/tests/scenarios/w-mode-cloud-to-local.test.ts +304 -0
- package/tests/scenarios/x-mode-cloud-backup.test.ts +201 -0
- package/tests/scenarios/y-conflict-matrix.test.ts +292 -0
- package/tests/scenarios/z-cross-ops.test.ts +285 -0
- package/tests/scenarios/zb-dir-rename-cross.test.ts +296 -0
- package/tests/scenarios/zc-crash-recovery.test.ts +189 -0
- package/tests/scenarios/zd-inode-reuse.test.ts +118 -0
- package/tests/scenarios/ze-move-into-new-dir.test.ts +130 -0
- package/tests/scenarios/zf-remote-change-unchanged-local.test.ts +81 -0
- package/tests/scenarios/zg-edit-during-scan.test.ts +68 -0
- package/tests/scenarios/zh-dir-delete-vs-child.test.ts +104 -0
- package/tests/scenarios/zi-smoke-test-outage.test.ts +78 -0
- package/tests/scenarios/zj-trash-cleanup.test.ts +133 -0
- package/tests/scenarios/zk-ignore-asymmetry.test.ts +150 -0
- package/tests/scenarios/zl-mode-atomicity.test.ts +104 -0
- package/tests/scenarios/zm-scan-concurrency.test.ts +78 -0
- package/tests/scenarios/zn-delta-ordering.test.ts +130 -0
- package/tests/scenarios/zo-download-temp-cleanup.test.ts +65 -0
- package/tests/unit/collapse-deltas.test.ts +276 -0
- package/tests/unit/dir-tree.test.ts +159 -0
- package/tests/unit/icloud.test.ts +115 -0
- package/tests/unit/ignorer-cache-regression.test.ts +70 -0
- package/tests/unit/ignorer.test.ts +63 -0
- package/tests/unit/ipc-lock.test.ts +438 -0
- package/tests/unit/lock.test.ts +135 -0
- package/tests/unit/n-unit.test.ts +632 -0
- package/tests/unit/remote-tree-unordered-regression.test.ts +101 -0
- package/tests/unit/semaphore-regression.test.ts +140 -0
- package/tests/unit/state-refencode-regression.test.ts +224 -0
- package/tests/unit/state.test.ts +809 -0
- package/tests/unit/tasks-dispatch-order-regression.test.ts +53 -0
- package/tests/unit/worker-api.test.ts +379 -0
- package/tsconfig.json +10 -1
- package/tsconfig.test.json +12 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/vitest.bench.config.ts +32 -0
- package/vitest.config.ts +27 -0
- package/vitest.e2e.config.ts +68 -0
- package/.eslintrc +0 -16
- package/jest.config.js +0 -5
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from "vitest"
|
|
2
|
+
import { createWorld, BASE_TIME, type CreateWorldOptions, type World } from "../harness/world"
|
|
3
|
+
import { type LocalItem, type LocalTree, type LocalTreeIgnored } from "../../src/lib/filesystems/local"
|
|
4
|
+
import { type RemoteItem, type RemoteTree } from "../../src/lib/filesystems/remote"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Category ZK — a remote deletion must not delete the LOCAL copy of a path that is IGNORED locally (M4).
|
|
8
|
+
*
|
|
9
|
+
* The local-deletions pass skips ignored paths (BUG-006): a path that is physically present locally but
|
|
10
|
+
* excluded from the scanned tree — over-long name, invalid path, a default-ignore that grew across an
|
|
11
|
+
* upgrade, a case-duplicate — must keep its cloud copy, never be reported as a deletion. The symmetric
|
|
12
|
+
* remote-deletions pass had no such guard: when that same ignored path's REMOTE copy was deleted, it
|
|
13
|
+
* emitted deleteLocal* and wiped the on-disk file the user never asked to sync.
|
|
14
|
+
*
|
|
15
|
+
* The end-of-process ignore filter does NOT cover this: it only consults the .filenignore matcher (and the
|
|
16
|
+
* dot-file flag), never the nameLength / pathLength / invalidPath / defaultIgnore / duplicate reasons that
|
|
17
|
+
* `currentLocalTreeIgnored` carries — so a .filenignore-based test would be masked and falsely pass. These
|
|
18
|
+
* tests therefore drive deltas.process() directly with an injected `currentLocalTreeIgnored` entry whose
|
|
19
|
+
* path the (empty) ignorer does NOT match, modelling exactly those non-.filenignore reasons.
|
|
20
|
+
*
|
|
21
|
+
* This is client-side delta-attribution logic; the backend's only role (a path deleted remotely) is already
|
|
22
|
+
* covered by the live deletion e2e tests, and the trigger — a base path that became ignored for a
|
|
23
|
+
* non-.filenignore reason — cannot be forced deterministically against the real backend, so there is no
|
|
24
|
+
* e2e counterpart (boundary noted in tests/e2e/regressions.e2e.test.ts).
|
|
25
|
+
*/
|
|
26
|
+
const FAKE_TIMERS = ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"] as const
|
|
27
|
+
|
|
28
|
+
async function withWorld(options: CreateWorldOptions, body: (world: World) => Promise<void>): Promise<void> {
|
|
29
|
+
vi.useFakeTimers({ toFake: [...FAKE_TIMERS] })
|
|
30
|
+
vi.setSystemTime(BASE_TIME)
|
|
31
|
+
|
|
32
|
+
try {
|
|
33
|
+
const world = await createWorld(options)
|
|
34
|
+
|
|
35
|
+
await body(world)
|
|
36
|
+
} finally {
|
|
37
|
+
vi.useRealTimers()
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function localFile(path: string, inode: number): LocalItem {
|
|
42
|
+
return { type: "file", path, inode, size: 100, lastModified: 1_700_000_000_000, creation: 1_690_000_000_000 }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function localDir(path: string, inode: number): LocalItem {
|
|
46
|
+
return { type: "directory", path, inode, size: 0, lastModified: 1_700_000_000_000, creation: 1_690_000_000_000 }
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function remoteFile(path: string, uuid: string, name: string): RemoteItem {
|
|
50
|
+
return {
|
|
51
|
+
type: "file",
|
|
52
|
+
uuid,
|
|
53
|
+
name,
|
|
54
|
+
size: 100,
|
|
55
|
+
mime: "text/plain",
|
|
56
|
+
lastModified: 1_700_000_000_000,
|
|
57
|
+
version: 2,
|
|
58
|
+
chunks: 1,
|
|
59
|
+
key: `key-${uuid}`,
|
|
60
|
+
bucket: "bucket",
|
|
61
|
+
region: "region",
|
|
62
|
+
path
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function remoteDir(path: string, uuid: string, name: string): RemoteItem {
|
|
67
|
+
return { type: "directory", uuid, name, size: 0, path }
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const ignoredEntry = (path: string): LocalTreeIgnored => ({ localPath: `/local${path}`, relativePath: path, reason: "nameLength" })
|
|
71
|
+
|
|
72
|
+
// A base where `path` was synced on both sides; the remote copy is now gone. The local copy is present on
|
|
73
|
+
// disk but the caller decides (via currentLocalTree / currentLocalTreeIgnored) whether the scan ignored it.
|
|
74
|
+
function baseWith(path: string, kind: "file" | "dir"): { previousLocalTree: LocalTree; previousRemoteTree: RemoteTree } {
|
|
75
|
+
const li = kind === "dir" ? localDir(path, 101) : localFile(path, 101)
|
|
76
|
+
const ri = kind === "dir" ? remoteDir(path, "u-1", path.slice(1)) : remoteFile(path, "u-1", path.slice(1))
|
|
77
|
+
|
|
78
|
+
return {
|
|
79
|
+
previousLocalTree: { tree: { [path]: li }, inodes: { 101: li }, size: 1 },
|
|
80
|
+
previousRemoteTree: { tree: { [path]: ri }, uuids: { "u-1": ri }, size: 1 }
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const EMPTY_LOCAL: LocalTree = { tree: {}, inodes: {}, size: 0 }
|
|
85
|
+
const EMPTY_REMOTE: RemoteTree = { tree: {}, uuids: {}, size: 0 }
|
|
86
|
+
|
|
87
|
+
describe("Category ZK — a remote deletion must not wipe a locally-ignored file", () => {
|
|
88
|
+
it("ZK1: an IGNORED local file whose remote copy was deleted is NOT deleted locally", async () => {
|
|
89
|
+
await withWorld({ mode: "twoWay" }, async world => {
|
|
90
|
+
const { previousLocalTree, previousRemoteTree } = baseWith("/over-long.txt", "file")
|
|
91
|
+
|
|
92
|
+
// Current state: remote deleted it (absent remotely), and the local scan IGNORED it (absent from the
|
|
93
|
+
// scanned tree, recorded in currentLocalTreeIgnored). The file is still on disk.
|
|
94
|
+
const { deltas } = await world.sync.deltas.process({
|
|
95
|
+
currentLocalTree: EMPTY_LOCAL,
|
|
96
|
+
currentRemoteTree: EMPTY_REMOTE,
|
|
97
|
+
previousLocalTree,
|
|
98
|
+
previousRemoteTree,
|
|
99
|
+
currentLocalTreeErrors: [],
|
|
100
|
+
currentLocalTreeIgnored: [ignoredEntry("/over-long.txt")]
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
const wipes = deltas.filter(d => d.type === "deleteLocalFile" && d.path === "/over-long.txt")
|
|
104
|
+
|
|
105
|
+
expect(wipes, "a remote delete must not propagate onto an ignored local file").toHaveLength(0)
|
|
106
|
+
// And it must not have been mistaken for a local deletion to push to the cloud either (BUG-006).
|
|
107
|
+
expect(deltas.filter(d => d.type === "deleteRemoteFile" && d.path === "/over-long.txt")).toHaveLength(0)
|
|
108
|
+
})
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
it("ZK2: an IGNORED local directory whose remote copy was deleted is NOT deleted locally", async () => {
|
|
112
|
+
await withWorld({ mode: "twoWay" }, async world => {
|
|
113
|
+
const { previousLocalTree, previousRemoteTree } = baseWith("/over-long-dir", "dir")
|
|
114
|
+
|
|
115
|
+
const { deltas } = await world.sync.deltas.process({
|
|
116
|
+
currentLocalTree: EMPTY_LOCAL,
|
|
117
|
+
currentRemoteTree: EMPTY_REMOTE,
|
|
118
|
+
previousLocalTree,
|
|
119
|
+
previousRemoteTree,
|
|
120
|
+
currentLocalTreeErrors: [],
|
|
121
|
+
currentLocalTreeIgnored: [ignoredEntry("/over-long-dir")]
|
|
122
|
+
})
|
|
123
|
+
|
|
124
|
+
expect(deltas.filter(d => d.type === "deleteLocalDirectory" && d.path === "/over-long-dir")).toHaveLength(0)
|
|
125
|
+
})
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
it("ZK3 (no over-suppression): a NON-ignored local file whose remote copy was deleted IS still deleted locally", async () => {
|
|
129
|
+
await withWorld({ mode: "twoWay" }, async world => {
|
|
130
|
+
const { previousLocalTree, previousRemoteTree } = baseWith("/normal.txt", "file")
|
|
131
|
+
const li = localFile("/normal.txt", 101)
|
|
132
|
+
|
|
133
|
+
// Same remote deletion, but the local file is present in the scanned tree (not ignored). The normal
|
|
134
|
+
// remote->local deletion propagation must still happen — the guard must not over-suppress.
|
|
135
|
+
const { deltas } = await world.sync.deltas.process({
|
|
136
|
+
currentLocalTree: { tree: { "/normal.txt": li }, inodes: { 101: li }, size: 1 },
|
|
137
|
+
currentRemoteTree: EMPTY_REMOTE,
|
|
138
|
+
previousLocalTree,
|
|
139
|
+
previousRemoteTree,
|
|
140
|
+
currentLocalTreeErrors: [],
|
|
141
|
+
currentLocalTreeIgnored: []
|
|
142
|
+
})
|
|
143
|
+
|
|
144
|
+
expect(
|
|
145
|
+
deltas.filter(d => d.type === "deleteLocalFile" && d.path === "/normal.txt"),
|
|
146
|
+
"a normal remote deletion must still propagate to the local copy"
|
|
147
|
+
).toHaveLength(1)
|
|
148
|
+
})
|
|
149
|
+
})
|
|
150
|
+
})
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from "vitest"
|
|
2
|
+
import { createWorld, BASE_TIME, type CreateWorldOptions, type World } from "../harness/world"
|
|
3
|
+
import { type LocalItem, type LocalTree } from "../../src/lib/filesystems/local"
|
|
4
|
+
import { type RemoteItem, type RemoteTree } from "../../src/lib/filesystems/remote"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Category ZL — one cycle must compute its whole delta set under a SINGLE mode (M6).
|
|
8
|
+
*
|
|
9
|
+
* process() is async (it awaits a content hash on the same-size/newer-mtime path) and reads the pair's
|
|
10
|
+
* mode in ~20 places across its passes. updateMode() mutates sync.mode synchronously from the main thread
|
|
11
|
+
* at any moment, so a mode switch landing during one of those awaits used to split a single cycle across
|
|
12
|
+
* two modes — e.g. the local-deletions pass running as twoWay while a later pass runs as cloudToLocal —
|
|
13
|
+
* producing a self-contradictory delta set (here: a brand-new local file silently dropped instead of
|
|
14
|
+
* uploaded). The fix snapshots the mode once at entry and reports it back.
|
|
15
|
+
*
|
|
16
|
+
* The mode flip is injected through the one awaited dependency inside the passes (createFileHash), which is
|
|
17
|
+
* exactly the real race window. This is client-side delta logic with no backend role, so there is no e2e
|
|
18
|
+
* counterpart (boundary noted in tests/e2e/regressions.e2e.test.ts).
|
|
19
|
+
*/
|
|
20
|
+
const FAKE_TIMERS = ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"] as const
|
|
21
|
+
|
|
22
|
+
async function withWorld(options: CreateWorldOptions, body: (world: World) => Promise<void>): Promise<void> {
|
|
23
|
+
vi.useFakeTimers({ toFake: [...FAKE_TIMERS] })
|
|
24
|
+
vi.setSystemTime(BASE_TIME)
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
const world = await createWorld(options)
|
|
28
|
+
|
|
29
|
+
await body(world)
|
|
30
|
+
} finally {
|
|
31
|
+
vi.useRealTimers()
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function localFile(path: string, inode: number, lastModified = 1_700_000_000_000): LocalItem {
|
|
36
|
+
return { type: "file", path, inode, size: 100, lastModified, creation: 1_690_000_000_000 }
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function remoteFile(path: string, uuid: string, name: string): RemoteItem {
|
|
40
|
+
return {
|
|
41
|
+
type: "file",
|
|
42
|
+
uuid,
|
|
43
|
+
name,
|
|
44
|
+
size: 100,
|
|
45
|
+
mime: "text/plain",
|
|
46
|
+
lastModified: 1_700_000_000_000,
|
|
47
|
+
version: 2,
|
|
48
|
+
chunks: 1,
|
|
49
|
+
key: `key-${uuid}`,
|
|
50
|
+
bucket: "bucket",
|
|
51
|
+
region: "region",
|
|
52
|
+
path
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const EMPTY_REMOTE: RemoteTree = { tree: {}, uuids: {}, size: 0 }
|
|
57
|
+
|
|
58
|
+
describe("Category ZL — a mid-cycle mode switch must not split one delta computation", () => {
|
|
59
|
+
it("ZL1: a mode switch DURING process() does not split the cycle across two modes", async () => {
|
|
60
|
+
await withWorld({ mode: "twoWay" }, async world => {
|
|
61
|
+
// "hashme.txt" forces the single awaited content hash inside the delta passes; the stub flips the
|
|
62
|
+
// pair's mode mid-flight, exactly as a racing updateMode() would. (Its own delta is irrelevant.)
|
|
63
|
+
const hashSpy = vi.spyOn(world.sync.localFileSystem, "createFileHash").mockImplementation(async () => {
|
|
64
|
+
world.sync.mode = "cloudToLocal"
|
|
65
|
+
|
|
66
|
+
return "cached-hash"
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
world.sync.localFileHashes["/hashme.txt"] = "cached-hash"
|
|
70
|
+
|
|
71
|
+
const baseHash = localFile("/hashme.txt", 201, 1_700_000_000_000)
|
|
72
|
+
const currentHash = localFile("/hashme.txt", 201, 1_700_000_010_000) // same size, newer mtime -> needs the hash
|
|
73
|
+
const remoteHash = remoteFile("/hashme.txt", "u-h", "hashme.txt")
|
|
74
|
+
const newLocal = localFile("/localnew.txt", 202)
|
|
75
|
+
|
|
76
|
+
const currentLocalTree: LocalTree = {
|
|
77
|
+
tree: { "/hashme.txt": currentHash, "/localnew.txt": newLocal },
|
|
78
|
+
inodes: { 201: currentHash, 202: newLocal },
|
|
79
|
+
size: 2
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const { deltas, mode } = await world.sync.deltas.process({
|
|
83
|
+
currentLocalTree,
|
|
84
|
+
currentRemoteTree: EMPTY_REMOTE, // both remote copies are gone
|
|
85
|
+
previousLocalTree: { tree: { "/hashme.txt": baseHash }, inodes: { 201: baseHash }, size: 1 },
|
|
86
|
+
previousRemoteTree: { tree: { "/hashme.txt": remoteHash }, uuids: { "u-h": remoteHash }, size: 1 },
|
|
87
|
+
currentLocalTreeErrors: [],
|
|
88
|
+
currentLocalTreeIgnored: []
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
expect(hashSpy, "the scenario must actually reach the awaited hash (the race window)").toHaveBeenCalled()
|
|
92
|
+
|
|
93
|
+
// The new local file is attributed by the local-additions pass, which runs AFTER that await. Under the
|
|
94
|
+
// entry mode (twoWay) it must upload; a live read of the flipped mode (cloudToLocal) would skip the
|
|
95
|
+
// whole pass and silently drop the file. The single-mode snapshot keeps the cycle on twoWay.
|
|
96
|
+
expect(
|
|
97
|
+
deltas.filter(d => d.type === "uploadFile" && d.path === "/localnew.txt"),
|
|
98
|
+
"a pass after the mid-cycle flip must still use the entry mode"
|
|
99
|
+
).toHaveLength(1)
|
|
100
|
+
expect(deltas.filter(d => d.type === "deleteLocalFile" && d.path === "/localnew.txt")).toHaveLength(0)
|
|
101
|
+
expect(mode, "process() reports the one mode the whole cycle ran under").toBe("twoWay")
|
|
102
|
+
})
|
|
103
|
+
})
|
|
104
|
+
})
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"
|
|
2
|
+
import { createWorld, BASE_TIME, LOCAL_ROOT } from "../harness/world"
|
|
3
|
+
import { LOCAL_SCAN_CONCURRENCY } from "../../src/lib/filesystems/local"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Category ZM — the local tree scan must bound how many filesystem stat operations it launches at once.
|
|
7
|
+
*
|
|
8
|
+
* getDirectoryTree() fans an `lstat` (+ `access`) out over every glob entry. The walk used to map ALL
|
|
9
|
+
* entries to promises in one go — `entries.map(async …)` eagerly invokes every async body, so a tree
|
|
10
|
+
* with N entries allocated N pending promises and N in-flight stat results simultaneously. The chunk
|
|
11
|
+
* size handed to the awaiting helper only batched the AWAIT, never the launch, so peak concurrency (and
|
|
12
|
+
* peak memory) was unbounded — O(N) on top of the unavoidable result tree, with matching pressure on the
|
|
13
|
+
* libuv thread pool and open file descriptors. Walking the entries in fixed-size batches caps the
|
|
14
|
+
* in-flight fan-out at LOCAL_SCAN_CONCURRENCY.
|
|
15
|
+
*
|
|
16
|
+
* The peak is measured by wrapping the per-entry `lstat`: the scan routes those through environment.fs,
|
|
17
|
+
* whereas FastGlob's own traversal uses the separate globFs adapter — so the wrapper observes ONLY the
|
|
18
|
+
* fan-out. Because each async body runs synchronously up to its first `await`, every body in a batch
|
|
19
|
+
* increments the in-flight counter before any stat resolves, so the observed peak equals the batch size
|
|
20
|
+
* deterministically, with no reliance on timing.
|
|
21
|
+
*/
|
|
22
|
+
describe("Category ZM — the local scan bounds concurrent stat operations", () => {
|
|
23
|
+
beforeEach(() => {
|
|
24
|
+
// Match the scenario harness: fake only the engine's timers + clock, leaving microtasks/setImmediate
|
|
25
|
+
// real so FastGlob's async traversal drains while the clock is frozen.
|
|
26
|
+
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"] })
|
|
27
|
+
vi.setSystemTime(BASE_TIME)
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
afterEach(() => {
|
|
31
|
+
vi.useRealTimers()
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
it("ZM1: getDirectoryTree never launches more than LOCAL_SCAN_CONCURRENCY lstat operations at once", async () => {
|
|
35
|
+
// Comfortably more entries than the bound, and not a round multiple of it, so the buggy unbounded
|
|
36
|
+
// fan-out (peak = fileCount) is clearly distinguishable from the bounded one (peak = the bound).
|
|
37
|
+
const fileCount = LOCAL_SCAN_CONCURRENCY * 2 + 137
|
|
38
|
+
const initialLocal: Record<string, string> = {}
|
|
39
|
+
|
|
40
|
+
for (let i = 0; i < fileCount; i++) {
|
|
41
|
+
initialLocal[`${LOCAL_ROOT}/f${i}.txt`] = "x"
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const world = await createWorld({ mode: "twoWay", initialLocal })
|
|
45
|
+
|
|
46
|
+
// Wrap the per-entry lstat to record peak concurrent in-flight calls. environment.fs IS world.vfs.fs,
|
|
47
|
+
// and the scan calls it once per entry; FastGlob's walk uses globFs, so this counts only the fan-out.
|
|
48
|
+
const realLstat = world.vfs.fs.lstat.bind(world.vfs.fs)
|
|
49
|
+
let inFlight = 0
|
|
50
|
+
let maxInFlight = 0
|
|
51
|
+
|
|
52
|
+
const countingLstat = (async (path: string) => {
|
|
53
|
+
inFlight++
|
|
54
|
+
maxInFlight = Math.max(maxInFlight, inFlight)
|
|
55
|
+
|
|
56
|
+
try {
|
|
57
|
+
return await realLstat(path)
|
|
58
|
+
} finally {
|
|
59
|
+
inFlight--
|
|
60
|
+
}
|
|
61
|
+
}) as typeof world.vfs.fs.lstat
|
|
62
|
+
|
|
63
|
+
// SyncFS declares lstat readonly; swap in the counting wrapper through a mutable view (the underlying
|
|
64
|
+
// memfs object is a plain mutable JS object — the readonly is only a compile-time annotation).
|
|
65
|
+
;(world.vfs.fs as { lstat: typeof world.vfs.fs.lstat }).lstat = countingLstat
|
|
66
|
+
|
|
67
|
+
const tree = await world.sync.localFileSystem.getDirectoryTree()
|
|
68
|
+
|
|
69
|
+
// Sanity: the scan visited every file (otherwise a no-op scan would trivially satisfy the bound).
|
|
70
|
+
expect(tree.result.size).toBe(fileCount)
|
|
71
|
+
// The fix: peak in-flight stat operations are capped at the bound. The buggy fan-out peaked at
|
|
72
|
+
// fileCount (every entry launched at once), so it fails this both as "> bound" and "≠ bound".
|
|
73
|
+
expect(
|
|
74
|
+
maxInFlight,
|
|
75
|
+
`scan should cap concurrent lstat fan-out at ${LOCAL_SCAN_CONCURRENCY}, but ${maxInFlight} of ${fileCount} entries were in flight at once`
|
|
76
|
+
).toBe(LOCAL_SCAN_CONCURRENCY)
|
|
77
|
+
})
|
|
78
|
+
})
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from "vitest"
|
|
2
|
+
import { createWorld, BASE_TIME, type CreateWorldOptions, type World } from "../harness/world"
|
|
3
|
+
import { type LocalItem, type LocalTree } from "../../src/lib/filesystems/local"
|
|
4
|
+
import { type RemoteItem, type RemoteTree } from "../../src/lib/filesystems/remote"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Category ZN — deltas.process() must return its deltas ordered shallow-to-deep (ascending path depth) so
|
|
8
|
+
* the executor always creates/renames/deletes a parent directory before touching its children.
|
|
9
|
+
*
|
|
10
|
+
* process() used to sort by depth TWICE: once over the full PRE-collapse set and again over the collapsed
|
|
11
|
+
* result. collapseDeltas only ever filters (drops subsumed descendants) or rewrites a rename's `from` — it
|
|
12
|
+
* never reorders and never changes a delta's `path` — and it indexes the renamed/deleted directories up
|
|
13
|
+
* front, so it is independent of input order. The pre-collapse sort was therefore redundant: a single sort
|
|
14
|
+
* on the (smaller) collapsed set yields the identical order, with each delta's depth computed once instead
|
|
15
|
+
* of re-splitting the path on every comparison. This locks the observable contract — the returned deltas
|
|
16
|
+
* are depth-ascending and child deletes still fold into their ancestor — so the dropped sort cannot
|
|
17
|
+
* silently regress ordering.
|
|
18
|
+
*
|
|
19
|
+
* Behavior-preserving performance change with no backend-observable effect (the executor consumes the same
|
|
20
|
+
* ordered set); correctness against the live backend is covered by every convergence e2e scenario.
|
|
21
|
+
*/
|
|
22
|
+
const FAKE_TIMERS = ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"] as const
|
|
23
|
+
|
|
24
|
+
async function withWorld(options: CreateWorldOptions, body: (world: World) => Promise<void>): Promise<void> {
|
|
25
|
+
vi.useFakeTimers({ toFake: [...FAKE_TIMERS] })
|
|
26
|
+
vi.setSystemTime(BASE_TIME)
|
|
27
|
+
|
|
28
|
+
try {
|
|
29
|
+
const world = await createWorld(options)
|
|
30
|
+
|
|
31
|
+
await body(world)
|
|
32
|
+
} finally {
|
|
33
|
+
vi.useRealTimers()
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function localFile(path: string, inode: number): LocalItem {
|
|
38
|
+
return { type: "file", path, inode, size: 100, lastModified: 1_700_000_000_000, creation: 1_690_000_000_000 }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function localDir(path: string, inode: number): LocalItem {
|
|
42
|
+
return { type: "directory", path, inode, size: 0, lastModified: 1_700_000_000_000, creation: 1_690_000_000_000 }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function remoteFile(path: string, uuid: string): RemoteItem {
|
|
46
|
+
return {
|
|
47
|
+
type: "file",
|
|
48
|
+
uuid,
|
|
49
|
+
name: path.slice(path.lastIndexOf("/") + 1),
|
|
50
|
+
size: 100,
|
|
51
|
+
mime: "text/plain",
|
|
52
|
+
lastModified: 1_700_000_000_000,
|
|
53
|
+
version: 2,
|
|
54
|
+
chunks: 1,
|
|
55
|
+
key: `key-${uuid}`,
|
|
56
|
+
bucket: "bucket",
|
|
57
|
+
region: "region",
|
|
58
|
+
path
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function remoteDir(path: string, uuid: string): RemoteItem {
|
|
63
|
+
return { type: "directory", uuid, name: path.slice(path.lastIndexOf("/") + 1), size: 0, path }
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
describe("Category ZN — process() returns depth-ordered deltas after a single post-collapse sort", () => {
|
|
67
|
+
it("ZN1: a directory delete that subsumes its child, alongside nested adds, yields depth-ascending output", async () => {
|
|
68
|
+
await withWorld({ mode: "twoWay" }, async world => {
|
|
69
|
+
// Base: /gone and /gone/inner.txt were synced on both sides.
|
|
70
|
+
const goneDir = localDir("/gone", 1)
|
|
71
|
+
const goneInner = localFile("/gone/inner.txt", 2)
|
|
72
|
+
const goneDirR = remoteDir("/gone", "u-gone")
|
|
73
|
+
const goneInnerR = remoteFile("/gone/inner.txt", "u-gone-inner")
|
|
74
|
+
|
|
75
|
+
const previousLocalTree: LocalTree = {
|
|
76
|
+
tree: { "/gone": goneDir, "/gone/inner.txt": goneInner },
|
|
77
|
+
inodes: { 1: goneDir, 2: goneInner },
|
|
78
|
+
size: 2
|
|
79
|
+
}
|
|
80
|
+
const previousRemoteTree: RemoteTree = {
|
|
81
|
+
tree: { "/gone": goneDirR, "/gone/inner.txt": goneInnerR },
|
|
82
|
+
uuids: { "u-gone": goneDirR, "u-gone-inner": goneInnerR },
|
|
83
|
+
size: 2
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Current local: /gone was deleted, and a fresh nested subtree /new/sub/deep.txt was added — so the
|
|
87
|
+
// raw deltas span depths 2..4 and include a child delete that must fold into its ancestor dir delete.
|
|
88
|
+
// The subtree is inserted DEEP-FIRST: the add pass emits in tree-iteration order, so the raw (pre-sort)
|
|
89
|
+
// deltas come out deepest-first — only the final depth sort restores parents-before-children, which is
|
|
90
|
+
// exactly what this test must catch if it ever regresses.
|
|
91
|
+
const newDeep = localFile("/new/sub/deep.txt", 12)
|
|
92
|
+
const newSub = localDir("/new/sub", 11)
|
|
93
|
+
const newDir = localDir("/new", 10)
|
|
94
|
+
const currentLocalTree: LocalTree = {
|
|
95
|
+
tree: { "/new/sub/deep.txt": newDeep, "/new/sub": newSub, "/new": newDir },
|
|
96
|
+
inodes: { 12: newDeep, 11: newSub, 10: newDir },
|
|
97
|
+
size: 3
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Current remote: unchanged (still holds /gone + child), so the local delete propagates to the remote.
|
|
101
|
+
const currentRemoteTree: RemoteTree = {
|
|
102
|
+
tree: { "/gone": goneDirR, "/gone/inner.txt": goneInnerR },
|
|
103
|
+
uuids: { "u-gone": goneDirR, "u-gone-inner": goneInnerR },
|
|
104
|
+
size: 2
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const { deltas } = await world.sync.deltas.process({
|
|
108
|
+
currentLocalTree,
|
|
109
|
+
currentRemoteTree,
|
|
110
|
+
previousLocalTree,
|
|
111
|
+
previousRemoteTree,
|
|
112
|
+
currentLocalTreeErrors: [],
|
|
113
|
+
currentLocalTreeIgnored: []
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
// The child delete folded into the ancestor directory delete (collapse ran).
|
|
117
|
+
expect(deltas.find(delta => delta.path === "/gone/inner.txt"), "child delete must be subsumed by /gone").toBeUndefined()
|
|
118
|
+
// The collapsed parent delete and the deepest add are both present, so the set spans depths 2..4.
|
|
119
|
+
expect(deltas.some(delta => delta.type === "deleteRemoteDirectory" && delta.path === "/gone")).toBe(true)
|
|
120
|
+
expect(deltas.some(delta => delta.type === "uploadFile" && delta.path === "/new/sub/deep.txt")).toBe(true)
|
|
121
|
+
|
|
122
|
+
// The contract P5 must preserve: the returned deltas are ordered ascending by path depth (parents
|
|
123
|
+
// before children), produced by the single post-collapse sort.
|
|
124
|
+
const depths = deltas.map(delta => delta.path.split("/").length)
|
|
125
|
+
expect(depths, `deltas must be depth-ascending, got: ${deltas.map(delta => delta.path).join(", ")}`).toEqual(
|
|
126
|
+
[...depths].sort((a, b) => a - b)
|
|
127
|
+
)
|
|
128
|
+
})
|
|
129
|
+
})
|
|
130
|
+
})
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest"
|
|
2
|
+
import { runScenario, runCycle, control } from "../harness/runner"
|
|
3
|
+
import { LOCAL_ROOT } from "../harness/world"
|
|
4
|
+
import { LOCAL_TRASH_NAME } from "../../src/constants"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Category ZO — a download that fails AFTER staging its file must not orphan the staged temp (L1).
|
|
8
|
+
*
|
|
9
|
+
* download() streams the remote file into a uuid-named temp under the local trash dir, then commits it
|
|
10
|
+
* with a single move into place. The size-mismatch guard already discards the temp before throwing, but
|
|
11
|
+
* the general catch (a rejected/partial transfer, or a failure during the commit move) re-threw without
|
|
12
|
+
* removing it — so every failed download leaked a temp file into .filen.trash.local. The periodic trash
|
|
13
|
+
* sweep only ages them out later, so under a flaky connection these accumulate and waste disk in the
|
|
14
|
+
* meantime. The fix discards the temp in the catch path too.
|
|
15
|
+
*
|
|
16
|
+
* Reproduced deterministically by letting the staging succeed and then failing the commit move (the only
|
|
17
|
+
* move whose SOURCE is in the trash dir), which lands execution in the catch with a fully-staged temp.
|
|
18
|
+
*
|
|
19
|
+
* No e2e counterpart: the leak is a LOCAL artifact and the trigger is a failure of the local commit move,
|
|
20
|
+
* which the real backend cannot be made to produce on demand (a backend-side download abort takes the
|
|
21
|
+
* size-mismatch path, which already cleaned up). Boundary noted in tests/e2e/regressions.e2e.test.ts.
|
|
22
|
+
*/
|
|
23
|
+
describe("Category ZO — a failed download does not orphan its staged temp file", () => {
|
|
24
|
+
it("ZO1: a download that fails while committing its staged file leaves no temp behind", async () => {
|
|
25
|
+
let moveAttempted = false
|
|
26
|
+
|
|
27
|
+
const result = await runScenario({
|
|
28
|
+
name: "ZO1",
|
|
29
|
+
mode: "cloudToLocal",
|
|
30
|
+
initialRemote: { "/file.txt": "hello world contents" },
|
|
31
|
+
steps: [
|
|
32
|
+
// Fail the commit move (staging dir -> final path) so the download lands in its catch with a
|
|
33
|
+
// fully-staged temp. Only the download's commit move has a trash-dir source, so nothing else is hit.
|
|
34
|
+
control(world => {
|
|
35
|
+
const realMove = world.vfs.fs.move.bind(world.vfs.fs)
|
|
36
|
+
|
|
37
|
+
;(world.vfs.fs as { move: typeof world.vfs.fs.move }).move = (async (
|
|
38
|
+
src: string,
|
|
39
|
+
dest: string,
|
|
40
|
+
opts?: { overwrite?: boolean }
|
|
41
|
+
) => {
|
|
42
|
+
if (src.includes(LOCAL_TRASH_NAME)) {
|
|
43
|
+
moveAttempted = true
|
|
44
|
+
|
|
45
|
+
throw new Error("simulated disk error committing the download")
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return realMove(src, dest, opts)
|
|
49
|
+
}) as typeof world.vfs.fs.move
|
|
50
|
+
}),
|
|
51
|
+
runCycle()
|
|
52
|
+
]
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
// The failure path was actually exercised: the staged temp's commit move was attempted (a temp existed).
|
|
56
|
+
expect(moveAttempted, "the download must have staged a temp and attempted to commit it").toBe(true)
|
|
57
|
+
// The download did not land (its commit failed), so the local file is absent...
|
|
58
|
+
expect(result.world.vfs.ifs.existsSync(`${LOCAL_ROOT}/file.txt`)).toBe(false)
|
|
59
|
+
// ...and crucially the staged temp file is NOT orphaned in the local trash directory.
|
|
60
|
+
const trashDir = `${LOCAL_ROOT}/${LOCAL_TRASH_NAME}`
|
|
61
|
+
const orphaned = result.world.vfs.ifs.existsSync(trashDir) ? result.world.vfs.ifs.readdirSync(trashDir) : []
|
|
62
|
+
|
|
63
|
+
expect(orphaned, "a failed download must not leak its staged temp file").toEqual([])
|
|
64
|
+
})
|
|
65
|
+
})
|