@filen/sync 0.2.1 → 0.3.2

package/tests/unit/remote-tree-unordered-regression.test.ts

@@ -0,0 +1,101 @@
+import { describe, it, expect } from "vitest"
+import { createWorld, type World } from "../harness/world"
+import { type SyncDirTreeFetcher } from "../../src/lib/filesystems/dirTree"
+
+/**
+ * Added regression tests for the order-independent remote tree build. The /v3/dir/tree API can return
+ * its files/folders arrays UNORDERED (no guarantee a parent precedes its children, nor that folders[0]
+ * is the sync root). The old builder built paths incrementally in array order and required
+ * folders[0].parent === "base", so an out-of-order response corrupted child paths or THREW. The fake
+ * cloud always emits BFS order, so the mocked suite never exercised this — these tests inject a
+ * hand-built response (ordered and shuffled) directly.
+ *
+ * On the OLD code the shuffled case throws "Invalid base folder parent" (the build rejects → empty
+ * changed:true result is never produced) — verified RED by stashing the rewrite. NEW FILE.
+ */
+
+type Folder = [string, string, string]
+type File = [string, string, string, number, string, string, number, number]
+
+function folderMeta(name: string): string {
+	return JSON.stringify({ name })
+}
+
+function fileMeta(name: string): string {
+	return JSON.stringify({ name, size: 10, mime: "text/plain", key: "k", lastModified: 1_700_000_000_000, creation: 1_690_000_000_000 })
+}
+
+/**
+ * A small tree:  root / a / b / c.txt , plus root / x.txt . Returns the folder+file tuples in BFS
+ * (parent-first) order; the caller can shuffle them.
+ */
+function buildTuples(rootUUID: string): { folders: Folder[]; files: File[] } {
+	const folders: Folder[] = [
+		[rootUUID, folderMeta("Sync"), "base"],
+		["uuid-a", folderMeta("a"), rootUUID],
+		["uuid-b", folderMeta("b"), "uuid-a"]
+	]
+	const files: File[] = [
+		["uuid-c", "bucket", "region", 1, "uuid-b", fileMeta("c.txt"), 2, 1_700_000_000_000],
+		["uuid-x", "bucket", "region", 1, rootUUID, fileMeta("x.txt"), 2, 1_700_000_000_000]
+	]
+
+	return { folders, files }
+}
+
+function inject(world: World, folders: Folder[], files: File[]): void {
+	const fetcher: SyncDirTreeFetcher = async () => ({ files, folders, raw: "" }) as unknown as Awaited<ReturnType<SyncDirTreeFetcher>>
+
+	world.sync.environment.fetchDirTree = fetcher
+}
+
+describe("RemoteFileSystem.getDirectoryTree — order-independent build", () => {
+	it("builds the correct tree from an ORDERED response (baseline)", async () => {
+		const world = await createWorld({ mode: "twoWay" })
+		const { folders, files } = buildTuples(world.cloud.controls.rootUUID)
+
+		inject(world, folders, files)
+
+		const result = await world.sync.remoteFileSystem.getDirectoryTree(true)
+
+		expect(result.result.tree["/a"]).toMatchObject({ type: "directory", path: "/a" })
+		expect(result.result.tree["/a/b"]).toMatchObject({ type: "directory", path: "/a/b" })
+		expect(result.result.tree["/a/b/c.txt"]).toMatchObject({ type: "file", path: "/a/b/c.txt" })
+		expect(result.result.tree["/x.txt"]).toMatchObject({ type: "file", path: "/x.txt" })
+		expect(result.result.size).toBe(4)
+	})
+
+	it("builds the SAME tree from a SHUFFLED response (child before parent, root not first)", async () => {
+		const world = await createWorld({ mode: "twoWay" })
+		const { folders, files } = buildTuples(world.cloud.controls.rootUUID)
+
+		// Reverse the folders so a child (b) precedes its parent (a) and the root is LAST — the exact shape
+		// that made the old builder throw / mis-path.
+		const shuffledFolders: Folder[] = [folders[2]!, folders[1]!, folders[0]!]
+		const shuffledFiles: File[] = [files[1]!, files[0]!]
+
+		inject(world, shuffledFolders, shuffledFiles)
+
+		const result = await world.sync.remoteFileSystem.getDirectoryTree(true)
+
+		// Despite the scrambled input, every path resolves correctly via the parent-chain walk.
+		expect(result.result.tree["/a"]).toMatchObject({ type: "directory", path: "/a" })
+		expect(result.result.tree["/a/b"]).toMatchObject({ type: "directory", path: "/a/b" })
+		expect(result.result.tree["/a/b/c.txt"]).toMatchObject({ type: "file", path: "/a/b/c.txt" })
+		expect(result.result.tree["/x.txt"]).toMatchObject({ type: "file", path: "/x.txt" })
+		expect(result.result.size).toBe(4)
+		expect(result.result.uuids["uuid-c"]).toMatchObject({ path: "/a/b/c.txt" })
+	})
+
+	it("still throws on a malformed response with NO sync-root folder (not treated as empty)", async () => {
+		const world = await createWorld({ mode: "twoWay" })
+
+		// No folder has parent "base" — a malformed response. Must throw rather than yield an empty tree
+		// (which would look like every remote node was deleted).
+		const folders: Folder[] = [["uuid-a", folderMeta("a"), "uuid-missing"]]
+
+		inject(world, folders, [])
+
+		await expect(world.sync.remoteFileSystem.getDirectoryTree(true)).rejects.toThrow()
+	})
+})

package/tests/unit/semaphore-regression.test.ts

@@ -0,0 +1,140 @@
+import { describe, it, expect } from "vitest"
+import { Semaphore } from "../../src/semaphore"
+
+/**
+ * Added regression tests pinning the behavioral contract of {@link Semaphore} under LARGE fan-out, so the
+ * O(1)-dequeue optimization (finding 001: replacing the O(n) `Array.shift()` with a head-index queue)
+ * cannot silently change semantics. These assert the invariants the optimization must preserve — FIFO
+ * wake order, the concurrency cap, and purge accounting — at queue depths where a regression would show.
+ *
+ * NEW FILE — does not touch the existing Semaphore tests in tests/unit/n-unit.test.ts.
+ */
+
+const tick = (): Promise<void> => new Promise<void>(resolve => setTimeout(resolve, 0))
+
+describe("Semaphore — large fan-out invariants (perf optimization guard)", () => {
+	it("preserves strict FIFO wake order across thousands of queued waiters", async () => {
+		const sem = new Semaphore(1)
+
+		await sem.acquire() // hold the only slot so everything queues
+
+		const order: number[] = []
+		const count = 3_000
+		const waiters: Promise<void>[] = []
+
+		for (let i = 0; i < count; i++) {
+			waiters.push(
+				(async () => {
+					await sem.acquire()
+
+					order.push(i)
+
+					sem.release()
+				})()
+			)
+		}
+
+		await tick() // let all `count` waiters enqueue
+		sem.release() // release the holder → FIFO cascade drains the queue
+
+		await Promise.all(waiters)
+
+		expect(order.length).toBe(count)
+
+		for (let i = 0; i < count; i++) {
+			expect(order[i]).toBe(i)
+		}
+
+		expect(sem.count()).toBe(0)
+	})
+
+	it("never exceeds the concurrency cap and fully reaches it under heavy contention", async () => {
+		const width = 10
+		const sem = new Semaphore(width)
+		let running = 0
+		let maxRunning = 0
+
+		await Promise.all(
+			Array.from({ length: 5_000 }, async () => {
+				await sem.acquire()
+
+				running++
+				maxRunning = Math.max(maxRunning, running)
+
+				await tick()
+
+				running--
+
+				sem.release()
+			})
+		)
+
+		expect(maxRunning).toBe(width)
+		expect(sem.count()).toBe(0)
+	})
+
+	it("drains thousands of sequential width-1 contenders (head-index compaction path)", async () => {
+		const sem = new Semaphore(1)
+
+		await sem.acquire()
+
+		let done = 0
+		const count = 8_000
+		const waiters = Array.from({ length: count }, async () => {
+			await sem.acquire()
+
+			done++
+
+			sem.release()
+		})
+
+		await tick()
+		sem.release()
+
+		await Promise.all(waiters)
+
+		expect(done).toBe(count)
+		expect(sem.count()).toBe(0)
+	})
+
+	it("purge() after a large fan-out rejects every waiter, returns the count, and resets", async () => {
+		const sem = new Semaphore(1)
+
+		await sem.acquire()
+
+		const count = 1_500
+		const rejected: Promise<unknown>[] = []
+
+		for (let i = 0; i < count; i++) {
+			rejected.push(sem.acquire().then(() => "resolved", () => "rejected"))
+		}
+
+		await tick()
+
+		expect(sem.purge()).toBe(count)
+		expect(sem.count()).toBe(0)
+
+		const outcomes = await Promise.all(rejected)
+
+		expect(outcomes.every(o => o === "rejected")).toBe(true)
+	})
+
+	it("reuses a width-1 mutex correctly across many serial acquire/release cycles", async () => {
+		const sem = new Semaphore(1)
+		let concurrent = 0
+		let maxConcurrent = 0
+
+		for (let i = 0; i < 2_000; i++) {
+			await sem.acquire()
+
+			concurrent++
+			maxConcurrent = Math.max(maxConcurrent, concurrent)
+			concurrent--
+
+			sem.release()
+		}
+
+		expect(maxConcurrent).toBe(1)
+		expect(sem.count()).toBe(0)
+	})
+})

package/tests/unit/state-refencode-regression.test.ts

@@ -0,0 +1,224 @@
+import { describe, it, expect } from "vitest"
+import { State } from "../../src/lib/state"
+import { type LocalItem, type LocalTree } from "../../src/lib/filesystems/local"
+import { type RemoteItem, type RemoteTree } from "../../src/lib/filesystems/remote"
+import { createVirtualFS, toPosixPath, type VirtualFS } from "../fakes/virtual-fs"
+import type Sync from "../../src/lib/sync"
+
+/**
+ * Added regression tests for the state index-file optimization: the previousLocalINodes /
+ * previousRemoteUUIDs files are now persisted as compact path-REFS (the in-memory indexes are rebuilt
+ * from the tree files on load), instead of duplicating every full item. This pins:
+ *   (1) the on-disk index files are refs (data = path string), not full items;
+ *   (2) a save -> load round-trip reconstructs the inode/uuid indexes exactly;
+ *   (3) BACK-COMPAT: an OLD on-disk state whose index files still hold FULL items loads identically
+ *       (the index files are ignored on load and rebuilt from the trees), so an upgrade is a no-op.
+ *
+ * NEW FILE — does not touch tests/unit/state.test.ts.
+ */
+
+const DB_ROOT = "/db"
+
+type StateSyncStub = {
+	dbPath: string
+	syncPair: { uuid: string }
+	environment: { fs: VirtualFS["fs"]; globFs: VirtualFS["globFs"] }
+	localFileHashes: Record<string, string>
+	previousLocalTree: LocalTree
+	previousRemoteTree: RemoteTree
+	isPreviousSavedTreeStateEmpty: boolean
+	removed: boolean
+}
+
+function makeSyncStub(vfs: VirtualFS, uuid: string): StateSyncStub {
+	return {
+		dbPath: DB_ROOT,
+		syncPair: { uuid },
+		environment: { fs: vfs.fs, globFs: vfs.globFs },
+		localFileHashes: {},
+		previousLocalTree: { tree: {}, inodes: {}, size: 0 },
+		previousRemoteTree: { tree: {}, uuids: {}, size: 0 },
+		isPreviousSavedTreeStateEmpty: true,
+		removed: false
+	}
+}
+
+function makeState(stub: StateSyncStub): State {
+	return new State(stub as unknown as Sync)
+}
+
+function localFile(path: string, inode: number): LocalItem {
+	return { type: "file", path, inode, size: 100, lastModified: 1_700_000_000_000, creation: 1_690_000_000_000 }
+}
+
+function localDir(path: string, inode: number): LocalItem {
+	return { type: "directory", path, inode, size: 0, lastModified: 1_700_000_000_000, creation: 1_690_000_000_000 }
+}
+
+function remoteFile(path: string, uuid: string, name: string): RemoteItem {
+	return { type: "file", uuid, name, size: 100, mime: "text/plain", lastModified: 1_700_000_000_000, version: 2, chunks: 1, key: `key-${uuid}`, bucket: "b", region: "r", path }
+}
+
+function remoteDir(path: string, uuid: string, name: string): RemoteItem {
+	return { type: "directory", uuid, name, size: 0, path }
+}
+
+function jsonLines(vfs: VirtualFS, filePath: string): { prop: string; data: unknown }[] {
+	return (vfs.ifs.readFileSync(toPosixPath(filePath), "utf-8") as string)
+		.split("\n")
+		.filter(line => line.length > 0)
+		.map(line => JSON.parse(line))
+}
+
+function seedTrees(stub: StateSyncStub): { localTree: LocalTree; remoteTree: RemoteTree } {
+	const fileA = localFile("/a.txt", 101)
+	const dir1 = localDir("/dir", 102)
+	const fileB = localFile("/dir/b.txt", 103)
+	const localTree: LocalTree = {
+		tree: { "/a.txt": fileA, "/dir": dir1, "/dir/b.txt": fileB },
+		inodes: { 101: fileA, 102: dir1, 103: fileB },
+		size: 3
+	}
+
+	const rfileA = remoteFile("/a.txt", "u-a", "a.txt")
+	const rdir = remoteDir("/dir", "u-dir", "dir")
+	const rfileB = remoteFile("/dir/b.txt", "u-b", "b.txt")
+	const remoteTree: RemoteTree = {
+		tree: { "/a.txt": rfileA, "/dir": rdir, "/dir/b.txt": rfileB },
+		uuids: { "u-a": rfileA, "u-dir": rdir, "u-b": rfileB },
+		size: 3
+	}
+
+	stub.previousLocalTree = localTree
+	stub.previousRemoteTree = remoteTree
+
+	return { localTree, remoteTree }
+}
+
+describe("State — index files persisted as path-refs (perf guard)", () => {
+	it("writes the inode/uuid index files as path-refs, not full items", async () => {
+		const vfs = createVirtualFS()
+		const stub = makeSyncStub(vfs, "ref-uuid")
+		const state = makeState(stub)
+
+		seedTrees(stub)
+
+		await state.savePreviousTrees()
+
+		// Each line's `data` is the item's path (a string), not the full item object.
+		for (const line of jsonLines(vfs, state.previousLocalINodesPath)) {
+			expect(typeof line.data).toBe("string")
+		}
+
+		for (const line of jsonLines(vfs, state.previousRemoteUUIDsPath)) {
+			expect(typeof line.data).toBe("string")
+		}
+
+		// Props are still the inode / uuid keys (the completeness gate + key layout are preserved).
+		expect(jsonLines(vfs, state.previousLocalINodesPath).map(l => l.prop).sort()).toEqual(["101", "102", "103"])
+		expect(jsonLines(vfs, state.previousRemoteUUIDsPath).map(l => l.prop).sort()).toEqual(["u-a", "u-b", "u-dir"])
+
+		// The tree files are unchanged: their `data` is the verbatim full item.
+		const firstTreeLine = jsonLines(vfs, state.previousLocalTreePath)[0]!
+
+		expect(typeof firstTreeLine.data).toBe("object")
+	})
+
+	it("round-trips: save (ref index) then load rebuilds the inode/uuid indexes exactly", async () => {
+		const vfs = createVirtualFS()
+		const source = makeSyncStub(vfs, "rt-uuid")
+		const { localTree, remoteTree } = seedTrees(source)
+
+		await makeState(source).save()
+
+		const reloaded = makeSyncStub(vfs, "rt-uuid")
+
+		await makeState(reloaded).initialize()
+
+		expect(reloaded.previousLocalTree.tree).toEqual(localTree.tree)
+		expect(reloaded.previousLocalTree.inodes).toEqual(localTree.inodes)
+		expect(reloaded.previousRemoteTree.tree).toEqual(remoteTree.tree)
+		expect(reloaded.previousRemoteTree.uuids).toEqual(remoteTree.uuids)
+		// The rebuilt index points at the SAME item objects held in the loaded tree.
+		expect(reloaded.previousLocalTree.inodes[101]).toBe(reloaded.previousLocalTree.tree["/a.txt"])
+		expect(reloaded.previousRemoteTree.uuids["u-a"]).toBe(reloaded.previousRemoteTree.tree["/a.txt"])
+	})
+
+	it("BACK-COMPAT: an old on-disk state with FULL-item index files loads identically", async () => {
+		const vfs = createVirtualFS()
+		const source = makeSyncStub(vfs, "legacy-uuid")
+		const { localTree, remoteTree } = seedTrees(source)
+		const state = makeState(source)
+
+		// Write the trees normally...
+		await state.writeLargeRecordSerializedAndAtomically(state.previousLocalTreePath, localTree.tree)
+		await state.writeLargeRecordSerializedAndAtomically(state.previousRemoteTreePath, remoteTree.tree)
+		// ...but write the index files in the OLD format: FULL items as data (the pre-optimization layout).
+		await state.writeLargeRecordSerializedAndAtomically(
+			state.previousLocalINodesPath,
+			localTree.inodes as unknown as Record<string, LocalItem>
+		)
+		await state.writeLargeRecordSerializedAndAtomically(
+			state.previousRemoteUUIDsPath,
+			remoteTree.uuids as unknown as Record<string, RemoteItem>
+		)
+		await state.saveLocalFileHashes()
+
+		// Confirm the on-disk index really is the legacy full-item format.
+		expect(typeof jsonLines(vfs, state.previousLocalINodesPath)[0]!.data).toBe("object")
+
+		const reloaded = makeSyncStub(vfs, "legacy-uuid")
+
+		await makeState(reloaded).loadPreviousTrees()
+
+		// New loader ignores the legacy index files and rebuilds from the trees → identical base.
+		expect(reloaded.isPreviousSavedTreeStateEmpty).toBe(false)
+		expect(reloaded.previousLocalTree.inodes).toEqual(localTree.inodes)
+		expect(reloaded.previousRemoteTree.uuids).toEqual(remoteTree.uuids)
+		expect(reloaded.previousLocalTree.size).toBe(3)
+		expect(reloaded.previousRemoteTree.size).toBe(3)
+	})
+
+	it("still degrades to no-saved-state when an index file is MISSING (gate preserved)", async () => {
+		const vfs = createVirtualFS()
+		const source = makeSyncStub(vfs, "gate-uuid")
+
+		seedTrees(source)
+
+		const state = makeState(source)
+
+		await state.save()
+
+		// Remove the (now ref-encoded) inodes file: the completeness gate must still trip.
+		vfs.ifs.rmSync(toPosixPath(state.previousLocalINodesPath))
+
+		const reloaded = makeSyncStub(vfs, "gate-uuid")
+
+		await makeState(reloaded).loadPreviousTrees()
+
+		expect(reloaded.isPreviousSavedTreeStateEmpty).toBe(true)
+		expect(reloaded.previousLocalTree.tree).toEqual({})
+		expect(reloaded.previousLocalTree.inodes).toEqual({})
+	})
+
+	it("rebuild handles two paths sharing an inode (hardlink): last in tree order wins, as the scan did", async () => {
+		const vfs = createVirtualFS()
+		const source = makeSyncStub(vfs, "hardlink-uuid")
+
+		// Two files share inode 700. The scan inserts tree + inodes in the same order, so the index ends up
+		// pointing at whichever comes LAST. Persisting the tree preserves that order; the rebuild must match.
+		const first = localFile("/first.txt", 700)
+		const second = localFile("/second.txt", 700)
+
+		source.previousLocalTree = { tree: { "/first.txt": first, "/second.txt": second }, inodes: { 700: second }, size: 2 }
+		source.previousRemoteTree = { tree: {}, uuids: {}, size: 0 }
+
+		await makeState(source).save()
+
+		const reloaded = makeSyncStub(vfs, "hardlink-uuid")
+
+		await makeState(reloaded).loadPreviousTrees()
+
+		expect(reloaded.previousLocalTree.inodes[700]!.path).toBe("/second.txt")
+	})
+})