@filen/sync 0.2.1 → 0.3.2

package/tests/bench/profile-incremental.ts

@@ -0,0 +1,90 @@
+/**
+ * Phase profiler for ONE incremental cycle (1 file changed in a large synced tree) — tells us exactly
+ * where the per-edit cost goes before we optimize. Not a *.bench.ts, so the bench suite ignores it.
+ *
+ *   NODE_OPTIONS='--expose-gc' tsx tests/bench/profile-incremental.ts [nodes]
+ */
+import { makeScaleWorld, sceneToVfsSpec, forceLocalRescan } from "./harness/scale-world"
+import { genWideScene, resetIdentity } from "./harness/trees"
+import { SYNC_INTERVAL } from "../../src/constants"
+import { LOCAL_ROOT } from "../harness/world"
+
+const nodes = Number(process.argv[2] ?? 20_000)
+
+function wrap<T extends object, K extends keyof T>(obj: T, key: K, label: string, acc: Record<string, number>): void {
+	const original = obj[key] as unknown as (...args: unknown[]) => unknown
+
+	obj[key] = (async (...args: unknown[]) => {
+		const t0 = performance.now()
+		const result = await original.apply(obj, args)
+
+		acc[label] = (acc[label] ?? 0) + (performance.now() - t0)
+
+		return result
+	}) as unknown as T[K]
+}
+
+async function main(): Promise<void> {
+	resetIdentity()
+
+	const scene = genWideScene(Math.max(1, Math.round(nodes / 101)), 100)
+	const world = await makeScaleWorld({ mode: "twoWay", initialLocal: sceneToVfsSpec(scene) })
+
+	// Settle the initial sync.
+	for (let i = 0; i < 3; i++) {
+		world.sync.localFileSystem.lastDirectoryChangeTimestamp = Date.now() - SYNC_INTERVAL - 1_000
+
+		await world.sync.runCycle()
+
+		world.messages.length = 0
+	}
+
+	const acc: Record<string, number> = {}
+
+	wrap(world.sync.ignorer, "initialize", "ignorer.initialize", acc)
+	wrap(world.sync.localFileSystem, "getDirectoryTree", "local.getDirectoryTree", acc)
+	wrap(world.sync.remoteFileSystem, "getDirectoryTree", "remote.getDirectoryTree", acc)
+	wrap(world.sync.deltas, "process", "deltas.process", acc)
+	wrap(world.sync.tasks, "process", "tasks.process", acc)
+	wrap(world.sync.state, "save", "state.save", acc)
+	wrap(world.sync.lock, "acquire", "lock.acquire", acc)
+	wrap(world.sync.lock, "release", "lock.release", acc)
+
+	// One measured incremental cycle: change a single file, force a rescan, run.
+	await world.vfs.fs.writeFile(`${LOCAL_ROOT}/dir-0/file-0.txt`, "y".repeat(99), { encoding: "utf-8" })
+
+	forceLocalRescan(world)
+
+	world.sync.localFileSystem.lastDirectoryChangeTimestamp = Date.now() - SYNC_INTERVAL - 1_000
+
+	const t0 = performance.now()
+
+	await world.sync.runCycle()
+
+	const total = performance.now() - t0
+
+	process.stdout.write(`\nIncremental cycle profile — ${scene.length} nodes, 1 file changed:\n`)
+
+	const sorted = Object.entries(acc).sort((a, b) => b[1] - a[1])
+	let accounted = 0
+
+	for (const [label, ms] of sorted) {
+		accounted += ms
+		process.stdout.write(`  ${label.padEnd(26)} ${ms.toFixed(1).padStart(8)} ms  (${((ms / total) * 100).toFixed(1)}%)\n`)
+	}
+
+	process.stdout.write(`  ${"[unaccounted/other]".padEnd(26)} ${(total - accounted).toFixed(1).padStart(8)} ms\n`)
+	process.stdout.write(`  ${"TOTAL".padEnd(26)} ${total.toFixed(1).padStart(8)} ms\n`)
+
+	if (world.sync.cleanupLocalTrashInterval) {
+		clearInterval(world.sync.cleanupLocalTrashInterval)
+	}
+}
+
+main().then(
+	() => process.exit(0),
+	e => {
+		console.error(e)
+		process.exit(1)
+	}
+)

package/tests/bench/remote-build.bench.ts

@@ -0,0 +1,104 @@
+import { describe, it, expect } from "vitest"
+import { bench } from "./harness/measure"
+import { makeScaleWorld, sceneToDirTreeResponse, injectDirTreeResponse, forceRemoteRebuild } from "./harness/scale-world"
+import { genWideScene, genBalancedScene, resetIdentity, type Scene } from "./harness/trees"
+import { type World } from "../harness/world"
+
+/**
+ * Remote tree-build benchmark (target T2). `remoteFileSystem.getDirectoryTree` decrypts every folder
+ * (SEQUENTIALLY, to build parent paths) and every file (eagerly mapping N promises through a 1024-wide
+ * semaphore), then builds the tree+uuids maps.
+ *
+ * We feed the engine a PRE-BUILT dir-tree response (synthesised from a scene, JSON metadata) so the
+ * timed work is ONLY the engine's build loop — NOT the fake cloud's `buildFullTree`, which is O(N²)
+ * (test infra: it scans all nodes per directory) and would dominate + mask the engine cost. The fake
+ * cloud's decrypt (JSON.parse) is what runs per item, isolating the engine's per-node CPU + the
+ * eager-promise/closure memory T2 targets (real SDK crypto is a separate, SDK-owned cost).
+ */
+
+async function worldFor(scene: Scene): Promise<World> {
+	// Empty remote — we inject the response directly; the world only supplies sdk.crypto + environment.
+	const world = await makeScaleWorld({ mode: "twoWay" })
+
+	injectDirTreeResponse(world, sceneToDirTreeResponse(scene, world.cloud.controls.rootUUID))
+
+	return world
+}
+
+describe("remoteFileSystem.getDirectoryTree (engine build loop, isolated)", () => {
+	it("wide tree size sweep", async () => {
+		// Post-fix: linear (finding 001 + 004), so the sweep goes to 500k where the baseline capped at
+		// 200k (which took ~22s under the old O(N²) build).
+		for (const nodes of [10_000, 50_000, 200_000, 500_000]) {
+			resetIdentity()
+
+			const scene = genWideScene(Math.max(1, Math.round(nodes / 101)), 100)
+			const world = await worldFor(scene)
+
+			const first = await world.sync.remoteFileSystem.getDirectoryTree(true)
+
+			expect(first.result.size).toBe(scene.length)
+
+			await bench({
+				group: "remoteFileSystem.getDirectoryTree / wide",
+				name: `${nodes} nodes`,
+				n: scene.length,
+				iterations: nodes >= 200_000 ? 3 : 5,
+				setup: () => {
+					forceRemoteRebuild(world)
+
+					return world
+				},
+				run: w => w.sync.remoteFileSystem.getDirectoryTree(true)
+			})
+		}
+	})
+
+	it("directory-heavy tree (sequential folder-decrypt path)", async () => {
+		resetIdentity()
+
+		const scene = genWideScene(10_000, 2)
+		const world = await worldFor(scene)
+
+		const first = await world.sync.remoteFileSystem.getDirectoryTree(true)
+
+		expect(first.result.size).toBe(scene.length)
+
+		await bench({
+			group: "remoteFileSystem.getDirectoryTree / directory-heavy",
+			name: `50k dirs x2 files (${scene.length} nodes)`,
+			n: scene.length,
+			iterations: 4,
+			setup: () => {
+				forceRemoteRebuild(world)
+
+				return world
+			},
+			run: w => w.sync.remoteFileSystem.getDirectoryTree(true)
+		})
+	})
+
+	it("balanced tree", async () => {
+		resetIdentity()
+
+		const scene = genBalancedScene({ fanout: 4, depth: 5, filesPerDir: 8 })
+		const world = await worldFor(scene)
+
+		const first = await world.sync.remoteFileSystem.getDirectoryTree(true)
+
+		expect(first.result.size).toBe(scene.length)
+
+		await bench({
+			group: "remoteFileSystem.getDirectoryTree / balanced",
+			name: `fanout4 depth5 (${scene.length} nodes)`,
+			n: scene.length,
+			iterations: 4,
+			setup: () => {
+				forceRemoteRebuild(world)
+
+				return world
+			},
+			run: w => w.sync.remoteFileSystem.getDirectoryTree(true)
+		})
+	})
+})

package/tests/bench/render.ts

@@ -0,0 +1,14 @@
+import pathModule from "path"
+import { renderReport } from "./harness/measure"
+
+/**
+ * Standalone report renderer: reads the JSONL the bench run appended and writes a grouped markdown
+ * table. Decoupled from the vitest lifecycle (vitest workers don't reliably fire exit hooks).
+ *
+ *   tsx tests/bench/render.ts [label] [inJsonl] [outMd]
+ */
+const label = process.argv[2] ?? process.env["BENCH_LABEL"] ?? "run"
+const inJsonl = process.argv[3] ?? process.env["BENCH_JSONL"] ?? pathModule.join(process.cwd(), "docs/perf/benchmarks/_results.jsonl")
+const outMd = process.argv[4] ?? process.env["BENCH_OUT"] ?? pathModule.join(process.cwd(), `docs/perf/benchmarks/${label}.md`)
+
+renderReport(inJsonl, outMd, label)

package/tests/bench/semaphore.bench.ts

@@ -0,0 +1,79 @@
+import { describe, it, expect } from "vitest"
+import { bench } from "./harness/measure"
+import { Semaphore } from "../../src/semaphore"
+
+/**
+ * Semaphore throughput benchmark (finding 001). When N tasks contend on a semaphore of width `maxCount`,
+ * N−maxCount of them queue. The dequeue cost per release determines whether bulk fan-out (remote tree
+ * build, bulk transfers) is O(N) or O(N²). This drives a realistic fan-out: N tasks each acquire, yield a
+ * microtask, then release — exactly the shape of `dir.files.map(async () => { await sem.acquire(); … })`.
+ */
+
+async function drain(maxCount: number, tasks: number): Promise<void> {
+	const sem = new Semaphore(maxCount)
+
+	await Promise.all(
+		Array.from({ length: tasks }, async () => {
+			await sem.acquire()
+
+			try {
+				// Yield once so the semaphore actually queues waiters (mirrors an awaited decrypt / IO step).
+				await Promise.resolve()
+			} finally {
+				sem.release()
+			}
+		})
+	)
+}
+
+describe("Semaphore fan-out throughput", () => {
+	it("width 1024 (remote listSemaphore shape)", async () => {
+		// Post-fix sizes — the O(1) head-index dequeue is linear, so these complete fast where the old
+		// O(N²) shift took seconds-to-minutes. (Baseline 00-baseline.md used 5k–40k to show the quadratic.)
+		for (const tasks of [50_000, 200_000, 1_000_000]) {
+			await bench({
+				group: "Semaphore.drain / width 1024",
+				name: `${tasks} tasks`,
+				n: tasks,
+				iterations: 4,
+				setup: () => ({ maxCount: 1024, tasks }),
+				run: ({ maxCount, tasks: t }) => drain(maxCount, t)
+			})
+		}
+	})
+
+	it("width 10 (transfers semaphore shape — bulk upload/download)", async () => {
+		for (const tasks of [50_000, 200_000, 1_000_000]) {
+			await bench({
+				group: "Semaphore.drain / width 10",
+				name: `${tasks} tasks`,
+				n: tasks,
+				iterations: 4,
+				setup: () => ({ maxCount: 10, tasks }),
+				run: ({ maxCount, tasks: t }) => drain(maxCount, t)
+			})
+		}
+	})
+
+	it("FIFO order preserved", async () => {
+		// Correctness guard the optimization must keep: waiters resume in acquire order.
+		const sem = new Semaphore(1)
+		const order: number[] = []
+
+		await sem.acquire() // hold the only slot
+
+		const waiters = [0, 1, 2, 3, 4].map(async i => {
+			await sem.acquire()
+			order.push(i)
+			sem.release()
+		})
+
+		// Let them all queue, then release to drain in FIFO order.
+		await Promise.resolve()
+		sem.release()
+
+		await Promise.all(waiters)
+
+		expect(order).toEqual([0, 1, 2, 3, 4])
+	})
+})

package/tests/bench/state.bench.ts

@@ -0,0 +1,85 @@
+import { describe, it, expect } from "vitest"
+import { bench } from "./harness/measure"
+import State from "../../src/lib/state"
+import type Sync from "../../src/lib/sync"
+import { createVirtualFS } from "../fakes/virtual-fs"
+import { genWideScene, buildLocalTree, buildRemoteTree, resetIdentity } from "./harness/trees"
+
+/**
+ * State persistence benchmark (target T4). Every change-cycle persists 4 line-delimited-JSON files
+ * (previousLocalTree, previousLocalINodes, previousRemoteTree, previousRemoteUUIDs) and reloads them on
+ * restart. The tree and the inodes/uuids files hold the SAME item objects keyed differently, so each
+ * item is serialized TWICE — this measures the real cost and the duplication T4 targets. memfs backs the
+ * writes so we measure serialization + stream + parse CPU, not real disk latency.
+ */
+
+function makeState(scene: ReturnType<typeof genWideScene>): { state: State; sync: Sync } {
+	const vfs = createVirtualFS({ "/db": null })
+	const localTree = buildLocalTree(scene)
+	const remoteTree = buildRemoteTree(scene)
+
+	const sync = {
+		dbPath: "/db",
+		syncPair: { uuid: "bench-uuid" },
+		environment: { fs: vfs.fs, globFs: vfs.globFs },
+		localFileHashes: {},
+		previousLocalTree: { tree: localTree.tree, inodes: localTree.inodes, size: localTree.size },
+		previousRemoteTree: { tree: remoteTree.tree, uuids: remoteTree.uuids, size: remoteTree.size },
+		isPreviousSavedTreeStateEmpty: true
+	} as unknown as Sync
+
+	return { state: new State(sync), sync }
+}
+
+function sceneOf(nodes: number): ReturnType<typeof genWideScene> {
+	resetIdentity()
+
+	return genWideScene(Math.max(1, Math.round(nodes / 101)), 100)
+}
+
+describe("state persistence", () => {
+	it("savePreviousTrees (4 files, tree+inodes duplicated)", async () => {
+		for (const nodes of [10_000, 100_000, 300_000]) {
+			await bench({
+				group: "state.savePreviousTrees",
+				name: `${nodes} nodes`,
+				n: nodes,
+				iterations: 4,
+				setup: () => makeState(sceneOf(nodes)),
+				run: ({ state }) => state.savePreviousTrees()
+			})
+		}
+	})
+
+	it("loadPreviousTrees (round-trip read of the 4 files)", async () => {
+		for (const nodes of [10_000, 100_000, 300_000]) {
+			const scene = sceneOf(nodes)
+			const expectedSize = scene.length
+
+			await bench({
+				group: "state.loadPreviousTrees",
+				name: `${nodes} nodes`,
+				n: nodes,
+				iterations: 4,
+				setup: async () => {
+					const made = makeState(scene)
+
+					await made.state.savePreviousTrees()
+
+					// Reset the in-memory trees so the load actually re-reads + rebuilds from disk.
+					made.sync.previousLocalTree = { tree: {}, inodes: {}, size: 0 }
+					made.sync.previousRemoteTree = { tree: {}, uuids: {}, size: 0 }
+
+					return made
+				},
+				run: async ({ state, sync }) => {
+					await state.loadPreviousTrees()
+
+					expect(sync.previousLocalTree.size).toBe(expectedSize)
+
+					return sync.previousLocalTree.size
+				}
+			})
+		}
+	})
+})

package/tests/bench/tasks-dispatch.bench.ts

@@ -0,0 +1,156 @@
+import { describe, it, expect } from "vitest"
+import { bench } from "./harness/measure"
+import { type Delta } from "../../src/lib/deltas"
+
+/**
+ * Task-dispatch benchmark (target T1). `tasks.process` dispatches deltas in 12 type passes by scanning
+ * the WHOLE deltasSorted array 10 times (`for…if(type!==X)continue`) plus 2 `.filter()` passes for the
+ * up/down transfers — 12·O(D) just to order the work. This isolates that dispatch overhead (with a
+ * no-op task body) and compares it to a single bucketing pass, so we know whether T1 is worth doing
+ * before touching the real executor.
+ */
+
+const DISPATCH_ORDER = [
+	"renameLocalDirectory",
+	"renameLocalFile",
+	"renameRemoteDirectory",
+	"renameRemoteFile",
+	"deleteLocalDirectory",
+	"deleteLocalFile",
+	"deleteRemoteDirectory",
+	"deleteRemoteFile",
+	"createLocalDirectory",
+	"createRemoteDirectory"
+] as const
+
+const noopProcess = async (): Promise<void> => {}
+
+/** Current strategy: 10 full filter-scans (sequential) + 2 filter passes for transfers. */
+async function dispatchCurrent(deltas: Delta[]): Promise<void> {
+	for (const type of DISPATCH_ORDER) {
+		for (const delta of deltas) {
+			if (delta.type !== type) {
+				continue
+			}
+
+			await noopProcess()
+		}
+	}
+
+	await Promise.all(deltas.filter(d => d.type === "uploadFile").map(noopProcess))
+	await Promise.all(deltas.filter(d => d.type === "downloadFile").map(noopProcess))
+}
+
+/** Bucketed strategy: one pass to partition, then drain each bucket in the same order. */
+async function dispatchBucketed(deltas: Delta[]): Promise<void> {
+	const buckets = new Map<string, Delta[]>()
+
+	for (const delta of deltas) {
+		let bucket = buckets.get(delta.type)
+
+		if (!bucket) {
+			bucket = []
+
+			buckets.set(delta.type, bucket)
+		}
+
+		bucket.push(delta)
+	}
+
+	for (const type of DISPATCH_ORDER) {
+		const bucket = buckets.get(type)
+
+		if (bucket) {
+			for (let i = 0; i < bucket.length; i++) {
+				await noopProcess()
+			}
+		}
+	}
+
+	await Promise.all((buckets.get("uploadFile") ?? []).map(noopProcess))
+	await Promise.all((buckets.get("downloadFile") ?? []).map(noopProcess))
+}
+
+function genUploads(d: number): Delta[] {
+	const deltas: Delta[] = []
+
+	for (let i = 0; i < d; i++) {
+		deltas.push({ type: "uploadFile", path: `/dir-${i % 1000}/file-${i}.txt`, size: 64 })
+	}
+
+	return deltas
+}
+
+function genMixed(d: number): Delta[] {
+	const deltas: Delta[] = []
+
+	for (let i = 0; i < d; i++) {
+		const r = i % 5
+
+		if (r === 0) {
+			deltas.push({ type: "uploadFile", path: `/u-${i}.txt`, size: 64 })
+		} else if (r === 1) {
+			deltas.push({ type: "downloadFile", path: `/d-${i}.txt`, size: 64 })
+		} else if (r === 2) {
+			deltas.push({ type: "deleteRemoteFile", path: `/x-${i}.txt` })
+		} else if (r === 3) {
+			deltas.push({ type: "createRemoteDirectory", path: `/c-${i}` })
+		} else {
+			deltas.push({ type: "renameRemoteFile", path: `/r-${i}.txt`, from: `/r-${i}.txt`, to: `/r2-${i}.txt` })
+		}
+	}
+
+	return deltas
+}
+
+describe("tasks dispatch — 12-pass vs bucketed (T1)", () => {
+	it("all-uploads (bulk add — worst case for the 10 no-op rename/delete passes)", async () => {
+		for (const d of [10_000, 100_000, 1_000_000]) {
+			const deltas = genUploads(d)
+
+			await bench({
+				group: "tasks dispatch / all uploads",
+				name: `current 12-pass — D=${d}`,
+				n: d,
+				iterations: 5,
+				setup: () => deltas,
+				run: ds => dispatchCurrent(ds)
+			})
+
+			await bench({
+				group: "tasks dispatch / all uploads",
+				name: `bucketed — D=${d}`,
+				n: d,
+				iterations: 5,
+				setup: () => deltas,
+				run: ds => dispatchBucketed(ds)
+			})
+		}
+	})
+
+	it("mixed delta types", async () => {
+		for (const d of [100_000, 1_000_000]) {
+			const deltas = genMixed(d)
+
+			expect(deltas.length).toBe(d)
+
+			await bench({
+				group: "tasks dispatch / mixed types",
+				name: `current 12-pass — D=${d}`,
+				n: d,
+				iterations: 5,
+				setup: () => deltas,
+				run: ds => dispatchCurrent(ds)
+			})
+
+			await bench({
+				group: "tasks dispatch / mixed types",
+				name: `bucketed — D=${d}`,
+				n: d,
+				iterations: 5,
+				setup: () => deltas,
+				run: ds => dispatchBucketed(ds)
+			})
+		}
+	})
+})