@filen/sync 0.2.1 → 0.3.2

package/tests/bench/harness/scale-world.ts

@@ -0,0 +1,160 @@
+import { createWorld, type World, LOCAL_ROOT } from "../../harness/world"
+import { type SyncMode } from "../../../src/types"
+import { type VfsSpec } from "../../fakes/virtual-fs"
+import { type CloudSpec } from "../../fakes/fake-cloud"
+import { type SyncDirTreeFetcher, type DirTreeResponse } from "../../../src/lib/filesystems/dirTree"
+import { type Scene } from "./trees"
+
+/**
+ * Build a memfs {@link VfsSpec} (keys under the local root) from a scene. Files get tiny deterministic
+ * content; directories are explicit nulls so empty dirs still materialise.
+ */
+export function sceneToVfsSpec(scene: Scene, contentByte = "x"): VfsSpec {
+	const spec: VfsSpec = {}
+
+	for (const node of scene) {
+		if (node.type === "directory") {
+			spec[`${LOCAL_ROOT}${node.path}`] = null
+		} else {
+			spec[`${LOCAL_ROOT}${node.path}`] = contentByte.repeat(node.size)
+		}
+	}
+
+	return spec
+}
+
+/** Build a fake-cloud {@link CloudSpec} (keys are sync-root-absolute paths) from a scene. */
+export function sceneToCloudSpec(scene: Scene, contentByte = "x"): CloudSpec {
+	const spec: CloudSpec = {}
+
+	for (const node of scene) {
+		if (node.type === "directory") {
+			spec[node.path] = null
+		} else {
+			spec[node.path] = contentByte.repeat(node.size)
+		}
+	}
+
+	return spec
+}
+
+/**
+ * Create a wired in-memory world at scale and neutralise the background trash-cleanup interval so it
+ * cannot fire mid-benchmark. Runs under REAL timers (the bench measures wall time) — safe because the
+ * methods we benchmark (getDirectoryTree, runCycle with the debounce/timestamp pre-set) never await a
+ * scheduled timer in the uncontended in-memory path.
+ */
+export async function makeScaleWorld(options: {
+	mode: SyncMode
+	initialLocal?: VfsSpec
+	initialRemote?: CloudSpec
+}): Promise<World> {
+	const world = await createWorld(options)
+
+	if (world.sync.cleanupLocalTrashInterval) {
+		clearInterval(world.sync.cleanupLocalTrashInterval)
+
+		world.sync.cleanupLocalTrashInterval = undefined
+	}
+
+	return world
+}
+
+/** Reset the local directory-tree cache so the next getDirectoryTree() does a full rescan. */
+export function forceLocalRescan(world: World): void {
+	world.sync.localFileSystem.lastDirectoryChangeTimestamp = Date.now()
+	world.sync.localFileSystem.getDirectoryTreeCache = {
+		timestamp: 0,
+		tree: {},
+		inodes: {},
+		ignored: [],
+		errors: [],
+		size: 0
+	}
+}
+
+function basename(path: string): string {
+	const i = path.lastIndexOf("/")
+
+	return i === -1 ? path : path.slice(i + 1)
+}
+
+function parentPath(path: string): string {
+	const i = path.lastIndexOf("/")
+
+	return i <= 0 ? "" : path.slice(0, i)
+}
+
+/**
+ * Synthesize the exact `/v3/dir/tree` response (files/folders tuple arrays with JSON metadata — the fake
+ * cloud's decrypt is JSON.parse) directly from a scene, so the remote-build benchmark can feed it into
+ * the ENGINE's tree-build loop WITHOUT going through the fake cloud's O(N²) `buildFullTree`. Ordered
+ * parents-before-children (root first) by default — matching the order the engine's loop relies on.
+ * `shuffle` produces an out-of-order response to exercise the order-independence target T2.
+ */
+export function sceneToDirTreeResponse(scene: Scene, rootUUID: string, options?: { shuffle?: () => void; rootName?: string }): DirTreeResponse {
+	const uuidByPath = new Map<string, string>()
+
+	for (const node of scene) {
+		if (node.type === "directory") {
+			uuidByPath.set(node.path, node.uuid)
+		}
+	}
+
+	const parentUUIDOf = (path: string): string => {
+		const parent = parentPath(path)
+
+		return parent.length === 0 ? rootUUID : uuidByPath.get(parent) ?? rootUUID
+	}
+
+	const folders: unknown[] = [[rootUUID, JSON.stringify({ name: options?.rootName ?? "Sync" }), "base"]]
+	const files: unknown[] = []
+
+	for (const node of scene) {
+		if (node.type === "directory") {
+			folders.push([node.uuid, JSON.stringify({ name: basename(node.path) }), parentUUIDOf(node.path)])
+		} else {
+			files.push([
+				node.uuid,
+				"filen-1",
+				"de-1",
+				1,
+				parentUUIDOf(node.path),
+				JSON.stringify({
+					name: basename(node.path),
+					size: node.size,
+					mime: "application/octet-stream",
+					key: "k",
+					lastModified: node.mtime,
+					creation: node.creation,
+					hash: undefined
+				}),
+				2,
+				node.mtime
+			])
+		}
+	}
+
+	return { files, folders, raw: "" } as unknown as DirTreeResponse
+}
+
+/**
+ * Replace a world's remote tree fetcher with one that returns a fixed, pre-built response — so the timed
+ * `getDirectoryTree` measures ONLY the engine's build loop, not the fake cloud's response construction.
+ */
+export function injectDirTreeResponse(world: World, response: DirTreeResponse): void {
+	const fetcher: SyncDirTreeFetcher = async () => response
+
+	world.sync.environment.fetchDirTree = fetcher
+}
+
+/** Reset the remote directory-tree cache so the next getDirectoryTree() rebuilds fully. */
+export function forceRemoteRebuild(world: World): void {
+	world.sync.remoteFileSystem.getDirectoryTreeCache = {
+		timestamp: 0,
+		tree: {},
+		uuids: {},
+		ignored: [],
+		size: 0
+	}
+}

package/tests/bench/harness/trees.ts

@@ -0,0 +1,275 @@
+import { type LocalTree, type LocalItem } from "../../../src/lib/filesystems/local"
+import { type RemoteTree, type RemoteItem } from "../../../src/lib/filesystems/remote"
+
+/**
+ * A scene is an ordered (parents-before-children) list of nodes WITH stable identity (inode + uuid),
+ * from which a matching {@link LocalTree} and {@link RemoteTree} can be built. Identity is what lets a
+ * "current" scene derived from a "previous" one be recognised by the delta engine as a rename/modify
+ * (same inode/uuid, changed path/size) rather than an unrelated add+delete.
+ *
+ * Local identity (inode) and remote identity (uuid) are independent per node, exactly as the engine
+ * treats them — there is no cross-requirement between a path's inode and its uuid.
+ */
+export type GenNode = {
+	path: string
+	type: "file" | "directory"
+	inode: number
+	uuid: string
+	size: number
+	mtime: number
+	creation: number
+}
+
+export type Scene = GenNode[]
+
+const BASE_MTIME = new Date("2024-06-01T00:00:00.000Z").getTime()
+
+let inodeCounter = 1
+let uuidCounter = 1
+
+function nextInode(): number {
+	return inodeCounter++
+}
+
+function nextUUID(): string {
+	// Deterministic, cheap, uuid-shaped enough for the engine (it only uses uuids as map keys / identity).
+	const n = uuidCounter++
+
+	return `uuid-${n.toString(16).padStart(12, "0")}`
+}
+
+/** Reset the identity counters so independent benchmarks start from a clean, deterministic space. */
+export function resetIdentity(): void {
+	inodeCounter = 1
+	uuidCounter = 1
+}
+
+function fileNode(path: string, size = 64): GenNode {
+	return { path, type: "file", inode: nextInode(), uuid: nextUUID(), size, mtime: BASE_MTIME, creation: BASE_MTIME }
+}
+
+function dirNode(path: string): GenNode {
+	return { path, type: "directory", inode: nextInode(), uuid: nextUUID(), size: 0, mtime: BASE_MTIME, creation: BASE_MTIME }
+}
+
+// ---- shape generators (return ordered, parents-first scenes) -------------------------------------
+
+/** All `fileCount` files in the sync root (worst case for one flat directory). */
+export function genFlatScene(fileCount: number): Scene {
+	const scene: Scene = []
+
+	for (let i = 0; i < fileCount; i++) {
+		scene.push(fileNode(`/file-${i}.txt`))
+	}
+
+	return scene
+}
+
+/** `dirCount` directories each holding `filesPerDir` files (a wide, shallow tree). */
+export function genWideScene(dirCount: number, filesPerDir: number): Scene {
+	const scene: Scene = []
+
+	for (let d = 0; d < dirCount; d++) {
+		const dirPath = `/dir-${d}`
+
+		scene.push(dirNode(dirPath))
+
+		for (let f = 0; f < filesPerDir; f++) {
+			scene.push(fileNode(`${dirPath}/file-${f}.txt`))
+		}
+	}
+
+	return scene
+}
+
+/** A single chain `/d0/d1/.../d{depth-1}/leaf.txt` (deep nesting). */
+export function genDeepScene(depth: number): Scene {
+	const scene: Scene = []
+	let prefix = ""
+
+	for (let level = 0; level < depth; level++) {
+		prefix += `/d${level}`
+
+		scene.push(dirNode(prefix))
+	}
+
+	scene.push(fileNode(`${prefix}/leaf.txt`))
+
+	return scene
+}
+
+/**
+ * A balanced tree: each directory has `fanout` subdirectories down to `depth`, and every directory
+ * holds `filesPerDir` files. Realistic mix of files and directories. Ordered parents-first (BFS).
+ */
+export function genBalancedScene(options: { fanout: number; depth: number; filesPerDir: number }): Scene {
+	const { fanout, depth, filesPerDir } = options
+	const scene: Scene = []
+	const queue: { path: string; level: number }[] = [{ path: "", level: 0 }]
+
+	while (queue.length > 0) {
+		const { path, level } = queue.shift()!
+
+		for (let f = 0; f < filesPerDir; f++) {
+			scene.push(fileNode(`${path}/file-${f}.txt`))
+		}
+
+		if (level < depth) {
+			for (let c = 0; c < fanout; c++) {
+				const childPath = `${path}/dir-${level}-${c}`
+
+				scene.push(dirNode(childPath))
+				queue.push({ path: childPath, level: level + 1 })
+			}
+		}
+	}
+
+	// Drop a possible leading "/file..." with empty path → ensure all paths start with "/".
+	return scene
+}
+
+/** Generate a wide scene that targets roughly `targetNodes` total nodes. */
+export function genSceneOfSize(targetNodes: number, filesPerDir = 100): Scene {
+	const dirCount = Math.max(1, Math.round(targetNodes / (filesPerDir + 1)))
+
+	return genWideScene(dirCount, filesPerDir)
+}
+
+// ---- scene → engine tree builders ----------------------------------------------------------------
+
+export function buildLocalTree(scene: Scene): LocalTree {
+	const tree: Record<string, LocalItem> = {}
+	const inodes: Record<number, LocalItem> = {}
+
+	for (const node of scene) {
+		const item: LocalItem = {
+			lastModified: node.mtime,
+			type: node.type,
+			path: node.path,
+			size: node.size,
+			creation: node.creation,
+			inode: node.inode
+		}
+
+		tree[node.path] = item
+		inodes[node.inode] = item
+	}
+
+	return { tree, inodes, size: scene.length }
+}
+
+export function buildRemoteTree(scene: Scene): RemoteTree {
+	const tree: Record<string, RemoteItem> = {}
+	const uuids: Record<string, RemoteItem> = {}
+
+	for (const node of scene) {
+		const item: RemoteItem =
+			node.type === "directory"
+				? { type: "directory", uuid: node.uuid, name: basename(node.path), size: 0, path: node.path }
+				: {
+						type: "file",
+						uuid: node.uuid,
+						name: basename(node.path),
+						size: node.size,
+						mime: "application/octet-stream",
+						lastModified: node.mtime,
+						version: 2,
+						chunks: 1,
+						key: "k",
+						bucket: "filen-1",
+						region: "de-1",
+						creation: node.creation,
+						path: node.path
+				  }
+
+		tree[node.path] = item
+		uuids[node.uuid] = item
+	}
+
+	return { tree, uuids, size: scene.length }
+}
+
+function basename(path: string): string {
+	const i = path.lastIndexOf("/")
+
+	return i === -1 ? path : path.slice(i + 1)
+}
+
+// ---- mutators (return a NEW scene = the "current" side; never mutate the input) -------------------
+
+export function cloneScene(scene: Scene): Scene {
+	return scene.map(n => ({ ...n }))
+}
+
+/** Append `k` brand-new files (fresh identity) under `parent` (default sync root). */
+export function addFiles(scene: Scene, k: number, parent = ""): Scene {
+	const next = cloneScene(scene)
+
+	for (let i = 0; i < k; i++) {
+		next.push(fileNode(`${parent}/added-${i}-${nextInode()}.txt`))
+	}
+
+	return next
+}
+
+/** Remove the last `k` FILE nodes (keeps directories so the tree stays valid). */
+export function deleteFiles(scene: Scene, k: number): Scene {
+	const next = cloneScene(scene)
+	let removed = 0
+
+	for (let i = next.length - 1; i >= 0 && removed < k; i--) {
+		if (next[i]!.type === "file") {
+			next.splice(i, 1)
+
+			removed++
+		}
+	}
+
+	return next
+}
+
+/**
+ * Modify the first `k` files. `side: "local"` keeps inode (a real local edit), `side: "remote"` mints a
+ * new uuid (a real remote re-upload). Both bump size + mtime so the change is detected against the base.
+ */
+export function modifyFiles(scene: Scene, k: number, side: "local" | "remote"): Scene {
+	const next = cloneScene(scene)
+	let changed = 0
+
+	for (const node of next) {
+		if (changed >= k) {
+			break
+		}
+
+		if (node.type === "file") {
+			node.size += 1000
+			node.mtime += 60_000
+
+			if (side === "remote") {
+				node.uuid = nextUUID()
+			}
+
+			changed++
+		}
+	}
+
+	return next
+}
+
+/**
+ * Rename a top-level directory (and rebase every descendant) while preserving identity, so the delta
+ * engine sees a single directory rename carrying the subtree.
+ */
+export function renameTopDir(scene: Scene, fromPath: string, toPath: string): Scene {
+	const next = cloneScene(scene)
+
+	for (const node of next) {
+		if (node.path === fromPath) {
+			node.path = toPath
+		} else if (node.path.startsWith(`${fromPath}/`)) {
+			node.path = toPath + node.path.slice(fromPath.length)
+		}
+	}
+
+	return next
+}

package/tests/bench/local-scan.bench.ts

@@ -0,0 +1,74 @@
+import { describe, it, expect } from "vitest"
+import { bench } from "./harness/measure"
+import { makeScaleWorld, sceneToVfsSpec, forceLocalRescan } from "./harness/scale-world"
+import { genWideScene, genBalancedScene, resetIdentity, type Scene } from "./harness/trees"
+import { type World } from "../harness/world"
+
+/**
+ * Local tree-scan benchmark (targets T5, T10). `localFileSystem.getDirectoryTree` runs whenever the local
+ * side changed: fast-glob walks the whole tree (materialising an N-string array), then per entry does
+ * lstat + access(R_OK) (two fs ops) and builds the LocalItem maps. memfs backs the fs so we measure the
+ * walk + per-entry CPU + allocation, not real disk latency (the relative cost of fast-glob vs the loop
+ * vs the double-stat is what transfers to real disk).
+ */
+
+async function buildWorld(scene: Scene): Promise<World> {
+	return await makeScaleWorld({ mode: "twoWay", initialLocal: sceneToVfsSpec(scene) })
+}
+
+describe("localFileSystem.getDirectoryTree", () => {
+	it("wide tree size sweep", async () => {
+		for (const nodes of [10_000, 50_000, 100_000]) {
+			resetIdentity()
+
+			const scene = genWideScene(Math.max(1, Math.round(nodes / 101)), 100)
+			const world = await buildWorld(scene)
+
+			// Correctness: the scan finds exactly the scene's nodes.
+			forceLocalRescan(world)
+
+			const first = await world.sync.localFileSystem.getDirectoryTree()
+
+			expect(first.result.size).toBe(scene.length)
+
+			await bench({
+				group: "localFileSystem.getDirectoryTree / wide",
+				name: `${nodes} nodes`,
+				n: scene.length,
+				iterations: 4,
+				setup: () => {
+					forceLocalRescan(world)
+
+					return world
+				},
+				run: w => w.sync.localFileSystem.getDirectoryTree()
+			})
+		}
+	})
+
+	it("balanced (directory-heavy) tree", async () => {
+		resetIdentity()
+
+		const scene = genBalancedScene({ fanout: 4, depth: 6, filesPerDir: 8 })
+		const world = await buildWorld(scene)
+
+		forceLocalRescan(world)
+
+		const first = await world.sync.localFileSystem.getDirectoryTree()
+
+		expect(first.result.size).toBe(scene.length)
+
+		await bench({
+			group: "localFileSystem.getDirectoryTree / balanced",
+			name: `fanout4 depth6 (${scene.length} nodes)`,
+			n: scene.length,
+			iterations: 4,
+			setup: () => {
+				forceLocalRescan(world)
+
+				return world
+			},
+			run: w => w.sync.localFileSystem.getDirectoryTree()
+		})
+	})
+})

package/tests/bench/longrun.bench.ts

@@ -0,0 +1,130 @@
+import { describe, it, expect } from "vitest"
+import { recordCustom } from "./harness/measure"
+import { makeScaleWorld, sceneToVfsSpec, forceLocalRescan } from "./harness/scale-world"
+import { genWideScene, resetIdentity } from "./harness/trees"
+import { SYNC_INTERVAL } from "../../src/constants"
+import { LOCAL_ROOT, type World } from "../harness/world"
+
+/**
+ * Long-running benchmark — simulates many "fake hours" of a desktop sync (hundreds of cycles, a small
+ * change each cycle) and watches for two failure modes a long-lived process must not have:
+ *   1. memory leak — heapUsed trending up cycle over cycle (accumulating signals, caches, hashes…).
+ *   2. perf degradation — later cycles getting slower than earlier ones (growing structures, GC churn).
+ * Reports the heap slope (KB/cycle) and the late-vs-early cycle-time ratio.
+ */
+
+function ageDebounce(world: World): void {
+	world.sync.localFileSystem.lastDirectoryChangeTimestamp = Date.now() - SYNC_INTERVAL - 1_000
+}
+
+function gc(): void {
+	globalThis.gc?.()
+}
+
+/** Least-squares slope of y over x (MB per cycle here). */
+function slope(xs: number[], ys: number[]): number {
+	const n = xs.length
+	const meanX = xs.reduce((a, b) => a + b, 0) / n
+	const meanY = ys.reduce((a, b) => a + b, 0) / n
+	let num = 0
+	let den = 0
+
+	for (let i = 0; i < n; i++) {
+		num += (xs[i]! - meanX) * (ys[i]! - meanY)
+		den += (xs[i]! - meanX) ** 2
+	}
+
+	return den === 0 ? 0 : num / den
+}
+
+describe("long-running sync (leak + degradation)", () => {
+	it("hundreds of incremental cycles stay flat in memory and time", async () => {
+		const CYCLES = 300
+		const SAMPLE_EVERY = 25
+
+		resetIdentity()
+
+		const scene = genWideScene(50, 100) // ~5,050 nodes
+		const world = await makeScaleWorld({ mode: "twoWay", initialLocal: sceneToVfsSpec(scene) })
+
+		// Settle the initial sync.
+		for (let i = 0; i < 3; i++) {
+			ageDebounce(world)
+
+			await world.sync.runCycle()
+
+			world.messages.length = 0
+		}
+
+		const targetPath = `${LOCAL_ROOT}/dir-0/file-0.txt`
+		const cycleTimes: number[] = []
+		const sampleCycles: number[] = []
+		const sampleHeapMB: number[] = []
+
+		for (let c = 0; c < CYCLES; c++) {
+			await world.vfs.fs.writeFile(targetPath, "y".repeat(64 + (c % 500)), { encoding: "utf-8" })
+
+			forceLocalRescan(world)
+			ageDebounce(world)
+
+			const t0 = performance.now()
+
+			await world.sync.runCycle()
+
+			cycleTimes.push(performance.now() - t0)
+
+			// Clear the harness message log so we measure ENGINE memory, not the test's growing array.
+			world.messages.length = 0
+
+			if (c % SAMPLE_EVERY === 0) {
+				gc()
+
+				sampleCycles.push(c)
+				sampleHeapMB.push(process.memoryUsage().heapUsed / (1024 * 1024))
+			}
+		}
+
+		gc()
+
+		sampleCycles.push(CYCLES)
+		sampleHeapMB.push(process.memoryUsage().heapUsed / (1024 * 1024))
+
+		const heapSlopeMBPerCycle = slope(sampleCycles, sampleHeapMB)
+		const heapSlopeKBPerCycle = heapSlopeMBPerCycle * 1024
+
+		const half = Math.floor(cycleTimes.length / 2)
+		const earlyAvg = cycleTimes.slice(0, half).reduce((a, b) => a + b, 0) / half
+		const lateAvg = cycleTimes.slice(half).reduce((a, b) => a + b, 0) / (cycleTimes.length - half)
+		const degradationRatio = lateAvg / earlyAvg
+
+		recordCustom({
+			group: "long-run / leak + degradation",
+			name: `${CYCLES} incremental cycles, ${scene.length} nodes`,
+			n: CYCLES,
+			msMean: cycleTimes.reduce((a, b) => a + b, 0) / cycleTimes.length,
+			msMin: Math.min(...cycleTimes),
+			heapRetainedMB: sampleHeapMB[sampleHeapMB.length - 1]!,
+			extra: {
+				heapStartMB: Number(sampleHeapMB[0]!.toFixed(1)),
+				heapEndMB: Number(sampleHeapMB[sampleHeapMB.length - 1]!.toFixed(1)),
+				heapSlopeKBPerCycle: Number(heapSlopeKBPerCycle.toFixed(2)),
+				earlyCycleMs: Number(earlyAvg.toFixed(2)),
+				lateCycleMs: Number(lateAvg.toFixed(2)),
+				degradationRatio: Number(degradationRatio.toFixed(3))
+			}
+		})
+
+		process.stdout.write(
+			`[longrun] heap ${sampleHeapMB[0]!.toFixed(1)}→${sampleHeapMB[sampleHeapMB.length - 1]!.toFixed(
+				1
+			)}MB (slope ${heapSlopeKBPerCycle.toFixed(2)} KB/cycle) | cycle early ${earlyAvg.toFixed(2)}ms late ${lateAvg.toFixed(
+				2
+			)}ms (x${degradationRatio.toFixed(3)})\n`
+		)
+
+		// Leak guard: a flat engine should not trend up more than a few KB/cycle (well under a MB over 300).
+		expect(heapSlopeKBPerCycle).toBeLessThan(200)
+		// Degradation guard: late cycles within 2x of early ones (generous — absorbs gc noise).
+		expect(degradationRatio).toBeLessThan(2)
+	})
+})