@filen/sync 0.2.1 → 0.3.2

package/tests/e2e/harness/account.ts

@@ -0,0 +1,104 @@
+import FilenSDK, { type FilenSDKConfig } from "@filen/sdk"
+import os from "os"
+import pathModule from "path"
+import fs from "fs-extra"
+import { v4 as uuidv4 } from "uuid"
+
+/**
+ * Real-account login for the e2e suite.
+ *
+ * Credentials come from the environment so they never touch the repo, in preference order:
+ *   1. FILEN_SYNC_LEGACY_E2E_SDK_CONFIG — base64(JSON) of an already-authenticated SDK config. Reusing
+ *      a saved session means we DON'T hit the rate-limited login endpoint on every run. Preferred.
+ *   2. FILEN_SYNC_LEGACY_E2E_EMAIL + FILEN_SYNC_LEGACY_E2E_PASSWORD — a DEDICATED throwaway test account
+ *      (no 2FA). Fallback for local runs without a saved config.
+ *
+ * Either way the suite creates and permanently deletes data under the account and empties its trash.
+ * When NEITHER is present the whole e2e suite is skipped (see {@link E2E_ENABLED}); it never fails a
+ * build that simply has no secrets (local dev, fork PRs).
+ */
+export const E2E_ENABLED: boolean = Boolean(
+	process.env["FILEN_SYNC_LEGACY_E2E_SDK_CONFIG"] ||
+		(process.env["FILEN_SYNC_LEGACY_E2E_EMAIL"] && process.env["FILEN_SYNC_LEGACY_E2E_PASSWORD"])
+)
+
+/**
+ * A uniquely-named tmp dir the SDK uses for its own scratch (download chunks etc.). Random per process
+ * so parallel CI runners never collide; removed by {@link teardownTestSDK}.
+ */
+export const SDK_TMP_PATH: string = pathModule.join(os.tmpdir(), `filen-sync-e2e-sdk-${uuidv4()}`)
+
+let sdkPromise: Promise<FilenSDK> | null = null
+
+/**
+ * Log in once per worker and reuse the instance. `connectToSocket: false` — the suite drives discrete
+ * cycles and never needs the realtime socket (which would also keep the process alive past teardown).
+ */
+export function loginTestSDK(): Promise<FilenSDK> {
+	if (!E2E_ENABLED) {
+		throw new Error("e2e credentials are not set (FILEN_SYNC_LEGACY_E2E_SDK_CONFIG, or _EMAIL / _PASSWORD).")
+	}
+
+	if (!sdkPromise) {
+		sdkPromise = (async (): Promise<FilenSDK> => {
+			await fs.ensureDir(SDK_TMP_PATH)
+
+			// Preferred: reuse a saved, already-authenticated SDK config (base64 of its JSON) so we never
+			// touch the rate-limited login endpoint. We override only tmpPath so each worker gets its own
+			// scratch dir; everything else (apiKey, master keys, baseFolderUUID, …) comes from the config.
+			const configBase64 = process.env["FILEN_SYNC_LEGACY_E2E_SDK_CONFIG"]
+
+			if (configBase64 && configBase64.length > 0) {
+				const config = JSON.parse(Buffer.from(configBase64, "base64").toString("utf-8")) as FilenSDKConfig
+
+				return new FilenSDK({
+					...config,
+					tmpPath: SDK_TMP_PATH
+				})
+			}
+
+			// Fallback: a real login with email + password.
+			const email = process.env["FILEN_SYNC_LEGACY_E2E_EMAIL"]
+			const password = process.env["FILEN_SYNC_LEGACY_E2E_PASSWORD"]
+
+			if (!email || !password) {
+				throw new Error("e2e credentials are not set (FILEN_SYNC_LEGACY_E2E_SDK_CONFIG, or _EMAIL / _PASSWORD).")
+			}
+
+			const sdk = new FilenSDK({
+				metadataCache: true,
+				connectToSocket: false,
+				tmpPath: SDK_TMP_PATH
+			})
+
+			await sdk.login({ email, password })
+
+			return sdk
+		})()
+	}
+
+	return sdkPromise
+}
+
+/**
+ * Best-effort logout + scratch cleanup for the end of a run. Safe to call when login never happened.
+ */
+export async function teardownTestSDK(): Promise<void> {
+	if (sdkPromise) {
+		try {
+			const sdk = await sdkPromise
+
+			sdk.logout()
+		} catch {
+			// already gone / never logged in
+		}
+
+		sdkPromise = null
+	}
+
+	try {
+		await fs.rm(SDK_TMP_PATH, { force: true, recursive: true, maxRetries: 10, retryDelay: 100 })
+	} catch {
+		// best effort
+	}
+}

package/tests/e2e/harness/assert.ts

@@ -0,0 +1,127 @@
+import type FilenSDK from "@filen/sdk"
+import { type E2EWorld } from "./world"
+import { LOCAL_TRASH_NAME } from "../../../src/constants"
+import pathModule from "path"
+import fs from "fs-extra"
+import crypto from "crypto"
+import { v4 as uuidv4 } from "uuid"
+
+// The exact param shape the SDK's downloadFileToLocal expects, derived from its public surface so the
+// file-encryption-version type stays in lock-step.
+type DownloadParams = Parameters<ReturnType<FilenSDK["cloud"]>["downloadFileToLocal"]>[0]
+
+export type E2ESnapshotEntry = { type: "file" | "directory"; size: number; contentHash?: string }
+
+/**
+ * `relativePath -> { type, size, contentHash? }`. Local and remote snapshots are directly comparable:
+ * identical content yields the same sha512 on both sides. `contentHash` is only populated when
+ * `withContent` is requested (it costs a download per remote file), so size-only convergence stays cheap.
+ */
+export type E2ESnapshot = Record<string, E2ESnapshotEntry>
+
+// Control files that live locally but never sync to the remote root — excluded so the two sides match.
+const LOCAL_ONLY_ENTRIES = new Set<string>([LOCAL_TRASH_NAME, ".filenignore"])
+
+function sha512Hex(data: Buffer): string {
+	return crypto.createHash("sha512").update(Uint8Array.from(data)).digest("hex")
+}
+
+/**
+ * Snapshot the real local sync directory (relative POSIX paths), skipping the local trash and the
+ * `.filenignore` control file.
+ */
+export async function snapshotLocalReal(world: E2EWorld, options: { withContent?: boolean } = {}): Promise<E2ESnapshot> {
+	const result: E2ESnapshot = {}
+	const root = world.localRoot
+
+	const walk = async (directory: string): Promise<void> => {
+		const entries = await fs.readdir(directory)
+
+		for (const entry of entries) {
+			const full = pathModule.join(directory, entry)
+			const relativePath = full.slice(root.length).split(pathModule.sep).join("/")
+
+			if (relativePath.split("/").length === 2 && LOCAL_ONLY_ENTRIES.has(entry)) {
+				continue
+			}
+
+			const stats = await fs.lstat(full)
+
+			if (stats.isDirectory()) {
+				result[relativePath] = { type: "directory", size: 0 }
+
+				await walk(full)
+			} else if (stats.isFile()) {
+				const entryResult: E2ESnapshotEntry = { type: "file", size: stats.size }
+
+				if (options.withContent) {
+					entryResult.contentHash = sha512Hex(await fs.readFile(full))
+				}
+
+				result[relativePath] = entryResult
+			}
+		}
+	}
+
+	await walk(root)
+
+	return result
+}
+
+/**
+ * Snapshot the real remote `/<runId>` tree (relative POSIX paths) by recursively listing it. With
+ * `withContent`, each file is downloaded to a temp path, hashed, and removed.
+ */
+export async function snapshotRemoteReal(world: E2EWorld, options: { withContent?: boolean } = {}): Promise<E2ESnapshot> {
+	const result: E2ESnapshot = {}
+
+	const walk = async (uuid: string, prefix: string): Promise<void> => {
+		const items = await world.sdk.cloud().listDirectory({ uuid })
+
+		for (const item of items) {
+			const relativePath = `${prefix}/${item.name}`
+
+			if (item.type === "directory") {
+				result[relativePath] = { type: "directory", size: 0 }
+
+				await walk(item.uuid, relativePath)
+			} else {
+				const entryResult: E2ESnapshotEntry = { type: "file", size: item.size }
+
+				if (options.withContent) {
+					entryResult.contentHash = await downloadAndHash(world, item)
+				}
+
+				result[relativePath] = entryResult
+			}
+		}
+	}
+
+	await walk(world.remoteParentUUID, "")
+
+	return result
+}
+
+async function downloadAndHash(
+	world: E2EWorld,
+	item: Pick<DownloadParams, "uuid" | "bucket" | "region" | "chunks" | "version" | "key" | "size">
+): Promise<string> {
+	const to = pathModule.join(world.workRoot, `dl-${uuidv4()}.tmp`)
+
+	try {
+		await world.sdk.cloud().downloadFileToLocal({
+			uuid: item.uuid,
+			bucket: item.bucket,
+			region: item.region,
+			chunks: item.chunks,
+			version: item.version,
+			key: item.key,
+			size: item.size,
+			to
+		})
+
+		return sha512Hex(await fs.readFile(to))
+	} finally {
+		await fs.rm(to, { force: true, maxRetries: 5, retryDelay: 50 }).catch(() => {})
+	}
+}

package/tests/e2e/harness/drive.ts

@@ -0,0 +1,88 @@
+import { type E2EWorld } from "./world"
+import { snapshotLocalReal, snapshotRemoteReal } from "./assert"
+import { type SyncMessage } from "../../../src/types"
+import { expect } from "vitest"
+
+/**
+ * Drive exactly one synchronization cycle and return the messages it emitted.
+ *
+ * `resetCache` (default true) clears the in-memory local+remote tree caches and rolls the
+ * local-change timestamp back, so the cycle re-reads both trees from scratch and immediately passes the
+ * change-debounce — the deterministic way to make a cycle pick up a mutation we just applied. Pass
+ * `false` to exercise the warm-cache / deviceId "unchanged" path (e.g. proving a settled sync no-ops).
+ */
+export async function cycle(world: E2EWorld, options: { resetCache?: boolean } = {}): Promise<SyncMessage[]> {
+	const mark = world.messages.length
+
+	if (options.resetCache ?? true) {
+		world.worker.resetCache(world.syncPair.uuid)
+	}
+
+	await world.sync.runCycle()
+
+	return world.messages.slice(mark)
+}
+
+/**
+ * Run cycles until the world stops changing (a fixpoint of the combined local+remote structure/size
+ * snapshot) or `maxCycles` is hit. Real backends settle a small tree in 2-3 cycles (transfer, then a
+ * confirming no-op); the cap is a safety net, not the expected path.
+ */
+export async function settle(world: E2EWorld, options: { maxCycles?: number; resetCache?: boolean } = {}): Promise<void> {
+	const maxCycles = options.maxCycles ?? 8
+	let previous = ""
+
+	for (let i = 0; i < maxCycles; i++) {
+		await cycle(world, { resetCache: options.resetCache ?? true })
+
+		const [local, remote] = await Promise.all([snapshotLocalReal(world), snapshotRemoteReal(world)])
+		const signature = JSON.stringify([local, remote])
+
+		if (signature === previous) {
+			return
+		}
+
+		previous = signature
+	}
+}
+
+const FILE_TRANSFER_OPS = ["upload", "uploadFile", "download", "downloadFile"] as const
+
+/** The `of` discriminators of every "transfer" message in the stream. */
+export function transferKinds(messages: SyncMessage[]): string[] {
+	return messages.filter((message): message is Extract<SyncMessage, { type: "transfer" }> => message.type === "transfer").map(message => message.data.of)
+}
+
+/** Only the discriminators that represent an actual file transfer (not a dir/rename op). */
+export function transferOps(messages: SyncMessage[]): string[] {
+	return transferKinds(messages).filter(kind => (FILE_TRANSFER_OPS as readonly string[]).includes(kind))
+}
+
+/**
+ * Every operation kind in the stream — file transfers AND directory creates, deletes, and renames (the
+ * full set of `transfer.of` discriminators), deduplicated and sorted for readable failure output.
+ *
+ * `expect(allOps(messages)).toEqual([])` is the assertion for a COMPLETE no-op. transferOps() sees only
+ * file up/downloads, so a cycle that spuriously renamed, deleted, or mkdir'd a path — exactly what a wrong
+ * post-restart base re-derivation would emit — slips past it. A genuine no-op cycle starts no tasks and so
+ * emits no transfer messages of any kind, making this empty.
+ */
+export function allOps(messages: SyncMessage[]): string[] {
+	return [...new Set(transferKinds(messages))].sort()
+}
+
+/** All messages of a given `type`. */
+export function messagesOfType<T extends SyncMessage["type"]>(messages: SyncMessage[], type: T): Extract<SyncMessage, { type: T }>[] {
+	return messages.filter((message): message is Extract<SyncMessage, { type: T }> => message.type === type)
+}
+
+/**
+ * Assert the local and remote trees are identical. With `withContent` (default) every file is hashed on
+ * both sides (downloading the remote copy), so this proves real content identity, not just structure.
+ */
+export async function expectConverged(world: E2EWorld, options: { withContent?: boolean } = {}): Promise<void> {
+	const withContent = options.withContent ?? true
+	const [local, remote] = await Promise.all([snapshotLocalReal(world, { withContent }), snapshotRemoteReal(world, { withContent })])
+
+	expect(local).toEqual(remote)
+}

package/tests/e2e/harness/mutations.ts

@@ -0,0 +1,249 @@
+import { type E2EWorld } from "./world"
+import pathModule from "path"
+import fs from "fs-extra"
+import { v4 as uuidv4 } from "uuid"
+
+/** Resolve a root-relative path (with or without a leading slash) to an absolute local path. */
+function abs(world: E2EWorld, relativePath: string): string {
+	return pathModule.join(world.localRoot, relativePath.replace(/^\/+/, ""))
+}
+
+// ---- Local mutations (real filesystem under the sync root) -------------------------------------------
+
+export async function writeLocal(world: E2EWorld, relativePath: string, content: string | Uint8Array): Promise<void> {
+	const full = abs(world, relativePath)
+
+	await fs.ensureDir(pathModule.dirname(full))
+	await fs.writeFile(full, content)
+}
+
+/**
+ * Modify an existing file's content AND stamp it with a clearly-newer mtime. The engine compares
+ * whole-second mtimes (plus size), so a same-size edit that lands in the same second as the previous
+ * version is — by design — invisible to it. Tests that specifically assert "a modification propagates"
+ * use this to make the change deterministically detectable regardless of how fast the prior cycle ran.
+ */
+export async function modifyLocal(world: E2EWorld, relativePath: string, content: string): Promise<void> {
+	const full = abs(world, relativePath)
+
+	await fs.ensureDir(pathModule.dirname(full))
+	await fs.writeFile(full, content)
+
+	const newer = new Date(Date.now() + 5000)
+
+	await fs.utimes(full, newer, newer)
+}
+
+/**
+ * Overwrite a file with new content while RESTORING its previous mtime, reproducing an edit that
+ * lands in the same whole-second as the last sync (the exact E2E-OBS-002 condition). The whole-second
+ * mtime is unchanged, so only base-relative SIZE comparison can detect the edit.
+ */
+export async function writeLocalPreservingMtime(world: E2EWorld, relativePath: string, content: string): Promise<void> {
+	const full = abs(world, relativePath)
+	const { atime, mtime } = await fs.stat(full)
+
+	await fs.writeFile(full, content)
+	await fs.utimes(full, atime, mtime)
+}
+
+export async function readLocal(world: E2EWorld, relativePath: string): Promise<string> {
+	return await fs.readFile(abs(world, relativePath), { encoding: "utf-8" })
+}
+
+export async function mkdirLocal(world: E2EWorld, relativePath: string): Promise<void> {
+	await fs.ensureDir(abs(world, relativePath))
+}
+
+export async function rmLocal(world: E2EWorld, relativePath: string): Promise<void> {
+	await fs.rm(abs(world, relativePath), { force: true, recursive: true, maxRetries: 10, retryDelay: 100 })
+}
+
+export async function renameLocal(world: E2EWorld, fromRelative: string, toRelative: string): Promise<void> {
+	const to = abs(world, toRelative)
+
+	await fs.ensureDir(pathModule.dirname(to))
+	await fs.move(abs(world, fromRelative), to, { overwrite: true })
+}
+
+export async function existsLocal(world: E2EWorld, relativePath: string): Promise<boolean> {
+	return await fs.pathExists(abs(world, relativePath))
+}
+
+/**
+ * Create a symlink at `linkRelativePath` pointing at `target` (raw link contents — typically an
+ * absolute path). Throws on platforms/permissions that forbid symlink creation (e.g. Windows without
+ * Developer Mode); callers skip the test in that case.
+ */
+export async function symlinkLocal(world: E2EWorld, linkRelativePath: string, target: string): Promise<void> {
+	const full = abs(world, linkRelativePath)
+
+	await fs.ensureDir(pathModule.dirname(full))
+	await fs.symlink(target, full)
+}
+
+// ---- Remote mutations (real SDK calls under the /<runId> root) ---------------------------------------
+
+const segments = (relativePath: string): string[] => relativePath.split("/").map(segment => segment.trim()).filter(Boolean)
+
+/**
+ * Find-or-create the directory chain for a root-relative directory path, returning the leaf directory's
+ * uuid. An empty path resolves to the remote root itself.
+ */
+export async function ensureRemoteDir(world: E2EWorld, relativeDir: string): Promise<string> {
+	let parentUUID = world.remoteParentUUID
+
+	for (const segment of segments(relativeDir)) {
+		const items = await world.sdk.cloud().listDirectory({ uuid: parentUUID })
+		const existing = items.find(item => item.type === "directory" && item.name === segment)
+
+		parentUUID = existing ? existing.uuid : await world.sdk.cloud().createDirectory({ name: segment, parent: parentUUID })
+	}
+
+	return parentUUID
+}
+
+export async function mkdirRemote(world: E2EWorld, relativePath: string): Promise<string> {
+	return await ensureRemoteDir(world, relativePath)
+}
+
+/** Upload `content` to the remote at `relativePath`, creating any missing parent directories. */
+export async function uploadRemote(world: E2EWorld, relativePath: string, content: string): Promise<void> {
+	const parts = segments(relativePath)
+	const name = parts[parts.length - 1]
+
+	if (!name) {
+		throw new Error(`Invalid remote upload path: ${relativePath}`)
+	}
+
+	const parentUUID = await ensureRemoteDir(world, parts.slice(0, -1).join("/"))
+	const source = pathModule.join(world.workRoot, `upload-${uuidv4()}.tmp`)
+
+	await fs.writeFile(source, content)
+
+	try {
+		await world.sdk.cloud().uploadLocalFile({ source, parent: parentUUID, name })
+	} finally {
+		await fs.rm(source, { force: true, maxRetries: 5, retryDelay: 50 }).catch(() => {})
+	}
+}
+
+/** The CloudItem at a root-relative path, or null if absent. */
+export async function resolveRemote(
+	world: E2EWorld,
+	relativePath: string
+): Promise<Awaited<ReturnType<ReturnType<E2EWorld["sdk"]["cloud"]>["listDirectory"]>>[number] | null> {
+	const parts = segments(relativePath)
+
+	if (parts.length === 0) {
+		return null
+	}
+
+	let parentUUID = world.remoteParentUUID
+
+	for (let i = 0; i < parts.length; i++) {
+		const items = await world.sdk.cloud().listDirectory({ uuid: parentUUID })
+		const match = items.find(item => item.name === parts[i])
+
+		if (!match) {
+			return null
+		}
+
+		if (i === parts.length - 1) {
+			return match
+		}
+
+		if (match.type !== "directory") {
+			return null
+		}
+
+		parentUUID = match.uuid
+	}
+
+	return null
+}
+
+/**
+ * Rename a remote DIRECTORY within its current parent (simulating a peer client's rename), via the SDK
+ * renameDirectory call the engine itself uses.
+ */
+export async function renameRemoteDir(world: E2EWorld, fromRelative: string, toRelative: string): Promise<void> {
+	const item = await resolveRemote(world, fromRelative)
+
+	if (!item || item.type !== "directory") {
+		throw new Error(`renameRemoteDir: no remote directory at ${fromRelative}`)
+	}
+
+	const newName = segments(toRelative).pop()
+
+	if (!newName) {
+		throw new Error(`renameRemoteDir: invalid target ${toRelative}`)
+	}
+
+	await world.sdk.cloud().renameDirectory({
+		uuid: item.uuid,
+		name: newName,
+		overwriteIfExists: true
+	})
+}
+
+/**
+ * Rename a remote FILE within its current directory (simulating a peer client's rename), via the SDK
+ * renameFile call the engine itself uses. Only same-directory file renames are needed by the conflict
+ * suite, so the metadata is rebuilt from the resolved item.
+ */
+export async function renameRemote(world: E2EWorld, fromRelative: string, toRelative: string): Promise<void> {
+	const item = await resolveRemote(world, fromRelative)
+
+	if (!item || item.type !== "file") {
+		throw new Error(`renameRemote: no remote file at ${fromRelative}`)
+	}
+
+	const newName = segments(toRelative).pop()
+
+	if (!newName) {
+		throw new Error(`renameRemote: invalid target ${toRelative}`)
+	}
+
+	await world.sdk.cloud().renameFile({
+		uuid: item.uuid,
+		name: newName,
+		metadata: {
+			name: newName,
+			size: item.size,
+			mime: item.mime,
+			key: item.key,
+			lastModified: item.lastModified,
+			...(item.creation !== undefined ? { creation: item.creation } : {}),
+			...(item.hash !== undefined ? { hash: item.hash } : {})
+		},
+		overwriteIfExists: true
+	})
+}
+
+/** Force a local file's mtime to a specific epoch-ms (e.g. to age it before a newer conflicting write). */
+export async function setLocalMtime(world: E2EWorld, relativePath: string, epochMs: number): Promise<void> {
+	const when = new Date(epochMs)
+
+	await fs.utimes(abs(world, relativePath), when, when)
+}
+
+/** Change a local file's permission bits (e.g. 0o000 to make it unreadable for a fault-tolerance test). */
+export async function chmodLocal(world: E2EWorld, relativePath: string, mode: number): Promise<void> {
+	await fs.chmod(abs(world, relativePath), mode)
+}
+
+/** Permanently delete (no trash) the remote item at `relativePath`. */
+export async function deleteRemote(world: E2EWorld, relativePath: string): Promise<void> {
+	const item = await resolveRemote(world, relativePath)
+
+	if (!item) {
+		return
+	}
+
+	if (item.type === "directory") {
+		await world.sdk.cloud().deleteDirectory({ uuid: item.uuid })
+	} else {
+		await world.sdk.cloud().deleteFile({ uuid: item.uuid })
+	}
+}