@statewalker/webrun-files-sync 0.2.2

package/src/execute.ts

@@ -0,0 +1,154 @@
+/**
+ * {@link execute} — applies a {@link SyncPlan}, emitting {@link SyncEvent}s.
+ *
+ * - Every copy/update is verified at the destination per {@link VerificationMode}.
+ * - `move` never deletes a source until its destination is verified (the delete
+ *   action re-verifies before removing).
+ * - Interruptible + resumable: completed action indices are recorded in the
+ *   injected {@link CheckpointStore} before each `done` event, so a second call
+ *   with the same store completes the remainder with no duplicate work.
+ * - `check` mutates nothing.
+ * - After a fully-clean `bisync`, a fresh anchor is written from the merged state.
+ */
+
+import type { FileStats, FilesApi } from "@statewalker/webrun-files";
+import { buildAnchor } from "./anchor.js";
+import { createStreamingTransfer } from "./transfer.js";
+import type {
+  ByteStream,
+  SyncAction,
+  SyncEvent,
+  SyncOp,
+  SyncOptions,
+  SyncPlan,
+  Transfer,
+  VerificationMode,
+} from "./types.js";
+
+async function verifyCopy(
+  mode: VerificationMode,
+  source: FilesApi,
+  dest: FilesApi,
+  path: string,
+  hashContent: (b: ByteStream) => Promise<string>,
+): Promise<boolean> {
+  if (mode === "content-hash" || mode === "backend-hash" || mode === "read-after-write") {
+    const dstat = await dest.stats(path);
+    if (!dstat) return false;
+    const [hs, hd] = await Promise.all([
+      hashContent(source.read(path)),
+      hashContent(dest.read(path)),
+    ]);
+    return hs === hd;
+  }
+  // "size" (default) and "mtime" — cross-endpoint mtime is not comparable, so
+  // both settle on the byte-count check.
+  const [s, d] = await Promise.all([source.stats(path), dest.stats(path)]);
+  return !!s && !!d && sizeOf(s) === sizeOf(d);
+}
+
+/** Byte length of a file; `undefined` for a directory (two directories compare equal). */
+function sizeOf(stats: FileStats): number | undefined {
+  return stats.kind === "file" ? stats.size : undefined;
+}
+
+interface Outcome {
+  ok: boolean;
+  skipped?: boolean;
+  reason?: string;
+}
+
+async function applyAction(
+  action: SyncAction,
+  a: FilesApi,
+  b: FilesApi,
+  op: SyncOp,
+  transfer: Transfer,
+  verify: VerificationMode,
+  opts: SyncOptions,
+): Promise<Outcome> {
+  const pick = (s: "a" | "b") => (s === "a" ? a : b);
+  switch (action.kind) {
+    case "mkdir": {
+      // One-way ops create missing directories on the destination (B).
+      await b.mkdir(action.path);
+      return { ok: true };
+    }
+    case "copy":
+    case "update": {
+      const from = pick(action.from);
+      const to = action.from === "a" ? b : a;
+      await transfer.run(action, from, to);
+      const ok = await verifyCopy(verify, from, to, action.path, opts.hashContent);
+      return ok ? { ok: true } : { ok: false, reason: "destination verification failed" };
+    }
+    case "move": {
+      await pick(action.side).move(action.from, action.to);
+      return { ok: true };
+    }
+    case "delete": {
+      // A `move` source cleanup: never delete until the destination verifies.
+      if (op === "move" && action.side === "a") {
+        const verified = await verifyCopy(verify, a, b, action.path, opts.hashContent);
+        if (!verified) {
+          return { ok: false, skipped: true, reason: "destination not verified; source retained" };
+        }
+        await a.remove(action.path);
+        return { ok: true };
+      }
+      await pick(action.side).remove(action.path);
+      return { ok: true };
+    }
+  }
+}
+
+export async function* execute(
+  plan: SyncPlan,
+  a: FilesApi,
+  b: FilesApi,
+  opts: SyncOptions,
+): AsyncIterable<SyncEvent> {
+  const transfer = opts.transfer ?? createStreamingTransfer();
+  const verify = opts.verify ?? "size";
+
+  for (const message of plan.warnings ?? []) yield { type: "warning", message };
+  for (const conflict of plan.conflicts) yield { type: "conflict", conflict };
+
+  if (plan.op === "check") return; // compare-only: transfers nothing, mutates nothing.
+
+  const done = (await opts.checkpoint?.read()) ?? new Set<number>();
+  let anyFailure = false;
+
+  for (let i = 0; i < plan.actions.length; i++) {
+    if (done.has(i)) continue;
+    const action = plan.actions[i];
+    yield { type: "start", index: i, action };
+    try {
+      const outcome = await applyAction(action, a, b, plan.op, transfer, verify, opts);
+      if (!outcome.ok) {
+        anyFailure = true;
+        const reason = outcome.reason ?? "failed";
+        yield outcome.skipped
+          ? { type: "skipped", index: i, action, reason }
+          : { type: "failed", index: i, action, reason };
+        continue;
+      }
+      await opts.checkpoint?.add(i);
+      yield { type: "done", index: i, action };
+    } catch (error) {
+      anyFailure = true;
+      yield {
+        type: "failed",
+        index: i,
+        action,
+        reason: String((error as Error)?.message ?? error),
+      };
+    }
+  }
+
+  // A clean bisync converges both endpoints; record the merged state as the
+  // next base. Skipped if anything failed or any conflict was left unresolved.
+  if (plan.op === "bisync" && !anyFailure && plan.conflicts.length === 0 && opts.anchorStore) {
+    await opts.anchorStore.write(opts.pairKey ?? "default", await buildAnchor(a, opts.hashContent));
+  }
+}

package/src/index.ts

@@ -0,0 +1,35 @@
+/**
+ * @statewalker/webrun-files-sync
+ *
+ * rclone-like file-synchronisation engine (copy / sync / bisync / move / check)
+ * over two `@statewalker/webrun-files` endpoints. Plan-then-execute: {@link plan}
+ * is pure and produces a serializable {@link SyncPlan}; {@link execute} applies
+ * it, verifying and resuming. Bidirectional sync is a three-way merge over a
+ * sync-owned anchor, delegated to `@statewalker/webrun-merge`.
+ *
+ * Domain-neutral: this package imports ONLY webrun-files + webrun-merge and knows
+ * nothing about git or versioning (Axis A / Axis B ban).
+ */
+
+export { buildAnchor } from "./anchor.js";
+export { isChanged, snapshot } from "./change-detection.js";
+export { execute } from "./execute.js";
+export { plan } from "./plan.js";
+export { createStreamingTransfer } from "./transfer.js";
+export type {
+  AnchorStore,
+  ByteStream,
+  CheckpointStore,
+  FilesApi,
+  PathFilter,
+  Resolution,
+  SyncAction,
+  SyncAnchor,
+  SyncConflict,
+  SyncEvent,
+  SyncOp,
+  SyncOptions,
+  SyncPlan,
+  Transfer,
+  VerificationMode,
+} from "./types.js";

package/src/plan.ts

@@ -0,0 +1,60 @@
+/**
+ * {@link plan} — pure planning for all sync operations. It reads both endpoints
+ * (listing, and hashing only when the ladder demands it) but MUTATES NEITHER.
+ *
+ *   copy   — new/changed A files → B; extraneous B files KEPT.
+ *   sync   — copy, plus extraneous B files DELETED.
+ *   move   — copy each A file, then (at execute, after the destination is
+ *            verified) delete the source; source is never lost.
+ *   check  — the copy diff, transferring nothing (execute is a no-op).
+ *   bisync — three-way merge over the anchor (see bisync.ts).
+ */
+
+import type { FilesApi } from "@statewalker/webrun-files";
+import { planBisync } from "./bisync.js";
+import { isChanged, snapshot } from "./change-detection.js";
+import type { SyncAction, SyncOp, SyncOptions, SyncPlan } from "./types.js";
+
+export async function plan(
+  a: FilesApi,
+  b: FilesApi,
+  op: SyncOp,
+  opts: SyncOptions,
+): Promise<SyncPlan> {
+  if (op === "bisync") return planBisync(a, b, opts);
+
+  const [snapA, snapB] = await Promise.all([snapshot(a, opts.filter), snapshot(b, opts.filter)]);
+  const actions: SyncAction[] = [];
+
+  // A → B: create missing directories, copy new files, update changed files.
+  for (const [path, entry] of snapA) {
+    const other = snapB.get(path);
+    if (entry.kind === "directory") {
+      if (!other) actions.push({ kind: "mkdir", path });
+      continue;
+    }
+    if (!other) {
+      actions.push({ kind: "copy", path, from: "a" });
+    } else if (await isChanged(a, b, path, entry, other, opts)) {
+      actions.push({ kind: "update", path, from: "a" });
+    }
+  }
+
+  // move: schedule source cleanup after each copy (execute verifies first).
+  if (op === "move") {
+    for (const [path, entry] of snapA) {
+      if (entry.kind === "file") actions.push({ kind: "delete", path, side: "a" });
+    }
+  }
+
+  // sync: delete files in B that no longer exist in A.
+  if (op === "sync") {
+    for (const [path, entry] of snapB) {
+      if (entry.kind === "file" && !snapA.has(path)) {
+        actions.push({ kind: "delete", path, side: "b" });
+      }
+    }
+  }
+
+  return { op, actions, conflicts: [] };
+}

package/src/transfer.ts

@@ -0,0 +1,30 @@
+/**
+ * The default {@link Transfer}: a whole-file copy.
+ *
+ * When source and destination are the SAME backend instance, it uses the
+ * backend's native server-side `copy` ("capabilities allow"). Across two
+ * distinct backends — the normal sync case, and all MemFilesApi pairs — it falls
+ * back to streaming the bytes through `to.write(from.read(...))`.
+ *
+ * A later phase can plug a chunk-dedup Transfer into this same seam.
+ */
+
+import type { FilesApi } from "@statewalker/webrun-files";
+import type { SyncAction, Transfer } from "./types.js";
+
+export function createStreamingTransfer(): Transfer {
+  return {
+    async run(action: SyncAction, from: FilesApi, to: FilesApi): Promise<void> {
+      if (action.kind !== "copy" && action.kind !== "update") {
+        throw new Error(`streaming transfer only handles copy/update, got ${action.kind}`);
+      }
+      const path = action.path;
+      if (from === to) {
+        // Same backend: server-side copy avoids round-tripping the bytes.
+        await from.copy(path, path);
+        return;
+      }
+      await to.write(path, from.read(path));
+    },
+  };
+}

package/src/types.ts

@@ -0,0 +1,123 @@
+/**
+ * Public contract for the rclone-like file-synchronisation engine.
+ *
+ * The engine plans-then-executes over two {@link FilesApi} endpoints. {@link plan}
+ * is pure (mutates neither endpoint) and returns a serializable {@link SyncPlan};
+ * {@link execute} applies it, emitting {@link SyncEvent}s and resuming from an
+ * optional checkpoint after interruption.
+ */
+
+import type { FilesApi } from "@statewalker/webrun-files";
+// webrun-merge is the authority for the three-way merge result and conflict shapes.
+// It exports the conflict type as `Conflict`; we surface it under the sync name.
+import type { Resolution, Conflict as SyncConflict } from "@statewalker/webrun-merge";
+
+export type { FilesApi, Resolution, SyncConflict };
+
+/** A stream of bytes, as produced by {@link FilesApi.read}. */
+export type ByteStream = AsyncIterable<Uint8Array>;
+
+/**
+ * Path predicate: return `true` to INCLUDE a path in the sync, `false` to skip.
+ * `webrun-files` does not export a `PathFilter`, so it is defined here.
+ */
+export type PathFilter = (path: string) => boolean;
+
+export type SyncOp = "copy" | "sync" | "bisync" | "move" | "check";
+
+/**
+ * How a transferred file is verified at the destination.
+ * `size` compares byte counts (cheap default); `content-hash`/`backend-hash`/
+ * `read-after-write` re-hash the destination bytes; `mtime` falls back to size
+ * (cross-endpoint mtime is not comparable).
+ */
+export type VerificationMode =
+  | "size"
+  | "mtime"
+  | "backend-hash"
+  | "content-hash"
+  | "read-after-write";
+
+/**
+ * A single typed step of a {@link SyncPlan}. Serializable by construction.
+ * `copy`/`update` read `path` from the `from` endpoint and write it to the other;
+ * `move` renames within one `side`; `delete` removes `path` from one `side`;
+ * `mkdir` creates an (empty) directory on the destination of a one-way op.
+ *
+ * (The grilled note also listed a `metadata` action; it is intentionally omitted
+ * — the same note puts "arbitrary metadata beyond FilesApi size/mtime" out of
+ * scope, and MemFilesApi exposes no metadata setter, so it would be dead code.)
+ */
+export type SyncAction =
+  | { kind: "mkdir"; path: string }
+  | { kind: "copy"; path: string; from: "a" | "b" }
+  | { kind: "update"; path: string; from: "a" | "b" }
+  | { kind: "move"; from: string; to: string; side: "a" | "b" }
+  | { kind: "delete"; path: string; side: "a" | "b" };
+
+/** A serializable, dry-runnable, auditable plan. */
+export interface SyncPlan {
+  op: SyncOp;
+  actions: SyncAction[];
+  conflicts: SyncConflict[];
+  /** Non-fatal notices raised while planning (e.g. first bisync with no anchor). */
+  warnings?: string[];
+}
+
+/** Events streamed by {@link execute}. */
+export type SyncEvent =
+  | { type: "warning"; message: string }
+  | { type: "conflict"; conflict: SyncConflict }
+  | { type: "start"; index: number; action: SyncAction }
+  | { type: "done"; index: number; action: SyncAction }
+  | { type: "skipped"; index: number; action: SyncAction; reason: string }
+  | { type: "failed"; index: number; action: SyncAction; reason: string };
+
+/**
+ * Executes a single copy/update. Injected so a later phase can plug a
+ * chunk-dedup transfer into the SAME seam; the default streams whole files.
+ */
+export interface Transfer {
+  run(action: SyncAction, from: FilesApi, to: FilesApi): Promise<void>;
+}
+
+/** Content-id + metadata manifest of the last successful sync (bisync base). */
+export interface SyncAnchor {
+  entries: Record<string, { contentId: string; size: number; mtime: number }>;
+}
+
+/** Persistence for the {@link SyncAnchor} of an endpoint pair (bisync). */
+export interface AnchorStore {
+  read(pairKey: string): Promise<SyncAnchor | undefined>;
+  write(pairKey: string, anchor: SyncAnchor): Promise<void>;
+}
+
+/**
+ * Progress checkpoint enabling resume: records the indices of completed actions.
+ * `execute` skips completed indices and records each new one before yielding its
+ * `done` event, so an interrupted run resumes with no duplicate work.
+ */
+export interface CheckpointStore {
+  read(): Promise<Set<number>>;
+  add(index: number): Promise<void>;
+}
+
+export interface SyncOptions {
+  /** Content identity — the ladder terminal and the id shared with webrun-merge. */
+  hashContent: (input: ByteStream) => Promise<string>;
+  filter?: PathFilter;
+  /** Copy/update executor; defaults to whole-file streaming copy. */
+  transfer?: Transfer;
+  /** Destination verification mode; defaults to `size`. */
+  verify?: VerificationMode;
+  /** SyncAnchor persistence (bisync only). */
+  anchorStore?: AnchorStore;
+  /** Identifies the endpoint pair for the anchor store (bisync); defaults to "default". */
+  pairKey?: string;
+  /** Injected conflict-resolution policy (bisync). */
+  resolve?: (c: SyncConflict) => Resolution | undefined;
+  /** Enable the head/tail sample rung of the change-detection ladder. */
+  quickFingerprint?: boolean;
+  /** Resume support for {@link execute}. */
+  checkpoint?: CheckpointStore;
+}