@statewalker/webrun-files-sync 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +59 -0
- package/dist/anchor.d.ts +19 -0
- package/dist/anchor.d.ts.map +1 -0
- package/dist/bisync.d.ts +24 -0
- package/dist/bisync.d.ts.map +1 -0
- package/dist/change-detection.d.ts +33 -0
- package/dist/change-detection.d.ts.map +1 -0
- package/dist/execute.d.ts +16 -0
- package/dist/execute.d.ts.map +1 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +422 -0
- package/dist/plan.d.ts +15 -0
- package/dist/plan.d.ts.map +1 -0
- package/dist/transfer.d.ts +13 -0
- package/dist/transfer.d.ts.map +1 -0
- package/dist/types.d.ts +140 -0
- package/dist/types.d.ts.map +1 -0
- package/package.json +51 -0
- package/src/anchor.ts +107 -0
- package/src/bisync.ts +107 -0
- package/src/change-detection.ts +131 -0
- package/src/execute.ts +154 -0
- package/src/index.ts +35 -0
- package/src/plan.ts +60 -0
- package/src/transfer.ts +30 -0
- package/src/types.ts +123 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024-present StateWalker Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# @statewalker/webrun-files-sync
|
|
2
|
+
|
|
3
|
+
rclone-like file-synchronisation engine (copy / sync / bisync / move / check) over two `FilesApi` endpoints.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
`files-sync` is Axis A of the two-axis VCS architecture — the git-independent "sync" engine. It **plans, then executes**: `plan(a, b, op, opts)` is pure (mutates neither endpoint) and returns a serializable `SyncPlan`; `execute(plan, a, b, opts)` applies it, emitting events and resuming from an optional checkpoint after interruption. Bidirectional `bisync` is a genuine **three-way merge over a sync-owned `SyncAnchor`** (delegated to `@statewalker/webrun-merge`), never two chained one-way syncs.
|
|
8
|
+
|
|
9
|
+
It imports **only** `@statewalker/webrun-files` and `@statewalker/webrun-merge` and knows nothing of git or versioning — that is the hard Axis A ✗↔ Axis B ban. Content identity (`hashContent`), the per-copy `Transfer` strategy, and conflict resolution are all injected.
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pnpm add @statewalker/webrun-files-sync
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Quick Start
|
|
18
|
+
|
|
19
|
+
```typescript
|
|
20
|
+
import { createHash } from "node:crypto";
|
|
21
|
+
import { MemFilesApi } from "@statewalker/webrun-files-mem";
|
|
22
|
+
import { execute, plan } from "@statewalker/webrun-files-sync";
|
|
23
|
+
|
|
24
|
+
async function sha256(input: AsyncIterable<Uint8Array>): Promise<string> {
|
|
25
|
+
const h = createHash("sha256");
|
|
26
|
+
for await (const chunk of input) h.update(chunk);
|
|
27
|
+
return `sha256:${h.digest("hex")}`;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const a = new MemFilesApi({ initialFiles: { "/new.txt": "new", "/changed.txt": "v2" } });
|
|
31
|
+
const b = new MemFilesApi({ initialFiles: { "/changed.txt": "v1", "/extra.txt": "keep" } });
|
|
32
|
+
|
|
33
|
+
// Pure plan — dry-runnable, serializable, mutates nothing.
|
|
34
|
+
const p = await plan(a, b, "copy", { hashContent: sha256 });
|
|
35
|
+
|
|
36
|
+
// Execute, streaming per-action events.
|
|
37
|
+
for await (const event of execute(p, a, b, { hashContent: sha256 })) {
|
|
38
|
+
if (event.type === "done") console.log("synced", event.action);
|
|
39
|
+
}
|
|
40
|
+
// b now has new.txt + updated changed.txt; extra.txt is kept (copy never deletes).
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## API
|
|
44
|
+
|
|
45
|
+
- **`plan(a, b, op, opts): Promise<SyncPlan>`** — pure planner. `op` is `"copy" | "sync" | "bisync" | "move" | "check"`. `copy` never deletes destination files; `sync` deletes extraneous ones; `move` is copy+verify+delete; `check` compares only; `bisync` runs the 3-way merge.
|
|
46
|
+
- **`execute(plan, a, b, opts): AsyncIterable<SyncEvent>`** — applies a plan, verifying each action and skipping already-completed indices from `opts.checkpoint`.
|
|
47
|
+
- **`buildAnchor(...)`** — build a `SyncAnchor` (bisync base) from an endpoint.
|
|
48
|
+
- **`snapshot(files, filter?)` / `isChanged(...)`** — the change-detection ladder primitives.
|
|
49
|
+
- **`createStreamingTransfer(): Transfer`** — the default whole-file streaming copy executor.
|
|
50
|
+
|
|
51
|
+
`SyncOptions` carries the required `hashContent`, plus optional `filter`, `transfer`, `verify` (`VerificationMode`), `anchorStore` / `pairKey` (bisync), `resolve`, `quickFingerprint`, and `checkpoint`. Types: `SyncPlan`, `SyncAction`, `SyncConflict`, `SyncEvent`, `SyncAnchor`, `AnchorStore`, `CheckpointStore`, `Transfer`.
|
|
52
|
+
|
|
53
|
+
## Notes
|
|
54
|
+
|
|
55
|
+
- **Plan-then-execute + resumable.** Plans are serializable and auditable; `execute` records each completed action index, so an interrupted run resumes with no duplicate work.
|
|
56
|
+
- **Cheap-first change detection.** A short-circuiting ladder (path/type → size → stable mtime → optional quick fingerprint → full `hashContent`) avoids hashing byte-identical files.
|
|
57
|
+
- **Injected `Transfer` seam.** The default streams whole files; a chunk-dedup transfer (from `@statewalker/webrun-content-transfer`) plugs into the *same* seam with no rewrite.
|
|
58
|
+
- **bisync = 3-way over an anchor** via `@statewalker/webrun-merge`, using the same injected content id — not two one-way syncs.
|
|
59
|
+
- Built red/green TDD.
|
package/dist/anchor.d.ts
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* {@link SyncAnchor} helpers — building a manifest from an endpoint and
|
|
3
|
+
* reconstructing a read-only "base" {@link FilesApi} view from a manifest.
|
|
4
|
+
*/
|
|
5
|
+
import type { FilesApi } from "@statewalker/webrun-files";
|
|
6
|
+
import type { ByteStream, SyncAnchor } from "./types.js";
|
|
7
|
+
/** Build a fresh anchor manifest by hashing every file in an endpoint. */
|
|
8
|
+
export declare function buildAnchor(files: FilesApi, hashContent: (b: ByteStream) => Promise<string>): Promise<SyncAnchor>;
|
|
9
|
+
/**
|
|
10
|
+
* A read-only base tree reconstructed from a {@link SyncAnchor}.
|
|
11
|
+
*
|
|
12
|
+
* Contract decision 1: the anchor stores NO base bytes. Base content is resolved
|
|
13
|
+
* ON DEMAND by content-id from whichever live endpoint still holds it (both `a`
|
|
14
|
+
* and `b` are searched, hashing lazily). When no endpoint holds the id — i.e.
|
|
15
|
+
* both sides changed the file — `read` yields nothing, and the both-modified
|
|
16
|
+
* path resolves to a conflict in webrun-merge rather than a bogus merge.
|
|
17
|
+
*/
|
|
18
|
+
export declare function anchorBaseView(anchor: SyncAnchor, a: FilesApi, b: FilesApi, hashContent: (input: ByteStream) => Promise<string>): FilesApi;
|
|
19
|
+
//# sourceMappingURL=anchor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"anchor.d.ts","sourceRoot":"","sources":["../src/anchor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAGV,QAAQ,EAGT,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEzD,0EAA0E;AAC1E,wBAAsB,WAAW,CAC/B,KAAK,EAAE,QAAQ,EACf,WAAW,EAAE,CAAC,CAAC,EAAE,UAAU,KAAK,OAAO,CAAC,MAAM,CAAC,GAC9C,OAAO,CAAC,UAAU,CAAC,CAWrB;AAED;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,UAAU,EAClB,CAAC,EAAE,QAAQ,EACX,CAAC,EAAE,QAAQ,EACX,WAAW,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,MAAM,CAAC,GAClD,QAAQ,CA6DV"}
|
package/dist/bisync.d.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bidirectional sync — a THREE-WAY merge over a sync-owned {@link SyncAnchor},
|
|
3
|
+
* delegated to `@statewalker/webrun-merge`. This is NEVER two chained one-way syncs.
|
|
4
|
+
*
|
|
5
|
+
* base = the anchor manifest (with base bytes resolved on demand, see anchor.ts)
|
|
6
|
+
* left = endpoint `a` right = endpoint `b`
|
|
7
|
+
*
|
|
8
|
+
* webrun-merge returns operations relative to the merged tree; each is translated
|
|
9
|
+
* into {@link SyncAction}s applied to BOTH endpoints, and its conflicts are
|
|
10
|
+
* surfaced (or folded away by an injected `resolve`, inside webrun-merge).
|
|
11
|
+
*
|
|
12
|
+
* both-modified policy: files-sync installs a content merger that ALWAYS
|
|
13
|
+
* conflicts, so a file changed on both sides is a conflict (optionally resolved
|
|
14
|
+
* by the injected policy) — it is never silently line-merged.
|
|
15
|
+
*
|
|
16
|
+
* No-anchor first bisync: with no anchor there is no base, so a true 3-way merge
|
|
17
|
+
* is impossible. We fall back to a CONSERVATIVE UNION — propagate additions both
|
|
18
|
+
* ways, delete nothing — and emit a warning. Files present on both sides with
|
|
19
|
+
* differing content are surfaced as conflicts, never overwritten.
|
|
20
|
+
*/
|
|
21
|
+
import type { FilesApi } from "@statewalker/webrun-files";
|
|
22
|
+
import type { SyncOptions, SyncPlan } from "./types.js";
|
|
23
|
+
export declare function planBisync(a: FilesApi, b: FilesApi, opts: SyncOptions): Promise<SyncPlan>;
|
|
24
|
+
//# sourceMappingURL=bisync.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bisync.d.ts","sourceRoot":"","sources":["../src/bisync.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAI1D,OAAO,KAAK,EAA4B,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAIlF,wBAAsB,UAAU,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,CA2C/F"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The cheap-first change-detection ladder.
|
|
3
|
+
*
|
|
4
|
+
* Short-circuits from the cheapest signal to the most expensive:
|
|
5
|
+
* path/type → size → stable mtime → optional quick fingerprint →
|
|
6
|
+
* full content hash (injected `hashContent`).
|
|
7
|
+
*
|
|
8
|
+
* Each rung that can decide "same" or "changed" returns without paying for the
|
|
9
|
+
* next one — in particular a size difference marks a file changed WITHOUT ever
|
|
10
|
+
* hashing, and an equal size+mtime pair is treated as unchanged (rclone's
|
|
11
|
+
* default heuristic) without hashing either.
|
|
12
|
+
*/
|
|
13
|
+
import type { FilesApi } from "@statewalker/webrun-files";
|
|
14
|
+
import type { ByteStream, PathFilter } from "./types.js";
|
|
15
|
+
export interface Entry {
|
|
16
|
+
path: string;
|
|
17
|
+
kind: "file" | "directory";
|
|
18
|
+
size?: number;
|
|
19
|
+
mtime?: number;
|
|
20
|
+
}
|
|
21
|
+
export type Snapshot = Map<string, Entry>;
|
|
22
|
+
/** List a whole tree into a path-keyed snapshot, applying an optional filter. */
|
|
23
|
+
export declare function snapshot(files: FilesApi, filter?: PathFilter): Promise<Snapshot>;
|
|
24
|
+
export interface LadderOptions {
|
|
25
|
+
hashContent: (input: ByteStream) => Promise<string>;
|
|
26
|
+
quickFingerprint?: boolean;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Decide whether `from`'s file at `path` differs from `to`'s file, climbing the
|
|
30
|
+
* ladder only as far as needed. `fromEntry` must exist; `toEntry` may be absent.
|
|
31
|
+
*/
|
|
32
|
+
export declare function isChanged(from: FilesApi, to: FilesApi, path: string, fromEntry: Entry, toEntry: Entry | undefined, opts: LadderOptions): Promise<boolean>;
|
|
33
|
+
//# sourceMappingURL=change-detection.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"change-detection.d.ts","sourceRoot":"","sources":["../src/change-detection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEzD,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAE1C,iFAAiF;AACjF,wBAAsB,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,CAYtF;AAwCD,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACpD,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,QAAQ,EACd,EAAE,EAAE,QAAQ,EACZ,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,KAAK,EAChB,OAAO,EAAE,KAAK,GAAG,SAAS,EAC1B,IAAI,EAAE,aAAa,GAClB,OAAO,CAAC,OAAO,CAAC,CAoClB"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* {@link execute} — applies a {@link SyncPlan}, emitting {@link SyncEvent}s.
|
|
3
|
+
*
|
|
4
|
+
* - Every copy/update is verified at the destination per {@link VerificationMode}.
|
|
5
|
+
* - `move` never deletes a source until its destination is verified (the delete
|
|
6
|
+
* action re-verifies before removing).
|
|
7
|
+
* - Interruptible + resumable: completed action indices are recorded in the
|
|
8
|
+
* injected {@link CheckpointStore} before each `done` event, so a second call
|
|
9
|
+
* with the same store completes the remainder with no duplicate work.
|
|
10
|
+
* - `check` mutates nothing.
|
|
11
|
+
* - After a fully-clean `bisync`, a fresh anchor is written from the merged state.
|
|
12
|
+
*/
|
|
13
|
+
import type { FilesApi } from "@statewalker/webrun-files";
|
|
14
|
+
import type { SyncEvent, SyncOptions, SyncPlan } from "./types.js";
|
|
15
|
+
export declare function execute(plan: SyncPlan, a: FilesApi, b: FilesApi, opts: SyncOptions): AsyncIterable<SyncEvent>;
|
|
16
|
+
//# sourceMappingURL=execute.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../src/execute.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAa,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAGrE,OAAO,KAAK,EAGV,SAAS,EAET,WAAW,EACX,QAAQ,EAGT,MAAM,YAAY,CAAC;AA+EpB,wBAAuB,OAAO,CAC5B,IAAI,EAAE,QAAQ,EACd,CAAC,EAAE,QAAQ,EACX,CAAC,EAAE,QAAQ,EACX,IAAI,EAAE,WAAW,GAChB,aAAa,CAAC,SAAS,CAAC,CA4C1B"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @statewalker/webrun-files-sync
|
|
3
|
+
*
|
|
4
|
+
* rclone-like file-synchronisation engine (copy / sync / bisync / move / check)
|
|
5
|
+
* over two `@statewalker/webrun-files` endpoints. Plan-then-execute: {@link plan}
|
|
6
|
+
* is pure and produces a serializable {@link SyncPlan}; {@link execute} applies
|
|
7
|
+
* it, verifying and resuming. Bidirectional sync is a three-way merge over a
|
|
8
|
+
* sync-owned anchor, delegated to `@statewalker/webrun-merge`.
|
|
9
|
+
*
|
|
10
|
+
* Domain-neutral: this package imports ONLY webrun-files + webrun-merge and knows
|
|
11
|
+
* nothing about git or versioning (Axis A / Axis B ban).
|
|
12
|
+
*/
|
|
13
|
+
export { buildAnchor } from "./anchor.js";
|
|
14
|
+
export { isChanged, snapshot } from "./change-detection.js";
|
|
15
|
+
export { execute } from "./execute.js";
|
|
16
|
+
export { plan } from "./plan.js";
|
|
17
|
+
export { createStreamingTransfer } from "./transfer.js";
|
|
18
|
+
export type { AnchorStore, ByteStream, CheckpointStore, FilesApi, PathFilter, Resolution, SyncAction, SyncAnchor, SyncConflict, SyncEvent, SyncOp, SyncOptions, SyncPlan, Transfer, VerificationMode, } from "./types.js";
|
|
19
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AACxD,YAAY,EACV,WAAW,EACX,UAAU,EACV,eAAe,EACf,QAAQ,EACR,UAAU,EACV,UAAU,EACV,UAAU,EACV,UAAU,EACV,YAAY,EACZ,SAAS,EACT,MAAM,EACN,WAAW,EACX,QAAQ,EACR,QAAQ,EACR,gBAAgB,GACjB,MAAM,YAAY,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
import { merge } from "@statewalker/webrun-merge";
|
|
2
|
+
//#region src/anchor.ts
|
|
3
|
+
/** Build a fresh anchor manifest by hashing every file in an endpoint. */
|
|
4
|
+
async function buildAnchor(files, hashContent) {
|
|
5
|
+
const entries = {};
|
|
6
|
+
for await (const info of files.list("/", { recursive: true })) {
|
|
7
|
+
if (info.kind !== "file") continue;
|
|
8
|
+
entries[info.path] = {
|
|
9
|
+
contentId: await hashContent(files.read(info.path)),
|
|
10
|
+
size: info.size ?? 0,
|
|
11
|
+
mtime: info.lastModified ?? 0
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
return { entries };
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* A read-only base tree reconstructed from a {@link SyncAnchor}.
|
|
18
|
+
*
|
|
19
|
+
* Contract decision 1: the anchor stores NO base bytes. Base content is resolved
|
|
20
|
+
* ON DEMAND by content-id from whichever live endpoint still holds it (both `a`
|
|
21
|
+
* and `b` are searched, hashing lazily). When no endpoint holds the id — i.e.
|
|
22
|
+
* both sides changed the file — `read` yields nothing, and the both-modified
|
|
23
|
+
* path resolves to a conflict in webrun-merge rather than a bogus merge.
|
|
24
|
+
*/
|
|
25
|
+
function anchorBaseView(anchor, a, b, hashContent) {
|
|
26
|
+
let index;
|
|
27
|
+
const wanted = new Set(Object.values(anchor.entries).map((e) => e.contentId));
|
|
28
|
+
async function buildIndex() {
|
|
29
|
+
const map = /* @__PURE__ */ new Map();
|
|
30
|
+
for (const api of [a, b]) for await (const info of api.list("/", { recursive: true })) {
|
|
31
|
+
if (info.kind !== "file") continue;
|
|
32
|
+
const id = await hashContent(api.read(info.path));
|
|
33
|
+
if (wanted.has(id) && !map.has(id)) map.set(id, {
|
|
34
|
+
api,
|
|
35
|
+
path: info.path
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
return map;
|
|
39
|
+
}
|
|
40
|
+
async function locate(path) {
|
|
41
|
+
const entry = anchor.entries[path];
|
|
42
|
+
if (!entry) return void 0;
|
|
43
|
+
if (!index) index = await buildIndex();
|
|
44
|
+
return index.get(entry.contentId);
|
|
45
|
+
}
|
|
46
|
+
const readOnly = (op) => async () => {
|
|
47
|
+
throw new Error(`anchor base view is read-only (${op})`);
|
|
48
|
+
};
|
|
49
|
+
return {
|
|
50
|
+
async *read(path, _options) {
|
|
51
|
+
const loc = await locate(path);
|
|
52
|
+
if (!loc) return;
|
|
53
|
+
yield* loc.api.read(loc.path);
|
|
54
|
+
},
|
|
55
|
+
async *list(path, options) {
|
|
56
|
+
const prefix = path === "/" ? "/" : `${path}/`;
|
|
57
|
+
for (const [p, e] of Object.entries(anchor.entries)) {
|
|
58
|
+
if (p !== path && !p.startsWith(prefix)) continue;
|
|
59
|
+
if (!options?.recursive && p.slice(prefix.length).includes("/")) continue;
|
|
60
|
+
yield {
|
|
61
|
+
name: p.slice(p.lastIndexOf("/") + 1),
|
|
62
|
+
path: p,
|
|
63
|
+
kind: "file",
|
|
64
|
+
size: e.size,
|
|
65
|
+
lastModified: e.mtime
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
},
|
|
69
|
+
async stats(path) {
|
|
70
|
+
const e = anchor.entries[path];
|
|
71
|
+
return e ? {
|
|
72
|
+
kind: "file",
|
|
73
|
+
size: e.size,
|
|
74
|
+
lastModified: e.mtime
|
|
75
|
+
} : void 0;
|
|
76
|
+
},
|
|
77
|
+
async exists(path) {
|
|
78
|
+
return path in anchor.entries;
|
|
79
|
+
},
|
|
80
|
+
write: readOnly("write"),
|
|
81
|
+
mkdir: readOnly("mkdir"),
|
|
82
|
+
remove: readOnly("remove"),
|
|
83
|
+
move: readOnly("move"),
|
|
84
|
+
copy: readOnly("copy")
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
//#endregion
|
|
88
|
+
//#region src/change-detection.ts
|
|
89
|
+
/** List a whole tree into a path-keyed snapshot, applying an optional filter. */
|
|
90
|
+
async function snapshot(files, filter) {
|
|
91
|
+
const map = /* @__PURE__ */ new Map();
|
|
92
|
+
for await (const info of files.list("/", { recursive: true })) {
|
|
93
|
+
if (filter && info.kind === "file" && !filter(info.path)) continue;
|
|
94
|
+
map.set(info.path, {
|
|
95
|
+
path: info.path,
|
|
96
|
+
kind: info.kind,
|
|
97
|
+
size: info.kind === "file" ? info.size : void 0,
|
|
98
|
+
mtime: info.kind === "file" ? info.lastModified : void 0
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
return map;
|
|
102
|
+
}
|
|
103
|
+
const HEAD_TAIL = 64;
|
|
104
|
+
/** Read a bounded byte range as a single buffer (for fingerprint sampling). */
|
|
105
|
+
async function readRange(files, path, start, length) {
|
|
106
|
+
const parts = [];
|
|
107
|
+
for await (const chunk of files.read(path, {
|
|
108
|
+
start,
|
|
109
|
+
length
|
|
110
|
+
})) parts.push(chunk);
|
|
111
|
+
const total = parts.reduce((n, p) => n + p.length, 0);
|
|
112
|
+
const out = new Uint8Array(total);
|
|
113
|
+
let off = 0;
|
|
114
|
+
for (const p of parts) {
|
|
115
|
+
out.set(p, off);
|
|
116
|
+
off += p.length;
|
|
117
|
+
}
|
|
118
|
+
return out;
|
|
119
|
+
}
|
|
120
|
+
function bytesEqual(a, b) {
|
|
121
|
+
if (a.length !== b.length) return false;
|
|
122
|
+
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
|
|
123
|
+
return true;
|
|
124
|
+
}
|
|
125
|
+
/** head+tail sample of a file, used by the optional quick-fingerprint rung. */
|
|
126
|
+
async function fingerprint(files, path, size) {
|
|
127
|
+
if (size <= 128) return readRange(files, path, 0, size);
|
|
128
|
+
const head = await readRange(files, path, 0, HEAD_TAIL);
|
|
129
|
+
const tail = await readRange(files, path, size - HEAD_TAIL, HEAD_TAIL);
|
|
130
|
+
const out = new Uint8Array(head.length + tail.length);
|
|
131
|
+
out.set(head, 0);
|
|
132
|
+
out.set(tail, head.length);
|
|
133
|
+
return out;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Decide whether `from`'s file at `path` differs from `to`'s file, climbing the
|
|
137
|
+
* ladder only as far as needed. `fromEntry` must exist; `toEntry` may be absent.
|
|
138
|
+
*/
|
|
139
|
+
async function isChanged(from, to, path, fromEntry, toEntry, opts) {
|
|
140
|
+
if (!toEntry) return true;
|
|
141
|
+
if (fromEntry.kind !== toEntry.kind) return true;
|
|
142
|
+
if (fromEntry.kind === "directory") return false;
|
|
143
|
+
if (fromEntry.size !== toEntry.size) return true;
|
|
144
|
+
if (fromEntry.mtime !== void 0 && toEntry.mtime !== void 0 && fromEntry.mtime === toEntry.mtime) return false;
|
|
145
|
+
const size = fromEntry.size ?? 0;
|
|
146
|
+
if (opts.quickFingerprint) {
|
|
147
|
+
const [fa, fb] = await Promise.all([fingerprint(from, path, size), fingerprint(to, path, size)]);
|
|
148
|
+
if (!bytesEqual(fa, fb)) return true;
|
|
149
|
+
}
|
|
150
|
+
const [ha, hb] = await Promise.all([opts.hashContent(from.read(path)), opts.hashContent(to.read(path))]);
|
|
151
|
+
return ha !== hb;
|
|
152
|
+
}
|
|
153
|
+
//#endregion
|
|
154
|
+
//#region src/transfer.ts
|
|
155
|
+
function createStreamingTransfer() {
|
|
156
|
+
return { async run(action, from, to) {
|
|
157
|
+
if (action.kind !== "copy" && action.kind !== "update") throw new Error(`streaming transfer only handles copy/update, got ${action.kind}`);
|
|
158
|
+
const path = action.path;
|
|
159
|
+
if (from === to) {
|
|
160
|
+
await from.copy(path, path);
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
await to.write(path, from.read(path));
|
|
164
|
+
} };
|
|
165
|
+
}
|
|
166
|
+
//#endregion
|
|
167
|
+
//#region src/execute.ts
|
|
168
|
+
async function verifyCopy(mode, source, dest, path, hashContent) {
|
|
169
|
+
if (mode === "content-hash" || mode === "backend-hash" || mode === "read-after-write") {
|
|
170
|
+
if (!await dest.stats(path)) return false;
|
|
171
|
+
const [hs, hd] = await Promise.all([hashContent(source.read(path)), hashContent(dest.read(path))]);
|
|
172
|
+
return hs === hd;
|
|
173
|
+
}
|
|
174
|
+
const [s, d] = await Promise.all([source.stats(path), dest.stats(path)]);
|
|
175
|
+
return !!s && !!d && sizeOf(s) === sizeOf(d);
|
|
176
|
+
}
|
|
177
|
+
/** Byte length of a file; `undefined` for a directory (two directories compare equal). */
|
|
178
|
+
function sizeOf(stats) {
|
|
179
|
+
return stats.kind === "file" ? stats.size : void 0;
|
|
180
|
+
}
|
|
181
|
+
async function applyAction(action, a, b, op, transfer, verify, opts) {
|
|
182
|
+
const pick = (s) => s === "a" ? a : b;
|
|
183
|
+
switch (action.kind) {
|
|
184
|
+
case "mkdir":
|
|
185
|
+
await b.mkdir(action.path);
|
|
186
|
+
return { ok: true };
|
|
187
|
+
case "copy":
|
|
188
|
+
case "update": {
|
|
189
|
+
const from = pick(action.from);
|
|
190
|
+
const to = action.from === "a" ? b : a;
|
|
191
|
+
await transfer.run(action, from, to);
|
|
192
|
+
return await verifyCopy(verify, from, to, action.path, opts.hashContent) ? { ok: true } : {
|
|
193
|
+
ok: false,
|
|
194
|
+
reason: "destination verification failed"
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
case "move":
|
|
198
|
+
await pick(action.side).move(action.from, action.to);
|
|
199
|
+
return { ok: true };
|
|
200
|
+
case "delete":
|
|
201
|
+
if (op === "move" && action.side === "a") {
|
|
202
|
+
if (!await verifyCopy(verify, a, b, action.path, opts.hashContent)) return {
|
|
203
|
+
ok: false,
|
|
204
|
+
skipped: true,
|
|
205
|
+
reason: "destination not verified; source retained"
|
|
206
|
+
};
|
|
207
|
+
await a.remove(action.path);
|
|
208
|
+
return { ok: true };
|
|
209
|
+
}
|
|
210
|
+
await pick(action.side).remove(action.path);
|
|
211
|
+
return { ok: true };
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
async function* execute(plan, a, b, opts) {
|
|
215
|
+
const transfer = opts.transfer ?? createStreamingTransfer();
|
|
216
|
+
const verify = opts.verify ?? "size";
|
|
217
|
+
for (const message of plan.warnings ?? []) yield {
|
|
218
|
+
type: "warning",
|
|
219
|
+
message
|
|
220
|
+
};
|
|
221
|
+
for (const conflict of plan.conflicts) yield {
|
|
222
|
+
type: "conflict",
|
|
223
|
+
conflict
|
|
224
|
+
};
|
|
225
|
+
if (plan.op === "check") return;
|
|
226
|
+
const done = await opts.checkpoint?.read() ?? /* @__PURE__ */ new Set();
|
|
227
|
+
let anyFailure = false;
|
|
228
|
+
for (let i = 0; i < plan.actions.length; i++) {
|
|
229
|
+
if (done.has(i)) continue;
|
|
230
|
+
const action = plan.actions[i];
|
|
231
|
+
yield {
|
|
232
|
+
type: "start",
|
|
233
|
+
index: i,
|
|
234
|
+
action
|
|
235
|
+
};
|
|
236
|
+
try {
|
|
237
|
+
const outcome = await applyAction(action, a, b, plan.op, transfer, verify, opts);
|
|
238
|
+
if (!outcome.ok) {
|
|
239
|
+
anyFailure = true;
|
|
240
|
+
const reason = outcome.reason ?? "failed";
|
|
241
|
+
yield outcome.skipped ? {
|
|
242
|
+
type: "skipped",
|
|
243
|
+
index: i,
|
|
244
|
+
action,
|
|
245
|
+
reason
|
|
246
|
+
} : {
|
|
247
|
+
type: "failed",
|
|
248
|
+
index: i,
|
|
249
|
+
action,
|
|
250
|
+
reason
|
|
251
|
+
};
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
await opts.checkpoint?.add(i);
|
|
255
|
+
yield {
|
|
256
|
+
type: "done",
|
|
257
|
+
index: i,
|
|
258
|
+
action
|
|
259
|
+
};
|
|
260
|
+
} catch (error) {
|
|
261
|
+
anyFailure = true;
|
|
262
|
+
yield {
|
|
263
|
+
type: "failed",
|
|
264
|
+
index: i,
|
|
265
|
+
action,
|
|
266
|
+
reason: String(error?.message ?? error)
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
if (plan.op === "bisync" && !anyFailure && plan.conflicts.length === 0 && opts.anchorStore) await opts.anchorStore.write(opts.pairKey ?? "default", await buildAnchor(a, opts.hashContent));
|
|
271
|
+
}
|
|
272
|
+
//#endregion
|
|
273
|
+
//#region src/bisync.ts
|
|
274
|
+
const alwaysConflictMerger = { merge: () => ({ conflict: true }) };
|
|
275
|
+
async function planBisync(a, b, opts) {
|
|
276
|
+
const pairKey = opts.pairKey ?? "default";
|
|
277
|
+
const anchor = await opts.anchorStore?.read(pairKey);
|
|
278
|
+
if (!anchor) return planUnion(a, b, opts);
|
|
279
|
+
const base = anchorBaseView(anchor, a, b, opts.hashContent);
|
|
280
|
+
const result = await merge(base, a, b, {
|
|
281
|
+
hashContent: opts.hashContent,
|
|
282
|
+
contentMerger: alwaysConflictMerger,
|
|
283
|
+
resolve: opts.resolve
|
|
284
|
+
});
|
|
285
|
+
const [snapA, snapB] = await Promise.all([snapshot(a, opts.filter), snapshot(b, opts.filter)]);
|
|
286
|
+
const actions = [];
|
|
287
|
+
for (const op of result.operations) switch (op.op) {
|
|
288
|
+
case "add":
|
|
289
|
+
case "modify": {
|
|
290
|
+
if (!op.source) throw new Error(`bisync cannot apply an inline-content ${op.op} for ${op.path}`);
|
|
291
|
+
const from = op.source.side === "left" ? "a" : "b";
|
|
292
|
+
actions.push({
|
|
293
|
+
kind: op.op === "add" ? "copy" : "update",
|
|
294
|
+
path: op.path,
|
|
295
|
+
from
|
|
296
|
+
});
|
|
297
|
+
break;
|
|
298
|
+
}
|
|
299
|
+
case "delete":
|
|
300
|
+
if (snapA.has(op.path)) actions.push({
|
|
301
|
+
kind: "delete",
|
|
302
|
+
path: op.path,
|
|
303
|
+
side: "a"
|
|
304
|
+
});
|
|
305
|
+
if (snapB.has(op.path)) actions.push({
|
|
306
|
+
kind: "delete",
|
|
307
|
+
path: op.path,
|
|
308
|
+
side: "b"
|
|
309
|
+
});
|
|
310
|
+
break;
|
|
311
|
+
case "rename": {
|
|
312
|
+
const from = op.from;
|
|
313
|
+
if (snapA.has(from)) actions.push({
|
|
314
|
+
kind: "move",
|
|
315
|
+
from,
|
|
316
|
+
to: op.path,
|
|
317
|
+
side: "a"
|
|
318
|
+
});
|
|
319
|
+
if (snapB.has(from)) actions.push({
|
|
320
|
+
kind: "move",
|
|
321
|
+
from,
|
|
322
|
+
to: op.path,
|
|
323
|
+
side: "b"
|
|
324
|
+
});
|
|
325
|
+
break;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
return {
|
|
329
|
+
op: "bisync",
|
|
330
|
+
actions,
|
|
331
|
+
conflicts: result.conflicts
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
/** Conservative union used for the first bisync (no anchor yet). */
|
|
335
|
+
async function planUnion(a, b, opts) {
|
|
336
|
+
const [snapA, snapB] = await Promise.all([snapshot(a, opts.filter), snapshot(b, opts.filter)]);
|
|
337
|
+
const actions = [];
|
|
338
|
+
const conflicts = [];
|
|
339
|
+
for (const [path, entry] of snapA) {
|
|
340
|
+
if (entry.kind !== "file") continue;
|
|
341
|
+
const other = snapB.get(path);
|
|
342
|
+
if (!other) actions.push({
|
|
343
|
+
kind: "copy",
|
|
344
|
+
path,
|
|
345
|
+
from: "a"
|
|
346
|
+
});
|
|
347
|
+
else if (await isChanged(a, b, path, entry, other, opts)) conflicts.push({
|
|
348
|
+
kind: "content",
|
|
349
|
+
path,
|
|
350
|
+
left: {
|
|
351
|
+
path,
|
|
352
|
+
kind: "file"
|
|
353
|
+
},
|
|
354
|
+
right: {
|
|
355
|
+
path,
|
|
356
|
+
kind: "file"
|
|
357
|
+
}
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
for (const [path, entry] of snapB) {
|
|
361
|
+
if (entry.kind !== "file") continue;
|
|
362
|
+
if (!snapA.has(path)) actions.push({
|
|
363
|
+
kind: "copy",
|
|
364
|
+
path,
|
|
365
|
+
from: "b"
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
return {
|
|
369
|
+
op: "bisync",
|
|
370
|
+
actions,
|
|
371
|
+
conflicts,
|
|
372
|
+
warnings: ["no sync anchor; performing conservative union (additions only, no deletions)"]
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
//#endregion
|
|
376
|
+
//#region src/plan.ts
|
|
377
|
+
async function plan(a, b, op, opts) {
|
|
378
|
+
if (op === "bisync") return planBisync(a, b, opts);
|
|
379
|
+
const [snapA, snapB] = await Promise.all([snapshot(a, opts.filter), snapshot(b, opts.filter)]);
|
|
380
|
+
const actions = [];
|
|
381
|
+
for (const [path, entry] of snapA) {
|
|
382
|
+
const other = snapB.get(path);
|
|
383
|
+
if (entry.kind === "directory") {
|
|
384
|
+
if (!other) actions.push({
|
|
385
|
+
kind: "mkdir",
|
|
386
|
+
path
|
|
387
|
+
});
|
|
388
|
+
continue;
|
|
389
|
+
}
|
|
390
|
+
if (!other) actions.push({
|
|
391
|
+
kind: "copy",
|
|
392
|
+
path,
|
|
393
|
+
from: "a"
|
|
394
|
+
});
|
|
395
|
+
else if (await isChanged(a, b, path, entry, other, opts)) actions.push({
|
|
396
|
+
kind: "update",
|
|
397
|
+
path,
|
|
398
|
+
from: "a"
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
if (op === "move") {
|
|
402
|
+
for (const [path, entry] of snapA) if (entry.kind === "file") actions.push({
|
|
403
|
+
kind: "delete",
|
|
404
|
+
path,
|
|
405
|
+
side: "a"
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
if (op === "sync") {
|
|
409
|
+
for (const [path, entry] of snapB) if (entry.kind === "file" && !snapA.has(path)) actions.push({
|
|
410
|
+
kind: "delete",
|
|
411
|
+
path,
|
|
412
|
+
side: "b"
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
return {
|
|
416
|
+
op,
|
|
417
|
+
actions,
|
|
418
|
+
conflicts: []
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
//#endregion
|
|
422
|
+
export { buildAnchor, createStreamingTransfer, execute, isChanged, plan, snapshot };
|
package/dist/plan.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* {@link plan} — pure planning for all sync operations. It reads both endpoints
|
|
3
|
+
* (listing, and hashing only when the ladder demands it) but MUTATES NEITHER.
|
|
4
|
+
*
|
|
5
|
+
* copy — new/changed A files → B; extraneous B files KEPT.
|
|
6
|
+
* sync — copy, plus extraneous B files DELETED.
|
|
7
|
+
* move — copy each A file, then (at execute, after the destination is
|
|
8
|
+
* verified) delete the source; source is never lost.
|
|
9
|
+
* check — the copy diff, transferring nothing (execute is a no-op).
|
|
10
|
+
* bisync — three-way merge over the anchor (see bisync.ts).
|
|
11
|
+
*/
|
|
12
|
+
import type { FilesApi } from "@statewalker/webrun-files";
|
|
13
|
+
import type { SyncOp, SyncOptions, SyncPlan } from "./types.js";
|
|
14
|
+
export declare function plan(a: FilesApi, b: FilesApi, op: SyncOp, opts: SyncOptions): Promise<SyncPlan>;
|
|
15
|
+
//# sourceMappingURL=plan.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plan.d.ts","sourceRoot":"","sources":["../src/plan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAG1D,OAAO,KAAK,EAAc,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE5E,wBAAsB,IAAI,CACxB,CAAC,EAAE,QAAQ,EACX,CAAC,EAAE,QAAQ,EACX,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,WAAW,GAChB,OAAO,CAAC,QAAQ,CAAC,CAqCnB"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The default {@link Transfer}: a whole-file copy.
|
|
3
|
+
*
|
|
4
|
+
* When source and destination are the SAME backend instance, it uses the
|
|
5
|
+
* backend's native server-side `copy` ("capabilities allow"). Across two
|
|
6
|
+
* distinct backends — the normal sync case, and all MemFilesApi pairs — it falls
|
|
7
|
+
* back to streaming the bytes through `to.write(from.read(...))`.
|
|
8
|
+
*
|
|
9
|
+
* A later phase can plug a chunk-dedup Transfer into this same seam.
|
|
10
|
+
*/
|
|
11
|
+
import type { Transfer } from "./types.js";
|
|
12
|
+
export declare function createStreamingTransfer(): Transfer;
|
|
13
|
+
//# sourceMappingURL=transfer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transfer.d.ts","sourceRoot":"","sources":["../src/transfer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAc,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEvD,wBAAgB,uBAAuB,IAAI,QAAQ,CAelD"}
|