@statewalker/webrun-files-sync 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +59 -0
- package/dist/anchor.d.ts +19 -0
- package/dist/anchor.d.ts.map +1 -0
- package/dist/bisync.d.ts +24 -0
- package/dist/bisync.d.ts.map +1 -0
- package/dist/change-detection.d.ts +33 -0
- package/dist/change-detection.d.ts.map +1 -0
- package/dist/execute.d.ts +16 -0
- package/dist/execute.d.ts.map +1 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +422 -0
- package/dist/plan.d.ts +15 -0
- package/dist/plan.d.ts.map +1 -0
- package/dist/transfer.d.ts +13 -0
- package/dist/transfer.d.ts.map +1 -0
- package/dist/types.d.ts +140 -0
- package/dist/types.d.ts.map +1 -0
- package/package.json +51 -0
- package/src/anchor.ts +107 -0
- package/src/bisync.ts +107 -0
- package/src/change-detection.ts +131 -0
- package/src/execute.ts +154 -0
- package/src/index.ts +35 -0
- package/src/plan.ts +60 -0
- package/src/transfer.ts +30 -0
- package/src/types.ts +123 -0
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public contract for the rclone-like file-synchronisation engine.
|
|
3
|
+
*
|
|
4
|
+
* The engine plans-then-executes over two {@link FilesApi} endpoints. {@link plan}
|
|
5
|
+
* is pure (mutates neither endpoint) and returns a serializable {@link SyncPlan};
|
|
6
|
+
* {@link execute} applies it, emitting {@link SyncEvent}s and resuming from an
|
|
7
|
+
* optional checkpoint after interruption.
|
|
8
|
+
*/
|
|
9
|
+
import type { FilesApi } from "@statewalker/webrun-files";
|
|
10
|
+
import type { Resolution, Conflict as SyncConflict } from "@statewalker/webrun-merge";
|
|
11
|
+
export type { FilesApi, Resolution, SyncConflict };
|
|
12
|
+
/** A stream of bytes, as produced by {@link FilesApi.read}. */
|
|
13
|
+
export type ByteStream = AsyncIterable<Uint8Array>;
|
|
14
|
+
/**
|
|
15
|
+
* Path predicate: return `true` to INCLUDE a path in the sync, `false` to skip.
|
|
16
|
+
* `webrun-files` does not export a `PathFilter`, so it is defined here.
|
|
17
|
+
*/
|
|
18
|
+
export type PathFilter = (path: string) => boolean;
|
|
19
|
+
export type SyncOp = "copy" | "sync" | "bisync" | "move" | "check";
|
|
20
|
+
/**
|
|
21
|
+
* How a transferred file is verified at the destination.
|
|
22
|
+
* `size` compares byte counts (cheap default); `content-hash`/`backend-hash`/
|
|
23
|
+
* `read-after-write` re-hash the destination bytes; `mtime` falls back to size
|
|
24
|
+
* (cross-endpoint mtime is not comparable).
|
|
25
|
+
*/
|
|
26
|
+
export type VerificationMode = "size" | "mtime" | "backend-hash" | "content-hash" | "read-after-write";
|
|
27
|
+
/**
|
|
28
|
+
* A single typed step of a {@link SyncPlan}. Serializable by construction.
|
|
29
|
+
* `copy`/`update` read `path` from the `from` endpoint and write it to the other;
|
|
30
|
+
* `move` renames within one `side`; `delete` removes `path` from one `side`;
|
|
31
|
+
* `mkdir` creates an (empty) directory on the destination of a one-way op.
|
|
32
|
+
*
|
|
33
|
+
* (The grilled note also listed a `metadata` action; it is intentionally omitted
|
|
34
|
+
* — the same note puts "arbitrary metadata beyond FilesApi size/mtime" out of
|
|
35
|
+
* scope, and MemFilesApi exposes no metadata setter, so it would be dead code.)
|
|
36
|
+
*/
|
|
37
|
+
export type SyncAction = {
|
|
38
|
+
kind: "mkdir";
|
|
39
|
+
path: string;
|
|
40
|
+
} | {
|
|
41
|
+
kind: "copy";
|
|
42
|
+
path: string;
|
|
43
|
+
from: "a" | "b";
|
|
44
|
+
} | {
|
|
45
|
+
kind: "update";
|
|
46
|
+
path: string;
|
|
47
|
+
from: "a" | "b";
|
|
48
|
+
} | {
|
|
49
|
+
kind: "move";
|
|
50
|
+
from: string;
|
|
51
|
+
to: string;
|
|
52
|
+
side: "a" | "b";
|
|
53
|
+
} | {
|
|
54
|
+
kind: "delete";
|
|
55
|
+
path: string;
|
|
56
|
+
side: "a" | "b";
|
|
57
|
+
};
|
|
58
|
+
/** A serializable, dry-runnable, auditable plan. */
|
|
59
|
+
export interface SyncPlan {
|
|
60
|
+
op: SyncOp;
|
|
61
|
+
actions: SyncAction[];
|
|
62
|
+
conflicts: SyncConflict[];
|
|
63
|
+
/** Non-fatal notices raised while planning (e.g. first bisync with no anchor). */
|
|
64
|
+
warnings?: string[];
|
|
65
|
+
}
|
|
66
|
+
/** Events streamed by {@link execute}. */
|
|
67
|
+
export type SyncEvent = {
|
|
68
|
+
type: "warning";
|
|
69
|
+
message: string;
|
|
70
|
+
} | {
|
|
71
|
+
type: "conflict";
|
|
72
|
+
conflict: SyncConflict;
|
|
73
|
+
} | {
|
|
74
|
+
type: "start";
|
|
75
|
+
index: number;
|
|
76
|
+
action: SyncAction;
|
|
77
|
+
} | {
|
|
78
|
+
type: "done";
|
|
79
|
+
index: number;
|
|
80
|
+
action: SyncAction;
|
|
81
|
+
} | {
|
|
82
|
+
type: "skipped";
|
|
83
|
+
index: number;
|
|
84
|
+
action: SyncAction;
|
|
85
|
+
reason: string;
|
|
86
|
+
} | {
|
|
87
|
+
type: "failed";
|
|
88
|
+
index: number;
|
|
89
|
+
action: SyncAction;
|
|
90
|
+
reason: string;
|
|
91
|
+
};
|
|
92
|
+
/**
|
|
93
|
+
* Executes a single copy/update. Injected so a later phase can plug a
|
|
94
|
+
* chunk-dedup transfer into the SAME seam; the default streams whole files.
|
|
95
|
+
*/
|
|
96
|
+
export interface Transfer {
|
|
97
|
+
run(action: SyncAction, from: FilesApi, to: FilesApi): Promise<void>;
|
|
98
|
+
}
|
|
99
|
+
/** Content-id + metadata manifest of the last successful sync (bisync base). */
|
|
100
|
+
export interface SyncAnchor {
|
|
101
|
+
entries: Record<string, {
|
|
102
|
+
contentId: string;
|
|
103
|
+
size: number;
|
|
104
|
+
mtime: number;
|
|
105
|
+
}>;
|
|
106
|
+
}
|
|
107
|
+
/** Persistence for the {@link SyncAnchor} of an endpoint pair (bisync). */
|
|
108
|
+
export interface AnchorStore {
|
|
109
|
+
read(pairKey: string): Promise<SyncAnchor | undefined>;
|
|
110
|
+
write(pairKey: string, anchor: SyncAnchor): Promise<void>;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Progress checkpoint enabling resume: records the indices of completed actions.
|
|
114
|
+
* `execute` skips completed indices and records each new one before yielding its
|
|
115
|
+
* `done` event, so an interrupted run resumes with no duplicate work.
|
|
116
|
+
*/
|
|
117
|
+
export interface CheckpointStore {
|
|
118
|
+
read(): Promise<Set<number>>;
|
|
119
|
+
add(index: number): Promise<void>;
|
|
120
|
+
}
|
|
121
|
+
export interface SyncOptions {
|
|
122
|
+
/** Content identity — the ladder terminal and the id shared with webrun-merge. */
|
|
123
|
+
hashContent: (input: ByteStream) => Promise<string>;
|
|
124
|
+
filter?: PathFilter;
|
|
125
|
+
/** Copy/update executor; defaults to whole-file streaming copy. */
|
|
126
|
+
transfer?: Transfer;
|
|
127
|
+
/** Destination verification mode; defaults to `size`. */
|
|
128
|
+
verify?: VerificationMode;
|
|
129
|
+
/** SyncAnchor persistence (bisync only). */
|
|
130
|
+
anchorStore?: AnchorStore;
|
|
131
|
+
/** Identifies the endpoint pair for the anchor store (bisync); defaults to "default". */
|
|
132
|
+
pairKey?: string;
|
|
133
|
+
/** Injected conflict-resolution policy (bisync). */
|
|
134
|
+
resolve?: (c: SyncConflict) => Resolution | undefined;
|
|
135
|
+
/** Enable the head/tail sample rung of the change-detection ladder. */
|
|
136
|
+
quickFingerprint?: boolean;
|
|
137
|
+
/** Resume support for {@link execute}. */
|
|
138
|
+
checkpoint?: CheckpointStore;
|
|
139
|
+
}
|
|
140
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAG1D,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAEtF,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC;AAEnD,+DAA+D;AAC/D,MAAM,MAAM,UAAU,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;AAEnD;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;AAEnE;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GACxB,MAAM,GACN,OAAO,GACP,cAAc,GACd,cAAc,GACd,kBAAkB,CAAC;AAEvB;;;;;;;;;GASG;AACH,MAAM,MAAM,UAAU,GAClB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC/B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,GAAG,GAAG,GAAG,CAAA;CAAE,GAC/C;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,GAAG,GAAG,GAAG,CAAA;CAAE,GACjD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,GAAG,GAAG,GAAG,CAAA;CAAE,GAC3D;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,GAAG,GAAG,GAAG,CAAA;CAAE,CAAC;AAEtD,oDAAoD;AACpD,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,kFAAkF;IAClF,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,0CAA0C;AAC1C,MAAM,MAAM,SAAS,GACjB;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,YAAY,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,UAAU,CAAA;CAAE,GACpD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,UAAU,CAAA;CAAE,GACnD;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACtE;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1E;;;GAGG;AACH,MAAM,WAAW,QAAQ;IACvB,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtE;AAED,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7E;AAED,2EAA2E;AAC3E,MAAM,WAAW,WAAW;IAC1B,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IACvD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3D;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IAC7B,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,WAAW;IAC1B,kFAAkF;IAClF,WAAW,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,mEAAmE;IACnE,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,yDAAyD;IACzD,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,4CAA4C;IAC5C,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,yFAAyF;IACzF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,YAAY,KAAK,UAAU,GAAG,SAAS,CAAC;IACtD,uEAAuE;IACvE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,0CAA0C;IAC1C,UAAU,CAAC,EAAE,eAAe,CAAC;CAC9B"}
|
package/package.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@statewalker/webrun-files-sync",
|
|
3
|
+
"version": "0.2.2",
|
|
4
|
+
"description": "rclone-like file-synchronisation engine (copy/sync/bisync/move/check) over the webrun-files FilesApi",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "https://github.com/statewalker/webrun-vcs.git",
|
|
9
|
+
"directory": "packages/webrun-files-sync"
|
|
10
|
+
},
|
|
11
|
+
"private": false,
|
|
12
|
+
"publishConfig": {
|
|
13
|
+
"access": "public"
|
|
14
|
+
},
|
|
15
|
+
"type": "module",
|
|
16
|
+
"module": "./dist/index.js",
|
|
17
|
+
"types": "./dist/index.d.ts",
|
|
18
|
+
"exports": {
|
|
19
|
+
".": {
|
|
20
|
+
"source": "./src/index.ts",
|
|
21
|
+
"types": "./dist/index.d.ts",
|
|
22
|
+
"import": "./dist/index.js"
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
"files": [
|
|
26
|
+
"dist",
|
|
27
|
+
"src"
|
|
28
|
+
],
|
|
29
|
+
"dependencies": {
|
|
30
|
+
"@statewalker/webrun-files": "^0.10.0",
|
|
31
|
+
"@statewalker/webrun-merge": "^0.2.1"
|
|
32
|
+
},
|
|
33
|
+
"devDependencies": {
|
|
34
|
+
"@statewalker/webrun-files-mem": "^0.10.0",
|
|
35
|
+
"rolldown": "^1.2.12",
|
|
36
|
+
"vitest": "^5.0.3",
|
|
37
|
+
"typescript": "^7.0.2",
|
|
38
|
+
"@types/node": "^26.6.4"
|
|
39
|
+
},
|
|
40
|
+
"scripts": {
|
|
41
|
+
"build": "rm -rf dist && rolldown -c && tsc --emitDeclarationOnly --declaration",
|
|
42
|
+
"test": "vitest run",
|
|
43
|
+
"lint": "biome check --write .",
|
|
44
|
+
"lint:check": "biome check .",
|
|
45
|
+
"lint:fix": "biome check --write --unsafe .",
|
|
46
|
+
"format": "biome format --write .",
|
|
47
|
+
"format:check": "biome format .",
|
|
48
|
+
"format:fix": "biome check --write --unsafe .",
|
|
49
|
+
"typecheck": "tsc --noEmit"
|
|
50
|
+
}
|
|
51
|
+
}
|
package/src/anchor.ts
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* {@link SyncAnchor} helpers — building a manifest from an endpoint and
|
|
3
|
+
* reconstructing a read-only "base" {@link FilesApi} view from a manifest.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type {
|
|
7
|
+
FileInfo,
|
|
8
|
+
FileStats,
|
|
9
|
+
FilesApi,
|
|
10
|
+
ListOptions,
|
|
11
|
+
ReadOptions,
|
|
12
|
+
} from "@statewalker/webrun-files";
|
|
13
|
+
import type { ByteStream, SyncAnchor } from "./types.js";
|
|
14
|
+
|
|
15
|
+
/** Build a fresh anchor manifest by hashing every file in an endpoint. */
|
|
16
|
+
export async function buildAnchor(
|
|
17
|
+
files: FilesApi,
|
|
18
|
+
hashContent: (b: ByteStream) => Promise<string>,
|
|
19
|
+
): Promise<SyncAnchor> {
|
|
20
|
+
const entries: SyncAnchor["entries"] = {};
|
|
21
|
+
for await (const info of files.list("/", { recursive: true })) {
|
|
22
|
+
if (info.kind !== "file") continue;
|
|
23
|
+
entries[info.path] = {
|
|
24
|
+
contentId: await hashContent(files.read(info.path)),
|
|
25
|
+
size: info.size ?? 0,
|
|
26
|
+
mtime: info.lastModified ?? 0,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
return { entries };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* A read-only base tree reconstructed from a {@link SyncAnchor}.
|
|
34
|
+
*
|
|
35
|
+
* Contract decision 1: the anchor stores NO base bytes. Base content is resolved
|
|
36
|
+
* ON DEMAND by content-id from whichever live endpoint still holds it (both `a`
|
|
37
|
+
* and `b` are searched, hashing lazily). When no endpoint holds the id — i.e.
|
|
38
|
+
* both sides changed the file — `read` yields nothing, and the both-modified
|
|
39
|
+
* path resolves to a conflict in webrun-merge rather than a bogus merge.
|
|
40
|
+
*/
|
|
41
|
+
export function anchorBaseView(
|
|
42
|
+
anchor: SyncAnchor,
|
|
43
|
+
a: FilesApi,
|
|
44
|
+
b: FilesApi,
|
|
45
|
+
hashContent: (input: ByteStream) => Promise<string>,
|
|
46
|
+
): FilesApi {
|
|
47
|
+
// contentId -> {api, path}, built lazily across a then b on first miss.
|
|
48
|
+
let index: Map<string, { api: FilesApi; path: string }> | undefined;
|
|
49
|
+
const wanted = new Set(Object.values(anchor.entries).map((e) => e.contentId));
|
|
50
|
+
|
|
51
|
+
async function buildIndex(): Promise<Map<string, { api: FilesApi; path: string }>> {
|
|
52
|
+
const map = new Map<string, { api: FilesApi; path: string }>();
|
|
53
|
+
for (const api of [a, b]) {
|
|
54
|
+
for await (const info of api.list("/", { recursive: true })) {
|
|
55
|
+
if (info.kind !== "file") continue;
|
|
56
|
+
const id = await hashContent(api.read(info.path));
|
|
57
|
+
if (wanted.has(id) && !map.has(id)) map.set(id, { api, path: info.path });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return map;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function locate(path: string): Promise<{ api: FilesApi; path: string } | undefined> {
|
|
64
|
+
const entry = anchor.entries[path];
|
|
65
|
+
if (!entry) return undefined;
|
|
66
|
+
if (!index) index = await buildIndex();
|
|
67
|
+
return index.get(entry.contentId);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const readOnly = (op: string) => async (): Promise<never> => {
|
|
71
|
+
throw new Error(`anchor base view is read-only (${op})`);
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
return {
|
|
75
|
+
async *read(path: string, _options?: ReadOptions): AsyncIterable<Uint8Array> {
|
|
76
|
+
const loc = await locate(path);
|
|
77
|
+
if (!loc) return; // unresolved base content ⇒ empty (both sides changed it)
|
|
78
|
+
yield* loc.api.read(loc.path);
|
|
79
|
+
},
|
|
80
|
+
async *list(path: string, options?: ListOptions): AsyncIterable<FileInfo> {
|
|
81
|
+
const prefix = path === "/" ? "/" : `${path}/`;
|
|
82
|
+
for (const [p, e] of Object.entries(anchor.entries)) {
|
|
83
|
+
if (p !== path && !p.startsWith(prefix)) continue;
|
|
84
|
+
if (!options?.recursive && p.slice(prefix.length).includes("/")) continue;
|
|
85
|
+
yield {
|
|
86
|
+
name: p.slice(p.lastIndexOf("/") + 1),
|
|
87
|
+
path: p,
|
|
88
|
+
kind: "file",
|
|
89
|
+
size: e.size,
|
|
90
|
+
lastModified: e.mtime,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
async stats(path: string): Promise<FileStats | undefined> {
|
|
95
|
+
const e = anchor.entries[path];
|
|
96
|
+
return e ? { kind: "file", size: e.size, lastModified: e.mtime } : undefined;
|
|
97
|
+
},
|
|
98
|
+
async exists(path: string): Promise<boolean> {
|
|
99
|
+
return path in anchor.entries;
|
|
100
|
+
},
|
|
101
|
+
write: readOnly("write"),
|
|
102
|
+
mkdir: readOnly("mkdir"),
|
|
103
|
+
remove: readOnly("remove"),
|
|
104
|
+
move: readOnly("move"),
|
|
105
|
+
copy: readOnly("copy"),
|
|
106
|
+
} as FilesApi;
|
|
107
|
+
}
|
package/src/bisync.ts
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bidirectional sync — a THREE-WAY merge over a sync-owned {@link SyncAnchor},
|
|
3
|
+
* delegated to `@statewalker/webrun-merge`. This is NEVER two chained one-way syncs.
|
|
4
|
+
*
|
|
5
|
+
* base = the anchor manifest (with base bytes resolved on demand, see anchor.ts)
|
|
6
|
+
* left = endpoint `a` right = endpoint `b`
|
|
7
|
+
*
|
|
8
|
+
* webrun-merge returns operations relative to the merged tree; each is translated
|
|
9
|
+
* into {@link SyncAction}s applied to BOTH endpoints, and its conflicts are
|
|
10
|
+
* surfaced (or folded away by an injected `resolve`, inside webrun-merge).
|
|
11
|
+
*
|
|
12
|
+
* both-modified policy: files-sync installs a content merger that ALWAYS
|
|
13
|
+
* conflicts, so a file changed on both sides is a conflict (optionally resolved
|
|
14
|
+
* by the injected policy) — it is never silently line-merged.
|
|
15
|
+
*
|
|
16
|
+
* No-anchor first bisync: with no anchor there is no base, so a true 3-way merge
|
|
17
|
+
* is impossible. We fall back to a CONSERVATIVE UNION — propagate additions both
|
|
18
|
+
* ways, delete nothing — and emit a warning. Files present on both sides with
|
|
19
|
+
* differing content are surfaced as conflicts, never overwritten.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import type { FilesApi } from "@statewalker/webrun-files";
|
|
23
|
+
import { merge } from "@statewalker/webrun-merge";
|
|
24
|
+
import { anchorBaseView } from "./anchor.js";
|
|
25
|
+
import { isChanged, snapshot } from "./change-detection.js";
|
|
26
|
+
import type { SyncAction, SyncConflict, SyncOptions, SyncPlan } from "./types.js";
|
|
27
|
+
|
|
28
|
+
const alwaysConflictMerger = { merge: () => ({ conflict: true as const }) };
|
|
29
|
+
|
|
30
|
+
export async function planBisync(a: FilesApi, b: FilesApi, opts: SyncOptions): Promise<SyncPlan> {
|
|
31
|
+
const pairKey = opts.pairKey ?? "default";
|
|
32
|
+
const anchor = await opts.anchorStore?.read(pairKey);
|
|
33
|
+
|
|
34
|
+
if (!anchor) return planUnion(a, b, opts);
|
|
35
|
+
|
|
36
|
+
const base = anchorBaseView(anchor, a, b, opts.hashContent);
|
|
37
|
+
const result = await merge(base, a, b, {
|
|
38
|
+
hashContent: opts.hashContent,
|
|
39
|
+
contentMerger: alwaysConflictMerger,
|
|
40
|
+
resolve: opts.resolve,
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
const [snapA, snapB] = await Promise.all([snapshot(a, opts.filter), snapshot(b, opts.filter)]);
|
|
44
|
+
const actions: SyncAction[] = [];
|
|
45
|
+
|
|
46
|
+
for (const op of result.operations) {
|
|
47
|
+
switch (op.op) {
|
|
48
|
+
case "add":
|
|
49
|
+
case "modify": {
|
|
50
|
+
if (!op.source) {
|
|
51
|
+
throw new Error(`bisync cannot apply an inline-content ${op.op} for ${op.path}`);
|
|
52
|
+
}
|
|
53
|
+
const from = op.source.side === "left" ? "a" : "b";
|
|
54
|
+
actions.push({ kind: op.op === "add" ? "copy" : "update", path: op.path, from });
|
|
55
|
+
break;
|
|
56
|
+
}
|
|
57
|
+
case "delete": {
|
|
58
|
+
// Propagate the deletion to whichever side still holds the file.
|
|
59
|
+
if (snapA.has(op.path)) actions.push({ kind: "delete", path: op.path, side: "a" });
|
|
60
|
+
if (snapB.has(op.path)) actions.push({ kind: "delete", path: op.path, side: "b" });
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
case "rename": {
|
|
64
|
+
const from = op.from as string;
|
|
65
|
+
if (snapA.has(from)) actions.push({ kind: "move", from, to: op.path, side: "a" });
|
|
66
|
+
if (snapB.has(from)) actions.push({ kind: "move", from, to: op.path, side: "b" });
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return { op: "bisync", actions, conflicts: result.conflicts };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Conservative union used for the first bisync (no anchor yet). */
|
|
76
|
+
async function planUnion(a: FilesApi, b: FilesApi, opts: SyncOptions): Promise<SyncPlan> {
|
|
77
|
+
const [snapA, snapB] = await Promise.all([snapshot(a, opts.filter), snapshot(b, opts.filter)]);
|
|
78
|
+
const actions: SyncAction[] = [];
|
|
79
|
+
const conflicts: SyncConflict[] = [];
|
|
80
|
+
|
|
81
|
+
for (const [path, entry] of snapA) {
|
|
82
|
+
if (entry.kind !== "file") continue;
|
|
83
|
+
const other = snapB.get(path);
|
|
84
|
+
if (!other) {
|
|
85
|
+
actions.push({ kind: "copy", path, from: "a" });
|
|
86
|
+
} else if (await isChanged(a, b, path, entry, other, opts)) {
|
|
87
|
+
// Both sides hold divergent content and there is no base to arbitrate.
|
|
88
|
+
conflicts.push({
|
|
89
|
+
kind: "content",
|
|
90
|
+
path,
|
|
91
|
+
left: { path, kind: "file" },
|
|
92
|
+
right: { path, kind: "file" },
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
for (const [path, entry] of snapB) {
|
|
97
|
+
if (entry.kind !== "file") continue;
|
|
98
|
+
if (!snapA.has(path)) actions.push({ kind: "copy", path, from: "b" });
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
return {
|
|
102
|
+
op: "bisync",
|
|
103
|
+
actions,
|
|
104
|
+
conflicts,
|
|
105
|
+
warnings: ["no sync anchor; performing conservative union (additions only, no deletions)"],
|
|
106
|
+
};
|
|
107
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The cheap-first change-detection ladder.
|
|
3
|
+
*
|
|
4
|
+
* Short-circuits from the cheapest signal to the most expensive:
|
|
5
|
+
* path/type → size → stable mtime → optional quick fingerprint →
|
|
6
|
+
* full content hash (injected `hashContent`).
|
|
7
|
+
*
|
|
8
|
+
* Each rung that can decide "same" or "changed" returns without paying for the
|
|
9
|
+
* next one — in particular a size difference marks a file changed WITHOUT ever
|
|
10
|
+
* hashing, and an equal size+mtime pair is treated as unchanged (rclone's
|
|
11
|
+
* default heuristic) without hashing either.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type { FilesApi } from "@statewalker/webrun-files";
|
|
15
|
+
import type { ByteStream, PathFilter } from "./types.js";
|
|
16
|
+
|
|
17
|
+
export interface Entry {
|
|
18
|
+
path: string;
|
|
19
|
+
kind: "file" | "directory";
|
|
20
|
+
size?: number;
|
|
21
|
+
mtime?: number;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export type Snapshot = Map<string, Entry>;
|
|
25
|
+
|
|
26
|
+
/** List a whole tree into a path-keyed snapshot, applying an optional filter. */
|
|
27
|
+
export async function snapshot(files: FilesApi, filter?: PathFilter): Promise<Snapshot> {
|
|
28
|
+
const map: Snapshot = new Map();
|
|
29
|
+
for await (const info of files.list("/", { recursive: true })) {
|
|
30
|
+
if (filter && info.kind === "file" && !filter(info.path)) continue;
|
|
31
|
+
map.set(info.path, {
|
|
32
|
+
path: info.path,
|
|
33
|
+
kind: info.kind,
|
|
34
|
+
size: info.kind === "file" ? info.size : undefined,
|
|
35
|
+
mtime: info.kind === "file" ? info.lastModified : undefined,
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
return map;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const HEAD_TAIL = 64;
|
|
42
|
+
|
|
43
|
+
/** Read a bounded byte range as a single buffer (for fingerprint sampling). */
|
|
44
|
+
async function readRange(
|
|
45
|
+
files: FilesApi,
|
|
46
|
+
path: string,
|
|
47
|
+
start: number,
|
|
48
|
+
length: number,
|
|
49
|
+
): Promise<Uint8Array> {
|
|
50
|
+
const parts: Uint8Array[] = [];
|
|
51
|
+
for await (const chunk of files.read(path, { start, length })) parts.push(chunk);
|
|
52
|
+
const total = parts.reduce((n, p) => n + p.length, 0);
|
|
53
|
+
const out = new Uint8Array(total);
|
|
54
|
+
let off = 0;
|
|
55
|
+
for (const p of parts) {
|
|
56
|
+
out.set(p, off);
|
|
57
|
+
off += p.length;
|
|
58
|
+
}
|
|
59
|
+
return out;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
|
|
63
|
+
if (a.length !== b.length) return false;
|
|
64
|
+
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
|
|
65
|
+
return true;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** head+tail sample of a file, used by the optional quick-fingerprint rung. */
|
|
69
|
+
async function fingerprint(files: FilesApi, path: string, size: number): Promise<Uint8Array> {
|
|
70
|
+
if (size <= HEAD_TAIL * 2) return readRange(files, path, 0, size);
|
|
71
|
+
const head = await readRange(files, path, 0, HEAD_TAIL);
|
|
72
|
+
const tail = await readRange(files, path, size - HEAD_TAIL, HEAD_TAIL);
|
|
73
|
+
const out = new Uint8Array(head.length + tail.length);
|
|
74
|
+
out.set(head, 0);
|
|
75
|
+
out.set(tail, head.length);
|
|
76
|
+
return out;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export interface LadderOptions {
|
|
80
|
+
hashContent: (input: ByteStream) => Promise<string>;
|
|
81
|
+
quickFingerprint?: boolean;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Decide whether `from`'s file at `path` differs from `to`'s file, climbing the
|
|
86
|
+
* ladder only as far as needed. `fromEntry` must exist; `toEntry` may be absent.
|
|
87
|
+
*/
|
|
88
|
+
export async function isChanged(
|
|
89
|
+
from: FilesApi,
|
|
90
|
+
to: FilesApi,
|
|
91
|
+
path: string,
|
|
92
|
+
fromEntry: Entry,
|
|
93
|
+
toEntry: Entry | undefined,
|
|
94
|
+
opts: LadderOptions,
|
|
95
|
+
): Promise<boolean> {
|
|
96
|
+
// path/type rung
|
|
97
|
+
if (!toEntry) return true;
|
|
98
|
+
if (fromEntry.kind !== toEntry.kind) return true;
|
|
99
|
+
if (fromEntry.kind === "directory") return false;
|
|
100
|
+
|
|
101
|
+
// size rung — a difference is decisive; never hashes.
|
|
102
|
+
if (fromEntry.size !== toEntry.size) return true;
|
|
103
|
+
|
|
104
|
+
// stable-mtime rung — equal size + equal mtime ⇒ unchanged, no hashing.
|
|
105
|
+
if (
|
|
106
|
+
fromEntry.mtime !== undefined &&
|
|
107
|
+
toEntry.mtime !== undefined &&
|
|
108
|
+
fromEntry.mtime === toEntry.mtime
|
|
109
|
+
) {
|
|
110
|
+
return false;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const size = fromEntry.size ?? 0;
|
|
114
|
+
|
|
115
|
+
// optional quick-fingerprint rung — a sample mismatch decides "changed"
|
|
116
|
+
// without a full hash; a match is inconclusive and falls through.
|
|
117
|
+
if (opts.quickFingerprint) {
|
|
118
|
+
const [fa, fb] = await Promise.all([
|
|
119
|
+
fingerprint(from, path, size),
|
|
120
|
+
fingerprint(to, path, size),
|
|
121
|
+
]);
|
|
122
|
+
if (!bytesEqual(fa, fb)) return true;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// full content-hash rung — the ambiguous case.
|
|
126
|
+
const [ha, hb] = await Promise.all([
|
|
127
|
+
opts.hashContent(from.read(path)),
|
|
128
|
+
opts.hashContent(to.read(path)),
|
|
129
|
+
]);
|
|
130
|
+
return ha !== hb;
|
|
131
|
+
}
|