@elaraai/e3-cloud-files 0.0.0-stage → 1.0.91-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +54 -0
- package/README.md +227 -2
- package/dist/src/authoring.d.ts +114 -0
- package/dist/src/authoring.d.ts.map +1 -0
- package/dist/src/authoring.js +178 -0
- package/dist/src/authoring.js.map +1 -0
- package/dist/src/check/index.d.ts +15 -0
- package/dist/src/check/index.d.ts.map +1 -0
- package/dist/src/check/index.js +15 -0
- package/dist/src/check/index.js.map +1 -0
- package/dist/src/check.d.ts +66 -0
- package/dist/src/check.d.ts.map +1 -0
- package/dist/src/check.js +140 -0
- package/dist/src/check.js.map +1 -0
- package/dist/src/index.d.ts +27 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +27 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/paths.d.ts +59 -0
- package/dist/src/paths.d.ts.map +1 -0
- package/dist/src/paths.js +114 -0
- package/dist/src/paths.js.map +1 -0
- package/dist/src/s3/index.d.ts +37 -0
- package/dist/src/s3/index.d.ts.map +1 -0
- package/dist/src/s3/index.js +76 -0
- package/dist/src/s3/index.js.map +1 -0
- package/dist/src/storage/index.d.ts +16 -0
- package/dist/src/storage/index.d.ts.map +1 -0
- package/dist/src/storage/index.js +117 -0
- package/dist/src/storage/index.js.map +1 -0
- package/dist/src/sync/engine.d.ts +136 -0
- package/dist/src/sync/engine.d.ts.map +1 -0
- package/dist/src/sync/engine.js +345 -0
- package/dist/src/sync/engine.js.map +1 -0
- package/dist/src/sync/index.d.ts +15 -0
- package/dist/src/sync/index.d.ts.map +1 -0
- package/dist/src/sync/index.js +15 -0
- package/dist/src/sync/index.js.map +1 -0
- package/dist/src/sync/memory.d.ts +65 -0
- package/dist/src/sync/memory.d.ts.map +1 -0
- package/dist/src/sync/memory.js +127 -0
- package/dist/src/sync/memory.js.map +1 -0
- package/dist/test/decode.examples.d.ts +33 -0
- package/dist/test/decode.examples.d.ts.map +1 -0
- package/dist/test/decode.examples.js +103 -0
- package/dist/test/decode.examples.js.map +1 -0
- package/dist/test/orders.examples.d.ts +88 -0
- package/dist/test/orders.examples.d.ts.map +1 -0
- package/dist/test/orders.examples.js +71 -0
- package/dist/test/orders.examples.js.map +1 -0
- package/package.json +76 -3
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* `@elaraai/e3-cloud-files/storage`: the sync's writer over an e3 store, the
|
|
7
|
+
* one a local repository and a deployed workspace both write through.
|
|
8
|
+
*
|
|
9
|
+
* A `file`, `latest` or `listing` input is written as any input is
|
|
10
|
+
* (`workspaceSetDataset`). A `folder` is updated a piece at a time: e3 stores
|
|
11
|
+
* a collection as pieces, each holding the entries between two keys, so only
|
|
12
|
+
* the pieces holding a changed or removed file are read and cut again, and
|
|
13
|
+
* every other piece is carried over as it is stored (`storeCollection`), never
|
|
14
|
+
* read. However large the folder, a sync holds the pieces it changes and the
|
|
15
|
+
* files it read. The new folder is then taken in as the input's value by e3's
|
|
16
|
+
* own door (`datasetAdoptKnown`), which checks its type, holds the dataset's
|
|
17
|
+
* write lock and versions it as any input write.
|
|
18
|
+
*
|
|
19
|
+
* Needs `@elaraai/e3-core`, which a solution's own code never imports: the
|
|
20
|
+
* CLI and the platform do.
|
|
21
|
+
*
|
|
22
|
+
* @packageDocumentation
|
|
23
|
+
*/
|
|
24
|
+
import { StringType, compareFor, decodeBeast2For, variant } from '@elaraai/east';
|
|
25
|
+
import { DatasetSegments, datasetAdoptKnown, storeCollection, workspaceGetDatasetHash, workspaceSetDataset, } from '@elaraai/e3-core';
|
|
26
|
+
import { FilesFileType, FilesFolderType, FilesListingType } from '@elaraai/e3-cloud-types';
|
|
27
|
+
/** An input's path: `.inputs.<input>`. */
|
|
28
|
+
function inputPath(input) {
|
|
29
|
+
return [variant('field', 'inputs'), variant('field', input)];
|
|
30
|
+
}
|
|
31
|
+
const compareKeys = compareFor(StringType);
|
|
32
|
+
const decodeFolder = decodeBeast2For(FilesFolderType);
|
|
33
|
+
/** Entries in a Dict's order. */
|
|
34
|
+
function ordered(entries) {
|
|
35
|
+
return [...entries].map(([k, v]) => [k, v]).sort(([a], [b]) => compareKeys(a, b));
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The sources a folder's new value is stored from: the pieces no change
|
|
39
|
+
* touches carried over by reference, and each piece a change touches read,
|
|
40
|
+
* changed and cut again.
|
|
41
|
+
*/
|
|
42
|
+
async function folderSources(storage, repo, hash, changes) {
|
|
43
|
+
const pieces = await DatasetSegments.open(storage, repo, hash);
|
|
44
|
+
if (pieces.counts.length === 0)
|
|
45
|
+
return [{ elements: ordered(changes.set) }];
|
|
46
|
+
const touched = new Map();
|
|
47
|
+
const at = async (path) => {
|
|
48
|
+
const i = await pieces.segmentFor(path);
|
|
49
|
+
let change = touched.get(i);
|
|
50
|
+
if (change === undefined)
|
|
51
|
+
touched.set(i, change = { set: [], remove: [] });
|
|
52
|
+
return change;
|
|
53
|
+
};
|
|
54
|
+
for (const [path, bytes] of changes.set)
|
|
55
|
+
(await at(path)).set.push([path, bytes]);
|
|
56
|
+
for (const path of changes.remove)
|
|
57
|
+
(await at(path)).remove.push(path);
|
|
58
|
+
const sources = [];
|
|
59
|
+
let next = 0;
|
|
60
|
+
for (const i of [...touched.keys()].sort((a, b) => a - b)) {
|
|
61
|
+
if (i > next)
|
|
62
|
+
sources.push({ stored: pieces.hash, from: next, to: i });
|
|
63
|
+
const entries = new Map(decodeFolder(await pieces.segment(i)));
|
|
64
|
+
const change = touched.get(i);
|
|
65
|
+
for (const path of change.remove)
|
|
66
|
+
entries.delete(path);
|
|
67
|
+
for (const [path, bytes] of change.set)
|
|
68
|
+
entries.set(path, bytes);
|
|
69
|
+
sources.push({ elements: ordered(entries) });
|
|
70
|
+
next = i + 1;
|
|
71
|
+
}
|
|
72
|
+
if (next < pieces.counts.length)
|
|
73
|
+
sources.push({ stored: pieces.hash, from: next, to: pieces.counts.length });
|
|
74
|
+
return sources;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* The sync's writer over an e3 store, for one workspace.
|
|
78
|
+
*
|
|
79
|
+
* @param storage - The store: a local repository's (`LocalStorage`) or the platform's
|
|
80
|
+
* @param repo - The repository: a local repository's path, or the platform's repository name
|
|
81
|
+
* @param ws - The workspace
|
|
82
|
+
* @returns The writer
|
|
83
|
+
*/
|
|
84
|
+
export function storageWriter(storage, repo, ws) {
|
|
85
|
+
const current = async (input) => (await workspaceGetDatasetHash(storage, repo, ws, inputPath(input))).hash;
|
|
86
|
+
const written = async (input) => {
|
|
87
|
+
const hash = await current(input);
|
|
88
|
+
if (hash === null)
|
|
89
|
+
throw new Error(`input ${input} is unassigned after it was written`);
|
|
90
|
+
return hash;
|
|
91
|
+
};
|
|
92
|
+
return {
|
|
93
|
+
current,
|
|
94
|
+
async setFile(input, bytes) {
|
|
95
|
+
await workspaceSetDataset(storage, repo, ws, inputPath(input), bytes, FilesFileType);
|
|
96
|
+
return written(input);
|
|
97
|
+
},
|
|
98
|
+
async setListing(input, rows) {
|
|
99
|
+
const value = new Map([...rows].sort(([a], [b]) => compareKeys(a, b)));
|
|
100
|
+
await workspaceSetDataset(storage, repo, ws, inputPath(input), value, FilesListingType);
|
|
101
|
+
return written(input);
|
|
102
|
+
},
|
|
103
|
+
async updateFolder(input, changes) {
|
|
104
|
+
const hash = changes.fresh ? null : await current(input);
|
|
105
|
+
const sources = hash === null ? [{ elements: ordered(changes.set) }] : await folderSources(storage, repo, hash, changes);
|
|
106
|
+
const stored = await storeCollection(storage, repo, FilesFolderType, sources);
|
|
107
|
+
// The store knows the new folder as the manifest it is, so e3's door
|
|
108
|
+
// takes it in as the input's value, checked and versioned, reading nothing
|
|
109
|
+
await storage.refs.adoptionWrite(repo, stored, stored);
|
|
110
|
+
const adopted = await datasetAdoptKnown(storage, repo, ws, inputPath(input), stored);
|
|
111
|
+
if (adopted === null)
|
|
112
|
+
throw new Error(`input ${input} could not take the folder ${stored}`);
|
|
113
|
+
return adopted.hash;
|
|
114
|
+
},
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACjF,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,uBAAuB,EACvB,mBAAmB,GAGpB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,gBAAgB,EAAwB,MAAM,yBAAyB,CAAC;AAGjH,0CAA0C;AAC1C,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,WAAW,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;AAC3C,MAAM,YAAY,GAAG,eAAe,CAAC,eAAe,CAAC,CAAC;AAEtD,iCAAiC;AACjC,SAAS,OAAO,CAAC,OAAgD;IAC/D,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAwB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAC1G,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,aAAa,CAAC,OAAuB,EAAE,IAAY,EAAE,IAAY,EAAE,OAAsB;IACtG,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC/D,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6D,CAAC;IACrF,MAAM,EAAE,GAAG,KAAK,EAAE,IAAY,EAAE,EAAE;QAChC,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3E,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,GAAG;QAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;IAClF,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,MAAM;QAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEtE,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,GAAG,IAAI;YAAE,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;QACvE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAqB,YAAY,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACvD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG;YAAE,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;IACD,IAAI,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7G,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,OAAuB,EAAE,IAAY,EAAE,EAAU;IAC7E,MAAM,OAAO,GAAG,KAAK,EAAE,KAAa,EAA0B,EAAE,CAC9D,CAAC,MAAM,uBAAuB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5E,MAAM,OAAO,GAAG,KAAK,EAAE,KAAa,EAAmB,EAAE;QACvD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,IAAI,KAAK,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,qCAAqC,CAAC,CAAC;QACxF,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,OAAO;QACL,OAAO;QACP,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK;YACxB,MAAM,mBAAmB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;YACrF,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,IAA0C;YAChE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YACvE,MAAM,mBAAmB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,gBAAgB,CAAC,CAAC;YACxF,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO;YAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,OAAO,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YACzH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YAC9E,qEAAqE;YACrE,2EAA2E;YAC3E,MAAM,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;YACrF,IAAI,OAAO,KAAK,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,8BAA8B,MAAM,EAAE,CAAC,CAAC;YAC5F,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
import { type FilesBinding, type FilesListingRow, type FilesManifest, type FilesProblem, type FilesSyncReport } from '@elaraai/e3-cloud-types';
|
|
6
|
+
/** A file of a source, as a listing gives it. */
|
|
7
|
+
export interface FilesObject {
|
|
8
|
+
/** Its key, relative to where the workspace binds the source */
|
|
9
|
+
readonly key: string;
|
|
10
|
+
/** Its size in bytes */
|
|
11
|
+
readonly size: number;
|
|
12
|
+
/** When it was last written */
|
|
13
|
+
readonly modified: Date;
|
|
14
|
+
/** Its ETag, which changes whenever its bytes do */
|
|
15
|
+
readonly etag: string;
|
|
16
|
+
}
|
|
17
|
+
/** Reads one bound source: a bucket, under the prefix the binding names. */
|
|
18
|
+
export interface FilesReader {
|
|
19
|
+
/** Every file under a prefix (relative to the binding), in key order */
|
|
20
|
+
list(prefix: string): AsyncIterable<FilesObject>;
|
|
21
|
+
/** One file's listing, or `null` when there is none */
|
|
22
|
+
stat(key: string): Promise<FilesObject | null>;
|
|
23
|
+
/** One file's bytes */
|
|
24
|
+
read(key: string): Promise<Uint8Array>;
|
|
25
|
+
}
|
|
26
|
+
/** Writes one workspace's inputs. Each write answers the value it set: its hash. */
|
|
27
|
+
export interface FilesWriter {
|
|
28
|
+
/** The value an input holds now, its hash; `null` while it is unassigned */
|
|
29
|
+
current(input: string): Promise<string | null>;
|
|
30
|
+
/** Sets a `file` or `latest` input to a file's bytes */
|
|
31
|
+
setFile(input: string, bytes: Uint8Array): Promise<string>;
|
|
32
|
+
/**
|
|
33
|
+
* Updates a `folder` input: each file in `set` added or replaced, each path
|
|
34
|
+
* in `remove` taken out, every other entry as it was; or, `fresh`, the
|
|
35
|
+
* folder made of `set` alone
|
|
36
|
+
*/
|
|
37
|
+
updateFolder(input: string, changes: FolderChanges): Promise<string>;
|
|
38
|
+
/** Sets a `listing` input */
|
|
39
|
+
setListing(input: string, rows: ReadonlyMap<string, FilesListingRow>): Promise<string>;
|
|
40
|
+
}
|
|
41
|
+
/** What a sync changes in a `folder` input. */
|
|
42
|
+
export interface FolderChanges {
|
|
43
|
+
/** Each file added or replaced, by its path relative to the prefix */
|
|
44
|
+
readonly set: ReadonlyMap<string, Uint8Array>;
|
|
45
|
+
/** Each path taken out */
|
|
46
|
+
readonly remove: ReadonlySet<string>;
|
|
47
|
+
/** Whether the folder is `set` alone, whatever the input holds now */
|
|
48
|
+
readonly fresh: boolean;
|
|
49
|
+
}
|
|
50
|
+
/** What an input took of one file. */
|
|
51
|
+
export interface LedgerEntry {
|
|
52
|
+
readonly etag: string;
|
|
53
|
+
readonly size: number;
|
|
54
|
+
}
|
|
55
|
+
/** What an input took last: each file, and the value the sync wrote. */
|
|
56
|
+
export interface LedgerState {
|
|
57
|
+
/** Each file it took, by key */
|
|
58
|
+
readonly entries: ReadonlyMap<string, LedgerEntry>;
|
|
59
|
+
/** The value the sync wrote, its hash; `null` before its first sync */
|
|
60
|
+
readonly value: string | null;
|
|
61
|
+
}
|
|
62
|
+
/** Remembers, for each input, what it took last. */
|
|
63
|
+
export interface FilesLedger {
|
|
64
|
+
/** What an input took last; nothing before its first sync */
|
|
65
|
+
read(input: string): Promise<LedgerState>;
|
|
66
|
+
/** Remembers what an input took */
|
|
67
|
+
write(input: string, state: LedgerState): Promise<void>;
|
|
68
|
+
}
|
|
69
|
+
/** What a sync is given. */
|
|
70
|
+
export interface SyncOptions {
|
|
71
|
+
/** The declarations */
|
|
72
|
+
readonly manifest: FilesManifest;
|
|
73
|
+
/** Where each source's files are */
|
|
74
|
+
readonly bindings: readonly FilesBinding[];
|
|
75
|
+
/** Reads a bound source */
|
|
76
|
+
readonly reader: (binding: FilesBinding) => FilesReader;
|
|
77
|
+
/** Writes the workspace's inputs */
|
|
78
|
+
readonly writer: FilesWriter;
|
|
79
|
+
/** Remembers what each input took */
|
|
80
|
+
readonly ledger: FilesLedger;
|
|
81
|
+
/** List what a sync would take, and take nothing */
|
|
82
|
+
readonly dryRun?: boolean;
|
|
83
|
+
/** Only these inputs; every mapping when absent */
|
|
84
|
+
readonly inputs?: readonly string[];
|
|
85
|
+
/** The time, for the report */
|
|
86
|
+
readonly now?: () => Date;
|
|
87
|
+
/** The most bytes of a folder's files held before they are written: {@link FOLDER_BATCH_BYTES} when absent */
|
|
88
|
+
readonly batchBytes?: number;
|
|
89
|
+
}
|
|
90
|
+
/** What one round of a sync is given: a sync's options, and when to stop. */
|
|
91
|
+
export interface SyncRoundOptions extends SyncOptions {
|
|
92
|
+
/**
|
|
93
|
+
* Asked between inputs and between a folder's batches, once the round has
|
|
94
|
+
* written something: `true` stops the round there, every input as it was
|
|
95
|
+
* last written and remembered
|
|
96
|
+
*/
|
|
97
|
+
readonly shouldYield?: () => boolean;
|
|
98
|
+
}
|
|
99
|
+
/** What a round did, and whether it stopped before the end. */
|
|
100
|
+
export interface FilesSyncRound {
|
|
101
|
+
/** What it did: each input it reached, with what it took */
|
|
102
|
+
readonly report: FilesSyncReport;
|
|
103
|
+
/** Whether it stopped when asked, with inputs or a folder's files left for the next */
|
|
104
|
+
readonly yielded: boolean;
|
|
105
|
+
}
|
|
106
|
+
/** The most bytes of a folder's files a sync holds before it writes them, when not told: 256 MiB. */
|
|
107
|
+
export declare const FOLDER_BATCH_BYTES: number;
|
|
108
|
+
/** Thrown when a sync cannot start, naming every problem. */
|
|
109
|
+
export declare class FilesSyncError extends Error {
|
|
110
|
+
readonly problems: readonly FilesProblem[];
|
|
111
|
+
constructor(problems: readonly FilesProblem[]);
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Brings each mapping's input up to date with its source's files, stopping
|
|
115
|
+
* when asked between inputs or between a folder's batches: one round of a
|
|
116
|
+
* sync that hands over, as the platform's do. What it stopped before, the
|
|
117
|
+
* next round takes.
|
|
118
|
+
*
|
|
119
|
+
* @param options - The declarations, the bindings, what reads, writes and remembers, and when to stop
|
|
120
|
+
* @returns What it did, per input it reached, and whether it stopped before the end
|
|
121
|
+
* @throws {FilesSyncError} When a mapping's source is not bound, before
|
|
122
|
+
* anything is read
|
|
123
|
+
* @throws {Error} When an input named is not one a declaration fills
|
|
124
|
+
*/
|
|
125
|
+
export declare function syncFilesRound(options: SyncRoundOptions): Promise<FilesSyncRound>;
|
|
126
|
+
/**
|
|
127
|
+
* Brings each mapping's input up to date with its source's files, to the end.
|
|
128
|
+
*
|
|
129
|
+
* @param options - The declarations, the bindings, and what reads, writes and remembers
|
|
130
|
+
* @returns What it did, per mapping
|
|
131
|
+
* @throws {FilesSyncError} When a mapping's source is not bound, before
|
|
132
|
+
* anything is read
|
|
133
|
+
* @throws {Error} When an input named is not one a declaration fills
|
|
134
|
+
*/
|
|
135
|
+
export declare function syncFiles(options: SyncOptions): Promise<FilesSyncReport>;
|
|
136
|
+
//# sourceMappingURL=engine.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../../src/sync/engine.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAmCH,OAAO,EAEL,KAAK,YAAY,EAAE,KAAK,eAAe,EAAE,KAAK,aAAa,EAC3D,KAAK,YAAY,EAAqB,KAAK,eAAe,EAC3D,MAAM,yBAAyB,CAAC;AAGjC,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,gEAAgE;IAChE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,wBAAwB;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,+BAA+B;IAC/B,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC;IACxB,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,4EAA4E;AAC5E,MAAM,WAAW,WAAW;IAC1B,wEAAwE;IACxE,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC;IACjD,uDAAuD;IACvD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;IAC/C,uBAAuB;IACvB,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CACxC;AAED,oFAAoF;AACpF,MAAM,WAAW,WAAW;IAC1B,4EAA4E;IAC5E,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC/C,wDAAwD;IACxD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3D;;;;OAIG;IACH,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,6BAA6B;IAC7B,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACxF;AAED,+CAA+C;AAC/C,MAAM,WAAW,aAAa;IAC5B,sEAAsE;IACtE,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC9C,0BAA0B;IAC1B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACrC,sEAAsE;IACtE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;CACzB;AAED,sCAAsC;AACtC,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,wEAAwE;AACxE,MAAM,WAAW,WAAW;IAC1B,gCAAgC;IAChC,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,uEAAuE;IACvE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,oDAAoD;AACpD,MAAM,WAAW,WAAW;IAC1B,6DAA6D;IAC7D,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAC1C,mCAAmC;IACnC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzD;AAED,4BAA4B;AAC5B,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,oCAAoC;IACpC,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C,2BAA2B;IAC3B,QAAQ,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,WAAW,CAAC;IACxD,oCAAoC;IACpC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,qCAAqC;IACrC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,oDAAoD;IACpD,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,mDAAmD;IACnD,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,+BAA+B;IAC/B,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B,8GAA8G;IAC9G,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,6EAA6E;AAC7E,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;;OAIG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,OAAO,CAAC;CACtC;AAED,+DAA+D;AAC/D,MAAM,WAAW,cAAc;IAC7B,4DAA4D;IAC5D,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;IACjC,uFAAuF;IACvF,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,qGAAqG;AACrG,eAAO,MAAM,kBAAkB,QAAoB,CAAC;AAKpD,6DAA6D;AAC7D,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;gBAE/B,QAAQ,EAAE,SAAS,YAAY,EAAE;CAK9C;AA2PD;;;;;;;;;;;GAWG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,cAAc,CAAC,CA6CvF;AAED;;;;;;;;GAQG;AACH,wBAAsB,SAAS,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,CAAC,CAE9E"}
|
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* The sync: what a source's files fill, brought up to date.
|
|
7
|
+
*
|
|
8
|
+
* One algorithm for a local repository and a deployed workspace alike: what
|
|
9
|
+
* reads a bucket, what writes a workspace's inputs and what remembers the
|
|
10
|
+
* files each input took are given to it ({@link FilesReader},
|
|
11
|
+
* {@link FilesWriter}, {@link FilesLedger}), so both behave the same.
|
|
12
|
+
*
|
|
13
|
+
* For each mapping: list the source, compare each file with what the input
|
|
14
|
+
* took last (its ETag and size), read what is new or changed, write the input,
|
|
15
|
+
* and remember what it took and the value it wrote. A file the same as before
|
|
16
|
+
* is never read again, so a sync with nothing new reads nothing; an input
|
|
17
|
+
* whose files are as they were is not written. A file over its shape's limit
|
|
18
|
+
* is refused, named in the report, and never truncated. An input is
|
|
19
|
+
* remembered only once it is written, so a sync that stops part way takes the
|
|
20
|
+
* rest next time.
|
|
21
|
+
*
|
|
22
|
+
* A folder's new and changed files are taken in batches of at most 256 MiB
|
|
23
|
+
* ({@link FOLDER_BATCH_BYTES}): each batch is read, written and remembered
|
|
24
|
+
* before the next is read, so a sync holds one batch of files, however large
|
|
25
|
+
* the folder. A sync can stop between inputs and between a folder's batches
|
|
26
|
+
* when asked ({@link syncFilesRound}, a cloud round handing over), and the
|
|
27
|
+
* next takes the rest.
|
|
28
|
+
*
|
|
29
|
+
* What an input took is trusted only while the input still holds the value
|
|
30
|
+
* the sync wrote: a deploy that resets the workspace's inputs, or anyone
|
|
31
|
+
* setting one by hand, leaves it holding another, and the next sync takes all
|
|
32
|
+
* of its files again.
|
|
33
|
+
*
|
|
34
|
+
* @packageDocumentation
|
|
35
|
+
*/
|
|
36
|
+
import { variant } from '@elaraai/east';
|
|
37
|
+
import { FILES_FILE_MAX_BYTES, FILES_FOLDER_ENTRY_MAX_BYTES, filesProblemText, } from '@elaraai/e3-cloud-types';
|
|
38
|
+
import { globMatcher } from '../paths.js';
|
|
39
|
+
/** The most bytes of a folder's files a sync holds before it writes them, when not told: 256 MiB. */
|
|
40
|
+
export const FOLDER_BATCH_BYTES = 256 * 1024 * 1024;
|
|
41
|
+
/** The most files of a folder a sync writes at once. */
|
|
42
|
+
const FOLDER_BATCH_FILES = 4096;
|
|
43
|
+
/** Thrown when a sync cannot start, naming every problem. */
|
|
44
|
+
export class FilesSyncError extends Error {
|
|
45
|
+
problems;
|
|
46
|
+
constructor(problems) {
|
|
47
|
+
super(`The files cannot be synced:\n${problems.map((p) => ` - ${filesProblemText(p)}`).join('\n')}`);
|
|
48
|
+
this.name = 'FilesSyncError';
|
|
49
|
+
this.problems = problems;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/** Whether a sync should stop here: asked to, once it has written something. */
|
|
53
|
+
function stopping(sync) {
|
|
54
|
+
if (!sync.yielded && sync.progressed && sync.options.shouldYield?.() === true)
|
|
55
|
+
sync.yielded = true;
|
|
56
|
+
return sync.yielded;
|
|
57
|
+
}
|
|
58
|
+
/** A tally of nothing. */
|
|
59
|
+
function emptyTally() {
|
|
60
|
+
return { added: 0n, changed: 0n, removed: 0n, unchanged: 0n, refused: [], bytes: 0n, updated: false };
|
|
61
|
+
}
|
|
62
|
+
/** A mapping's report, from its tally. */
|
|
63
|
+
function reportOf(mapping, tally) {
|
|
64
|
+
return { input: mapping.input, ...tally };
|
|
65
|
+
}
|
|
66
|
+
/** A key's path relative to a prefix. */
|
|
67
|
+
function relative(prefix, key) {
|
|
68
|
+
return key.slice(prefix.length);
|
|
69
|
+
}
|
|
70
|
+
/** Whether two ledger entries name the same bytes. */
|
|
71
|
+
function same(a, b) {
|
|
72
|
+
return a !== undefined && b !== undefined && a.etag === b.etag && a.size === b.size;
|
|
73
|
+
}
|
|
74
|
+
/** A file's ledger entry. */
|
|
75
|
+
function entryOf(object) {
|
|
76
|
+
return { etag: object.etag, size: object.size };
|
|
77
|
+
}
|
|
78
|
+
/** A mapping's binding; checked to exist before a sync starts. */
|
|
79
|
+
function bindingOf(mapping, options) {
|
|
80
|
+
return options.bindings.find((b) => b.source === mapping.source);
|
|
81
|
+
}
|
|
82
|
+
/** A mapping's reader: one per bound source, for the sync. */
|
|
83
|
+
function readerOf(sync, mapping) {
|
|
84
|
+
let reader = sync.readers.get(mapping.source);
|
|
85
|
+
if (reader === undefined)
|
|
86
|
+
sync.readers.set(mapping.source, reader = sync.options.reader(bindingOf(mapping, sync.options)));
|
|
87
|
+
return reader;
|
|
88
|
+
}
|
|
89
|
+
/** What an input took last, trusted only while it holds the value the sync wrote. */
|
|
90
|
+
async function before(sync, input) {
|
|
91
|
+
const [state, current] = await Promise.all([sync.options.ledger.read(input), sync.options.writer.current(input)]);
|
|
92
|
+
if (state.value !== null && state.value === current)
|
|
93
|
+
return { entries: state.entries, trusted: true, stale: false };
|
|
94
|
+
return { entries: new Map(), trusted: false, stale: current !== null || state.value !== null };
|
|
95
|
+
}
|
|
96
|
+
/** A folder's files to read, in batches of at most `maxBytes` (a file larger alone in its own). */
|
|
97
|
+
function batched(files, maxBytes) {
|
|
98
|
+
const batches = [];
|
|
99
|
+
let batch = [];
|
|
100
|
+
let bytes = 0;
|
|
101
|
+
for (const file of files) {
|
|
102
|
+
if (batch.length > 0 && (bytes + file.size > maxBytes || batch.length >= FOLDER_BATCH_FILES)) {
|
|
103
|
+
batches.push(batch);
|
|
104
|
+
batch = [];
|
|
105
|
+
bytes = 0;
|
|
106
|
+
}
|
|
107
|
+
batch.push(file);
|
|
108
|
+
bytes += file.size;
|
|
109
|
+
}
|
|
110
|
+
if (batch.length > 0)
|
|
111
|
+
batches.push(batch);
|
|
112
|
+
return batches;
|
|
113
|
+
}
|
|
114
|
+
/** The files a selection holds: under its prefix, matching its glob, keyed by their path relative to the prefix. */
|
|
115
|
+
async function selection(reader, prefix, include) {
|
|
116
|
+
const matches = globMatcher(include);
|
|
117
|
+
const files = new Map();
|
|
118
|
+
for await (const object of reader.list(prefix)) {
|
|
119
|
+
const path = relative(prefix, object.key);
|
|
120
|
+
// A key ending with / is a folder's placeholder, not a file
|
|
121
|
+
if (path === '' || object.key.endsWith('/'))
|
|
122
|
+
continue;
|
|
123
|
+
if (matches(path))
|
|
124
|
+
files.set(path, object);
|
|
125
|
+
}
|
|
126
|
+
return files;
|
|
127
|
+
}
|
|
128
|
+
/** Syncs a `file` or `latest` mapping to the one file it selects, or none. */
|
|
129
|
+
async function syncOne(sync, mapping, chosen, missing) {
|
|
130
|
+
const { options } = sync;
|
|
131
|
+
const report = emptyTally();
|
|
132
|
+
if (chosen === null) {
|
|
133
|
+
report.refused.push(missing);
|
|
134
|
+
return reportOf(mapping, report);
|
|
135
|
+
}
|
|
136
|
+
const prior = await before(sync, mapping.input);
|
|
137
|
+
if (same(prior.entries.get(chosen.key), entryOf(chosen))) {
|
|
138
|
+
report.unchanged = 1n;
|
|
139
|
+
return reportOf(mapping, report);
|
|
140
|
+
}
|
|
141
|
+
if (chosen.size > FILES_FILE_MAX_BYTES) {
|
|
142
|
+
report.refused.push({ key: chosen.key, reason: `is ${chosen.size} bytes, over the ${FILES_FILE_MAX_BYTES} a file input takes` });
|
|
143
|
+
return reportOf(mapping, report);
|
|
144
|
+
}
|
|
145
|
+
if (!prior.trusted && !prior.stale)
|
|
146
|
+
report.added = 1n;
|
|
147
|
+
else
|
|
148
|
+
report.changed = 1n;
|
|
149
|
+
report.bytes = BigInt(chosen.size);
|
|
150
|
+
report.updated = true;
|
|
151
|
+
if (options.dryRun === true)
|
|
152
|
+
return reportOf(mapping, report);
|
|
153
|
+
const value = await options.writer.setFile(mapping.input, await readerOf(sync, mapping).read(chosen.key));
|
|
154
|
+
await options.ledger.write(mapping.input, { entries: new Map([[chosen.key, entryOf(chosen)]]), value });
|
|
155
|
+
sync.progressed = true;
|
|
156
|
+
return reportOf(mapping, report);
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Syncs a `folder` mapping: what is new or changed read and written, what is
|
|
160
|
+
* gone taken out. The files to read are taken in batches, each written and
|
|
161
|
+
* remembered before the next is read; the round may stop between batches.
|
|
162
|
+
* What is gone goes with the first batch; an input the sync does not trust is
|
|
163
|
+
* made again, from the first batch alone, then added to.
|
|
164
|
+
*/
|
|
165
|
+
async function syncFolder(sync, mapping, prefix, files) {
|
|
166
|
+
const { options } = sync;
|
|
167
|
+
const report = emptyTally();
|
|
168
|
+
const prior = await before(sync, mapping.input);
|
|
169
|
+
const binding = bindingOf(mapping, options);
|
|
170
|
+
// On a local repository, `latest` keeps only the greatest keys
|
|
171
|
+
let kept = files;
|
|
172
|
+
if (binding.latest.type === 'some') {
|
|
173
|
+
const keys = [...files.keys()].sort();
|
|
174
|
+
kept = new Map(keys.slice(Math.max(0, keys.length - Number(binding.latest.value))).map((path) => [path, files.get(path)]));
|
|
175
|
+
}
|
|
176
|
+
// The plan: what is read, what stays as it is, what goes
|
|
177
|
+
const toRead = [];
|
|
178
|
+
const staying = new Set();
|
|
179
|
+
for (const object of kept.values()) {
|
|
180
|
+
const previous = prior.entries.get(object.key);
|
|
181
|
+
if (same(previous, entryOf(object))) {
|
|
182
|
+
report.unchanged += 1n;
|
|
183
|
+
staying.add(object.key);
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
if (object.size > FILES_FOLDER_ENTRY_MAX_BYTES) {
|
|
187
|
+
report.refused.push({ key: object.key, reason: `is ${object.size} bytes, over the ${FILES_FOLDER_ENTRY_MAX_BYTES} a folder's file may be` });
|
|
188
|
+
// What the input held of it, if anything, stays
|
|
189
|
+
if (previous !== undefined)
|
|
190
|
+
staying.add(object.key);
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
toRead.push(object);
|
|
194
|
+
}
|
|
195
|
+
const reading = new Set(toRead.map((object) => object.key));
|
|
196
|
+
const gone = [...prior.entries.keys()].filter((key) => !staying.has(key) && !reading.has(key));
|
|
197
|
+
// An input the sync does not trust is made again from the files alone, even when there are none
|
|
198
|
+
const fresh = !prior.trusted;
|
|
199
|
+
if (toRead.length === 0 && gone.length === 0 && !fresh)
|
|
200
|
+
return reportOf(mapping, report);
|
|
201
|
+
if (options.dryRun === true) {
|
|
202
|
+
for (const object of toRead) {
|
|
203
|
+
if (prior.entries.has(object.key))
|
|
204
|
+
report.changed += 1n;
|
|
205
|
+
else
|
|
206
|
+
report.added += 1n;
|
|
207
|
+
report.bytes += BigInt(object.size);
|
|
208
|
+
}
|
|
209
|
+
report.removed = BigInt(gone.length);
|
|
210
|
+
report.updated = true;
|
|
211
|
+
return reportOf(mapping, report);
|
|
212
|
+
}
|
|
213
|
+
const reader = readerOf(sync, mapping);
|
|
214
|
+
const entries = new Map(prior.entries);
|
|
215
|
+
const batches = batched(toRead, options.batchBytes ?? FOLDER_BATCH_BYTES);
|
|
216
|
+
let first = true;
|
|
217
|
+
for (const batch of batches.length === 0 ? [[]] : batches) {
|
|
218
|
+
if (!first && stopping(sync))
|
|
219
|
+
break;
|
|
220
|
+
const set = new Map();
|
|
221
|
+
for (const object of batch)
|
|
222
|
+
set.set(relative(prefix, object.key), await reader.read(object.key));
|
|
223
|
+
const remove = first ? new Set(gone.map((key) => relative(prefix, key))) : new Set();
|
|
224
|
+
const value = await options.writer.updateFolder(mapping.input, { set, remove, fresh: first && fresh });
|
|
225
|
+
if (first) {
|
|
226
|
+
for (const key of gone)
|
|
227
|
+
entries.delete(key);
|
|
228
|
+
report.removed = BigInt(gone.length);
|
|
229
|
+
}
|
|
230
|
+
for (const object of batch) {
|
|
231
|
+
if (prior.entries.has(object.key))
|
|
232
|
+
report.changed += 1n;
|
|
233
|
+
else
|
|
234
|
+
report.added += 1n;
|
|
235
|
+
report.bytes += BigInt(object.size);
|
|
236
|
+
entries.set(object.key, entryOf(object));
|
|
237
|
+
}
|
|
238
|
+
report.updated = true;
|
|
239
|
+
await options.ledger.write(mapping.input, { entries: new Map(entries), value });
|
|
240
|
+
sync.progressed = true;
|
|
241
|
+
first = false;
|
|
242
|
+
}
|
|
243
|
+
return reportOf(mapping, report);
|
|
244
|
+
}
|
|
245
|
+
/** Syncs a `listing` mapping: written whole whenever any file is new, changed or gone. */
|
|
246
|
+
async function syncListing(sync, mapping, files) {
|
|
247
|
+
const { options } = sync;
|
|
248
|
+
const report = emptyTally();
|
|
249
|
+
const prior = await before(sync, mapping.input);
|
|
250
|
+
const after = new Map();
|
|
251
|
+
const rows = new Map();
|
|
252
|
+
for (const [path, object] of files) {
|
|
253
|
+
const previous = prior.entries.get(object.key);
|
|
254
|
+
if (previous === undefined)
|
|
255
|
+
report.added += 1n;
|
|
256
|
+
else if (same(previous, entryOf(object)))
|
|
257
|
+
report.unchanged += 1n;
|
|
258
|
+
else
|
|
259
|
+
report.changed += 1n;
|
|
260
|
+
after.set(object.key, entryOf(object));
|
|
261
|
+
rows.set(path, { size: BigInt(object.size), modified: object.modified, etag: object.etag });
|
|
262
|
+
}
|
|
263
|
+
for (const key of prior.entries.keys())
|
|
264
|
+
if (!after.has(key))
|
|
265
|
+
report.removed += 1n;
|
|
266
|
+
report.updated = report.added > 0n || report.changed > 0n || report.removed > 0n || !prior.trusted;
|
|
267
|
+
if (options.dryRun === true || !report.updated)
|
|
268
|
+
return reportOf(mapping, report);
|
|
269
|
+
const value = await options.writer.setListing(mapping.input, rows);
|
|
270
|
+
await options.ledger.write(mapping.input, { entries: after, value });
|
|
271
|
+
sync.progressed = true;
|
|
272
|
+
return reportOf(mapping, report);
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Brings each mapping's input up to date with its source's files, stopping
|
|
276
|
+
* when asked between inputs or between a folder's batches: one round of a
|
|
277
|
+
* sync that hands over, as the platform's do. What it stopped before, the
|
|
278
|
+
* next round takes.
|
|
279
|
+
*
|
|
280
|
+
* @param options - The declarations, the bindings, what reads, writes and remembers, and when to stop
|
|
281
|
+
* @returns What it did, per input it reached, and whether it stopped before the end
|
|
282
|
+
* @throws {FilesSyncError} When a mapping's source is not bound, before
|
|
283
|
+
* anything is read
|
|
284
|
+
* @throws {Error} When an input named is not one a declaration fills
|
|
285
|
+
*/
|
|
286
|
+
export async function syncFilesRound(options) {
|
|
287
|
+
const now = options.now ?? (() => new Date());
|
|
288
|
+
const started = now();
|
|
289
|
+
const declared = new Set(options.manifest.mappings.map((m) => m.input));
|
|
290
|
+
const unknown = (options.inputs ?? []).filter((input) => !declared.has(input));
|
|
291
|
+
if (unknown.length > 0) {
|
|
292
|
+
throw new Error(`No declaration fills ${unknown.map((i) => `'${i}'`).join(', ')}: the inputs files fill are ${[...declared].join(', ') || 'none'}`);
|
|
293
|
+
}
|
|
294
|
+
const wanted = options.inputs === undefined ? null : new Set(options.inputs);
|
|
295
|
+
const mappings = options.manifest.mappings.filter((m) => wanted === null || wanted.has(m.input));
|
|
296
|
+
const bound = new Set(options.bindings.map((b) => b.source));
|
|
297
|
+
const problems = [...new Set(mappings.map((m) => m.source))]
|
|
298
|
+
.filter((source) => !bound.has(source))
|
|
299
|
+
.map((source) => variant('source_unbound', { source }));
|
|
300
|
+
if (problems.length > 0)
|
|
301
|
+
throw new FilesSyncError(problems);
|
|
302
|
+
const sync = { options, readers: new Map(), progressed: false, yielded: false };
|
|
303
|
+
const reports = [];
|
|
304
|
+
for (const mapping of mappings) {
|
|
305
|
+
if (stopping(sync))
|
|
306
|
+
break;
|
|
307
|
+
const reader = readerOf(sync, mapping);
|
|
308
|
+
const shape = mapping.shape;
|
|
309
|
+
if (shape.type === 'file') {
|
|
310
|
+
const missing = { key: shape.value.key, reason: 'no such file' };
|
|
311
|
+
reports.push(await syncOne(sync, mapping, await reader.stat(shape.value.key), missing));
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
const { prefix } = shape.value;
|
|
315
|
+
const files = await selection(reader, prefix, shape.value.include.type === 'some' ? shape.value.include.value : null);
|
|
316
|
+
switch (shape.type) {
|
|
317
|
+
case 'latest': {
|
|
318
|
+
const greatest = [...files.keys()].sort().at(-1);
|
|
319
|
+
const missing = { key: prefix, reason: 'no file under it matches' };
|
|
320
|
+
reports.push(await syncOne(sync, mapping, greatest === undefined ? null : files.get(greatest), missing));
|
|
321
|
+
break;
|
|
322
|
+
}
|
|
323
|
+
case 'folder':
|
|
324
|
+
reports.push(await syncFolder(sync, mapping, prefix, files));
|
|
325
|
+
break;
|
|
326
|
+
case 'listing':
|
|
327
|
+
reports.push(await syncListing(sync, mapping, files));
|
|
328
|
+
break;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
return { report: { started, finished: now(), dryRun: options.dryRun === true, mappings: reports }, yielded: sync.yielded };
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* Brings each mapping's input up to date with its source's files, to the end.
|
|
335
|
+
*
|
|
336
|
+
* @param options - The declarations, the bindings, and what reads, writes and remembers
|
|
337
|
+
* @returns What it did, per mapping
|
|
338
|
+
* @throws {FilesSyncError} When a mapping's source is not bound, before
|
|
339
|
+
* anything is read
|
|
340
|
+
* @throws {Error} When an input named is not one a declaration fills
|
|
341
|
+
*/
|
|
342
|
+
export async function syncFiles(options) {
|
|
343
|
+
return (await syncFilesRound(options)).report;
|
|
344
|
+
}
|
|
345
|
+
//# sourceMappingURL=engine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"engine.js","sourceRoot":"","sources":["../../../src/sync/engine.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EACL,oBAAoB,EAAE,4BAA4B,EAAE,gBAAgB,GAGrE,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAgH1C,qGAAqG;AACrG,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAEpD,wDAAwD;AACxD,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC,6DAA6D;AAC7D,MAAM,OAAO,cAAe,SAAQ,KAAK;IAC9B,QAAQ,CAA0B;IAE3C,YAAY,QAAiC;QAC3C,KAAK,CAAC,gCAAgC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtG,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;CACF;AAYD,gFAAgF;AAChF,SAAS,QAAQ,CAAC,IAAU;IAC1B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,KAAK,IAAI;QAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACnG,OAAO,IAAI,CAAC,OAAO,CAAC;AACtB,CAAC;AAuBD,0BAA0B;AAC1B,SAAS,UAAU;IACjB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AACxG,CAAC;AAED,0CAA0C;AAC1C,SAAS,QAAQ,CAAC,OAAqB,EAAE,KAAY;IACnD,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,KAAK,EAAE,CAAC;AAC5C,CAAC;AAED,yCAAyC;AACzC,SAAS,QAAQ,CAAC,MAAc,EAAE,GAAW;IAC3C,OAAO,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAClC,CAAC;AAED,sDAAsD;AACtD,SAAS,IAAI,CAAC,CAA0B,EAAE,CAA0B;IAClE,OAAO,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC;AACtF,CAAC;AAED,6BAA6B;AAC7B,SAAS,OAAO,CAAC,MAAmB;IAClC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;AAClD,CAAC;AAED,kEAAkE;AAClE,SAAS,SAAS,CAAC,OAAqB,EAAE,OAAoB;IAC5D,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAAE,CAAC;AACpE,CAAC;AAED,8DAA8D;AAC9D,SAAS,QAAQ,CAAC,IAAU,EAAE,OAAqB;IACjD,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9C,IAAI,MAAM,KAAK,SAAS;QAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC3H,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,qFAAqF;AACrF,KAAK,UAAU,MAAM,CAAC,IAAU,EAAE,KAAa;IAC7C,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAClH,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,KAAK,KAAK,OAAO;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IACpH,OAAO,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,KAAK,IAAI,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;AACjG,CAAC;AAED,mGAAmG;AACnG,SAAS,OAAO,CAAC,KAA6B,EAAE,QAAgB;IAC9D,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,IAAI,KAAK,GAAkB,EAAE,CAAC;IAC9B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,GAAG,QAAQ,IAAI,KAAK,CAAC,MAAM,IAAI,kBAAkB,CAAC,EAAE,CAAC;YAC7F,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,GAAG,EAAE,CAAC;YACX,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1C,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,oHAAoH;AACpH,KAAK,UAAU,SAAS,CAAC,MAAmB,EAAE,MAAc,EAAE,OAAsB;IAClF,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC7C,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1C,4DAA4D;QAC5D,IAAI,IAAI,KAAK,EAAE,IAAI,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QACtD,IAAI,OAAO,CAAC,IAAI,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,KAAK,UAAU,OAAO,CACpB,IAAU,EAAE,OAAqB,EAAE,MAA0B,EAAE,OAAqB;IAEpF,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IACzB,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACpB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAChD,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QACzD,MAAM,CAAC,SAAS,GAAG,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,GAAG,oBAAoB,EAAE,CAAC;QACvC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC,IAAI,oBAAoB,oBAAoB,qBAAqB,EAAE,CAAC,CAAC;QACjI,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK;QAAE,MAAM,CAAC,KAAK,GAAG,EAAE,CAAC;;QACjD,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC;IACzB,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI;QAAE,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1G,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACxG,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACvB,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,UAAU,CACvB,IAAU,EAAE,OAAqB,EAAE,MAAc,EAAE,KAA+B;IAElF,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IACzB,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5C,+DAA+D;IAC/D,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,CAAC,CAAC,CAAC;IAC9H,CAAC;IACD,yDAAyD;IACzD,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACxB,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;YAC/C,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC,IAAI,oBAAoB,4BAA4B,yBAAyB,EAAE,CAAC,CAAC;YAC7I,gDAAgD;YAChD,IAAI,QAAQ,KAAK,SAAS;gBAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACpD,SAAS;QACX,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5D,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/F,gGAAgG;IAChG,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;IAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK;QAAE,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACzF,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QAC5B,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;gBAAE,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;;gBACnD,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QACD,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC,CAAC;IAC1E,IAAI,KAAK,GAAG,IAAI,CAAC;IACjB,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAC1D,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM;QACpC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAsB,CAAC;QAC1C,KAAK,MAAM,MAAM,IAAI,KAAK;YAAE,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACjG,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAU,CAAC;QAC7F,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;QACvG,IAAI,KAAK,EAAE,CAAC;YACV,KAAK,MAAM,GAAG,IAAI,IAAI;gBAAE,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;gBAAE,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;;gBACnD,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,KAAK,GAAG,KAAK,CAAC;IAChB,CAAC;IACD,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;AACnC,CAAC;AAED,0FAA0F;AAC1F,KAAK,UAAU,WAAW,CACxB,IAAU,EAAE,OAAqB,EAAE,KAA+B;IAElE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IACzB,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAA2B,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,QAAQ,KAAK,SAAS;YAAE,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;aAC1C,IAAI,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAAE,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;;YAC5D,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;QAC1B,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9F,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IAClF,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,GAAG,EAAE,IAAI,MAAM,CAAC,OAAO,GAAG,EAAE,IAAI,MAAM,CAAC,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;IACnG,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACjF,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACnE,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACrE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACvB,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAyB;IAC5D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;IACtB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,wBAAwB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;IACtJ,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACjG,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAmB,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;SACzE,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SACtC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1D,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,cAAc,CAAC,QAAQ,CAAC,CAAC;IAE5D,MAAM,IAAI,GAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACtF,MAAM,OAAO,GAAyB,EAAE,CAAC;IACzC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM;QAC1B,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YACxF,SAAS;QACX,CAAC;QACD,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtH,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjD,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC;gBACpE,OAAO,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAE,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC1G,MAAM;YACR,CAAC;YACD,KAAK,QAAQ;gBACX,OAAO,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC7D,MAAM;YACR,KAAK,SAAS;gBACZ,OAAO,CAAC,IAAI,CAAC,MAAM,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBACtD,MAAM;QACV,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;AAC7H,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAoB;IAClD,OAAO,CAAC,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;AAChD,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* `@elaraai/e3-cloud-files/sync`: the sync, with what reads a bucket, writes
|
|
7
|
+
* a workspace and remembers what each input took given to it, so a local
|
|
8
|
+
* repository and a deployed workspace sync the same way; and those three in
|
|
9
|
+
* memory, for tests.
|
|
10
|
+
*
|
|
11
|
+
* @packageDocumentation
|
|
12
|
+
*/
|
|
13
|
+
export { syncFiles, syncFilesRound, FilesSyncError, FOLDER_BATCH_BYTES, type SyncOptions, type SyncRoundOptions, type FilesSyncRound, type FilesObject, type FilesReader, type FilesWriter, type FolderChanges, type FilesLedger, type LedgerEntry, type LedgerState, } from './engine.js';
|
|
14
|
+
export { MemoryBucket, MemoryWorkspace, MemoryLedger } from './memory.js';
|
|
15
|
+
//# sourceMappingURL=index.d.ts.map
|