@elaraai/e3-cloud-files 0.0.0-stage → 1.0.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +54 -0
- package/README.md +286 -2
- package/dist/src/authoring.d.ts +114 -0
- package/dist/src/authoring.d.ts.map +1 -0
- package/dist/src/authoring.js +178 -0
- package/dist/src/authoring.js.map +1 -0
- package/dist/src/check/index.d.ts +15 -0
- package/dist/src/check/index.d.ts.map +1 -0
- package/dist/src/check/index.js +15 -0
- package/dist/src/check/index.js.map +1 -0
- package/dist/src/check.d.ts +66 -0
- package/dist/src/check.d.ts.map +1 -0
- package/dist/src/check.js +140 -0
- package/dist/src/check.js.map +1 -0
- package/dist/src/index.d.ts +27 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +27 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/paths.d.ts +70 -0
- package/dist/src/paths.d.ts.map +1 -0
- package/dist/src/paths.js +205 -0
- package/dist/src/paths.js.map +1 -0
- package/dist/src/s3/index.d.ts +44 -0
- package/dist/src/s3/index.d.ts.map +1 -0
- package/dist/src/s3/index.js +138 -0
- package/dist/src/s3/index.js.map +1 -0
- package/dist/src/storage/index.d.ts +16 -0
- package/dist/src/storage/index.d.ts.map +1 -0
- package/dist/src/storage/index.js +120 -0
- package/dist/src/storage/index.js.map +1 -0
- package/dist/src/sync/engine.d.ts +191 -0
- package/dist/src/sync/engine.d.ts.map +1 -0
- package/dist/src/sync/engine.js +481 -0
- package/dist/src/sync/engine.js.map +1 -0
- package/dist/src/sync/index.d.ts +15 -0
- package/dist/src/sync/index.d.ts.map +1 -0
- package/dist/src/sync/index.js +15 -0
- package/dist/src/sync/index.js.map +1 -0
- package/dist/src/sync/memory.d.ts +75 -0
- package/dist/src/sync/memory.d.ts.map +1 -0
- package/dist/src/sync/memory.js +155 -0
- package/dist/src/sync/memory.js.map +1 -0
- package/dist/test/decode.examples.d.ts +33 -0
- package/dist/test/decode.examples.d.ts.map +1 -0
- package/dist/test/decode.examples.js +103 -0
- package/dist/test/decode.examples.js.map +1 -0
- package/dist/test/orders.examples.d.ts +88 -0
- package/dist/test/orders.examples.d.ts.map +1 -0
- package/dist/test/orders.examples.js +71 -0
- package/dist/test/orders.examples.js.map +1 -0
- package/package.json +76 -3
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* `@elaraai/e3-cloud-files/s3`: the sync's reader of a bucket (`s3://`), one
|
|
7
|
+
* bound source read through a client the caller makes. The CLI makes it with
|
|
8
|
+
* the developer's own profile, the platform with a session scoped to the
|
|
9
|
+
* binding's bucket and prefix; both read through this.
|
|
10
|
+
*
|
|
11
|
+
* A listing is the bucket's, a page of 1,000 keys at a time, in key order;
|
|
12
|
+
* of a prefix's own folder alone (a delimiter), when the sync asks. An ETag is
|
|
13
|
+
* kept without the quotes the bucket wraps it in. One file's listing is a
|
|
14
|
+
* listing of its key alone, not a HEAD: a session that may list only under
|
|
15
|
+
* its binding's prefix is answered 403 for a HEAD of a key that isn't there,
|
|
16
|
+
* not 404. Credentials that may read the file but list nothing are asked by
|
|
17
|
+
* a HEAD instead, a 404 its absence. A file is read whole, at the version its listing gave (`If-Match`
|
|
18
|
+
* its ETag): at most 1 GiB for a `file` or `latest` input, 64 MiB for a
|
|
19
|
+
* folder's file. A file changed, archived, gone or closed to the reader since
|
|
20
|
+
* it was listed is refused (`FilesUnreadableError`), the sync going on with
|
|
21
|
+
* the rest; any other refusal is thrown.
|
|
22
|
+
*
|
|
23
|
+
* Needs `@aws-sdk/client-s3`, which a solution's own code never imports: the
|
|
24
|
+
* CLI and the platform do.
|
|
25
|
+
*
|
|
26
|
+
* @packageDocumentation
|
|
27
|
+
*/
|
|
28
|
+
import { GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, S3ServiceException } from '@aws-sdk/client-s3';
|
|
29
|
+
import { FilesUnreadableError } from '../sync/engine.js';
|
|
30
|
+
/** An ETag without the quotes the bucket wraps it in. */
|
|
31
|
+
function bare(etag) {
|
|
32
|
+
return (etag ?? '').replace(/^"|"$/g, '');
|
|
33
|
+
}
|
|
34
|
+
/** Whether the bucket answered that there is no such key. */
|
|
35
|
+
function missing(err) {
|
|
36
|
+
return err instanceof S3ServiceException && (err.name === 'NotFound' || err.name === 'NoSuchKey' || err.$metadata.httpStatusCode === 404);
|
|
37
|
+
}
|
|
38
|
+
/** Whether the bucket refused a request for want of access. */
|
|
39
|
+
function denied(err) {
|
|
40
|
+
return err instanceof S3ServiceException && (err.name === 'AccessDenied' || err.$metadata.httpStatusCode === 403);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Why the bucket refused to read one file, when the refusal is the file's
|
|
44
|
+
* own; `null` for any other, which the sync does not go on past.
|
|
45
|
+
*/
|
|
46
|
+
function unreadable(err) {
|
|
47
|
+
if (!(err instanceof S3ServiceException))
|
|
48
|
+
return null;
|
|
49
|
+
switch (err.name) {
|
|
50
|
+
case 'PreconditionFailed': return 'changed since it was listed: the next sync takes it';
|
|
51
|
+
case 'InvalidObjectState': return 'is archived: its storage class must be restored before it can be read';
|
|
52
|
+
case 'AccessDenied': return "can't be read with the binding's access: a key policy or the bucket's refuses it";
|
|
53
|
+
case 'NoSuchKey': return 'went while the sync read it';
|
|
54
|
+
default: return err.$metadata.httpStatusCode === 404 ? 'went while the sync read it' : null;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* The reader of one bound source in its bucket.
|
|
59
|
+
*
|
|
60
|
+
* @param client - The client to read with: its credentials and region are the caller's
|
|
61
|
+
* @param location - The bucket, and the prefix the binding names
|
|
62
|
+
* @returns The reader: its keys relative to the prefix
|
|
63
|
+
*/
|
|
64
|
+
export function s3FilesReader(client, location) {
|
|
65
|
+
const absolute = (key) => `${location.prefix}${key}`;
|
|
66
|
+
return {
|
|
67
|
+
async *list(prefix, options = {}) {
|
|
68
|
+
let token;
|
|
69
|
+
do {
|
|
70
|
+
const page = await client.send(new ListObjectsV2Command({
|
|
71
|
+
Bucket: location.bucket,
|
|
72
|
+
Prefix: absolute(prefix),
|
|
73
|
+
ContinuationToken: token,
|
|
74
|
+
// The prefix's own folder: its subfolders are common prefixes, never listed
|
|
75
|
+
...(options.topFolderOnly === true && { Delimiter: '/' }),
|
|
76
|
+
}));
|
|
77
|
+
for (const object of page.Contents ?? []) {
|
|
78
|
+
if (object.Key === undefined)
|
|
79
|
+
continue;
|
|
80
|
+
yield {
|
|
81
|
+
key: object.Key.slice(location.prefix.length),
|
|
82
|
+
size: object.Size ?? 0,
|
|
83
|
+
modified: object.LastModified ?? new Date(0),
|
|
84
|
+
etag: bare(object.ETag),
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
token = page.IsTruncated === true ? page.NextContinuationToken : undefined;
|
|
88
|
+
} while (token !== undefined);
|
|
89
|
+
},
|
|
90
|
+
async stat(key) {
|
|
91
|
+
// The key's own listing comes first among those it begins: it is the
|
|
92
|
+
// shortest of them, and a listing is in key order
|
|
93
|
+
let page;
|
|
94
|
+
try {
|
|
95
|
+
page = await client.send(new ListObjectsV2Command({ Bucket: location.bucket, Prefix: absolute(key), MaxKeys: 1 }));
|
|
96
|
+
}
|
|
97
|
+
catch (err) {
|
|
98
|
+
if (!denied(err))
|
|
99
|
+
throw err;
|
|
100
|
+
// Credentials that may read the file and list nothing: a HEAD's 404 is its absence
|
|
101
|
+
try {
|
|
102
|
+
const head = await client.send(new HeadObjectCommand({ Bucket: location.bucket, Key: absolute(key) }));
|
|
103
|
+
return { key, size: head.ContentLength ?? 0, modified: head.LastModified ?? new Date(0), etag: bare(head.ETag) };
|
|
104
|
+
}
|
|
105
|
+
catch (headErr) {
|
|
106
|
+
if (missing(headErr))
|
|
107
|
+
return null;
|
|
108
|
+
throw headErr;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
const object = page.Contents?.[0];
|
|
112
|
+
if (object?.Key !== absolute(key))
|
|
113
|
+
return null;
|
|
114
|
+
return { key, size: object.Size ?? 0, modified: object.LastModified ?? new Date(0), etag: bare(object.ETag) };
|
|
115
|
+
},
|
|
116
|
+
async read(file) {
|
|
117
|
+
let object;
|
|
118
|
+
try {
|
|
119
|
+
object = await client.send(new GetObjectCommand({ Bucket: location.bucket, Key: absolute(file.key), IfMatch: `"${file.etag}"` }));
|
|
120
|
+
}
|
|
121
|
+
catch (err) {
|
|
122
|
+
const reason = unreadable(err);
|
|
123
|
+
if (reason !== null)
|
|
124
|
+
throw new FilesUnreadableError(file.key, reason);
|
|
125
|
+
throw err;
|
|
126
|
+
}
|
|
127
|
+
if (object.Body === undefined)
|
|
128
|
+
throw new Error(`s3://${location.bucket}/${absolute(file.key)} answered no bytes`);
|
|
129
|
+
if (object.ContentLength !== undefined && object.ContentLength !== file.size) {
|
|
130
|
+
// Never buffered: its size was never checked
|
|
131
|
+
object.Body.destroy?.();
|
|
132
|
+
throw new FilesUnreadableError(file.key, `changed since it was listed (${object.ContentLength} bytes, not the ${file.size} listed): the next sync takes it`);
|
|
133
|
+
}
|
|
134
|
+
return object.Body.transformToByteArray();
|
|
135
|
+
},
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/s3/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,kBAAkB,EAAiB,MAAM,oBAAoB,CAAC;AAClI,OAAO,EAAE,oBAAoB,EAAsC,MAAM,mBAAmB,CAAC;AAS7F,yDAAyD;AACzD,SAAS,IAAI,CAAC,IAAwB;IACpC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED,6DAA6D;AAC7D,SAAS,OAAO,CAAC,GAAY;IAC3B,OAAO,GAAG,YAAY,kBAAkB,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,UAAU,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,CAAC,SAAS,CAAC,cAAc,KAAK,GAAG,CAAC,CAAC;AAC5I,CAAC;AAED,+DAA+D;AAC/D,SAAS,MAAM,CAAC,GAAY;IAC1B,OAAO,GAAG,YAAY,kBAAkB,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,GAAG,CAAC,SAAS,CAAC,cAAc,KAAK,GAAG,CAAC,CAAC;AACpH,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,GAAY;IAC9B,IAAI,CAAC,CAAC,GAAG,YAAY,kBAAkB,CAAC;QAAE,OAAO,IAAI,CAAC;IACtD,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,KAAK,oBAAoB,CAAC,CAAC,OAAO,qDAAqD,CAAC;QACxF,KAAK,oBAAoB,CAAC,CAAC,OAAO,uEAAuE,CAAC;QAC1G,KAAK,cAAc,CAAC,CAAC,OAAO,kFAAkF,CAAC;QAC/G,KAAK,WAAW,CAAC,CAAC,OAAO,6BAA6B,CAAC;QACvD,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC,SAAS,CAAC,cAAc,KAAK,GAAG,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9F,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,MAAgB,EAAE,QAAoB;IAClE,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;IAC7D,OAAO;QACL,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,GAAG,EAAE;YAC9B,IAAI,KAAyB,CAAC;YAC9B,GAAG,CAAC;gBACF,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC;oBACtD,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC;oBACxB,iBAAiB,EAAE,KAAK;oBACxB,4EAA4E;oBAC5E,GAAG,CAAC,OAAO,CAAC,aAAa,KAAK,IAAI,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;iBAC1D,CAAC,CAAC,CAAC;gBACJ,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;oBACzC,IAAI,MAAM,CAAC,GAAG,KAAK,SAAS;wBAAE,SAAS;oBACvC,MAAM;wBACJ,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;wBAC7C,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC;wBACtB,QAAQ,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC;wBAC5C,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;qBACxB,CAAC;gBACJ,CAAC;gBACD,KAAK,GAAG,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7E,CAAC,QAAQ,KAAK,KAAK,SAAS,EAAE;QAChC,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG;YACZ,qEAAqE;YACrE,kDAAkD;YAClD,IAAI,IAAI,CAAC;YACT,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACrH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;oBAAE,MAAM,GAAG,CAAC;gBAC5B,mFAAmF;gBACnF,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,iBAAiB,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;oBACvG,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnH,CAAC;gBAAC,OAAO,OAAO,EAAE,CAAC;oBACjB,IAAI,OAAO,CAAC,OAAO,CAAC;wBAAE,OAAO,IAAI,CAAC;oBAClC,MAAM,OAAO,CAAC;gBAChB,CAAC;YACH,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YAClC,IAAI,MAAM,EAAE,GAAG,KAAK,QAAQ,CAAC,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC/C,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAChH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI;YACb,IAAI,MAAM,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;YACpI,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC/B,IAAI,MAAM,KAAK,IAAI;oBAAE,MAAM,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBACtE,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YAClH,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC7E,6CAA6C;gBAC5C,MAAM,CAAC,IAAiC,CAAC,OAAO,EAAE,EAAE,CAAC;gBACtD,MAAM,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,gCAAgC,MAAM,CAAC,aAAa,mBAAmB,IAAI,CAAC,IAAI,kCAAkC,CAAC,CAAC;YAC/J,CAAC;YACD,OAAO,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5C,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
import { type StorageBackend } from '@elaraai/e3-core';
|
|
6
|
+
import type { FilesWriter } from '../sync/engine.js';
|
|
7
|
+
/**
|
|
8
|
+
* The sync's writer over an e3 store, for one workspace.
|
|
9
|
+
*
|
|
10
|
+
* @param storage - The store: a local repository's (`LocalStorage`) or the platform's
|
|
11
|
+
* @param repo - The repository: a local repository's path, or the platform's repository name
|
|
12
|
+
* @param ws - The workspace
|
|
13
|
+
* @returns The writer
|
|
14
|
+
*/
|
|
15
|
+
export declare function storageWriter(storage: StorageBackend, repo: string, ws: string): FilesWriter;
|
|
16
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuBH,OAAO,EAOL,KAAK,cAAc,EACpB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,KAAK,EAAE,WAAW,EAAiB,MAAM,mBAAmB,CAAC;AAgDpE;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,WAAW,CAkC5F"}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* `@elaraai/e3-cloud-files/storage`: the sync's writer over an e3 store, the
|
|
7
|
+
* one a local repository and a deployed workspace both write through.
|
|
8
|
+
*
|
|
9
|
+
* A `file`, `latest` or `listing` input is written as any input is
|
|
10
|
+
* (`workspaceSetDataset`). A `folder` is updated a piece at a time: e3 stores
|
|
11
|
+
* a collection as pieces, each holding the entries between two keys, so only
|
|
12
|
+
* the pieces holding a changed or removed file are read and cut again, and
|
|
13
|
+
* every other piece is carried over as it is stored (`storeCollection`), never
|
|
14
|
+
* read. However large the folder, a sync holds the pieces it changes and the
|
|
15
|
+
* files it read. The new folder is then taken in as the input's value by e3's
|
|
16
|
+
* own door (`datasetAdoptKnown`), which checks its type, holds the dataset's
|
|
17
|
+
* write lock and versions it as any input write.
|
|
18
|
+
*
|
|
19
|
+
* Needs `@elaraai/e3-core`, which a solution's own code never imports: the
|
|
20
|
+
* CLI and the platform do.
|
|
21
|
+
*
|
|
22
|
+
* @packageDocumentation
|
|
23
|
+
*/
|
|
24
|
+
import { StringType, compareFor, decodeBeast2For, variant } from '@elaraai/east';
|
|
25
|
+
import { DatasetSegments, datasetAdoptKnown, storeCollection, workspaceGetDatasetHash, workspaceSetDataset, } from '@elaraai/e3-core';
|
|
26
|
+
import { FilesFileType, FilesFolderType, FilesListingType } from '@elaraai/e3-cloud-types';
|
|
27
|
+
/** An input's path: `.inputs.<input>`. */
|
|
28
|
+
function inputPath(input) {
|
|
29
|
+
return [variant('field', 'inputs'), variant('field', input)];
|
|
30
|
+
}
|
|
31
|
+
const compareKeys = compareFor(StringType);
|
|
32
|
+
const decodeFolder = decodeBeast2For(FilesFolderType);
|
|
33
|
+
/** Entries in a Dict's order. */
|
|
34
|
+
function ordered(entries) {
|
|
35
|
+
return [...entries].map(([k, v]) => [k, v]).sort(([a], [b]) => compareKeys(a, b));
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The sources a folder's new value is stored from: the pieces no change
|
|
39
|
+
* touches carried over by reference, and each piece a change touches read,
|
|
40
|
+
* changed and cut again.
|
|
41
|
+
*/
|
|
42
|
+
async function folderSources(storage, repo, hash, changes) {
|
|
43
|
+
const pieces = await DatasetSegments.open(storage, repo, hash);
|
|
44
|
+
if (pieces.counts.length === 0)
|
|
45
|
+
return [{ elements: ordered(changes.set) }];
|
|
46
|
+
const touched = new Map();
|
|
47
|
+
const at = async (path) => {
|
|
48
|
+
const i = await pieces.segmentFor(path);
|
|
49
|
+
let change = touched.get(i);
|
|
50
|
+
if (change === undefined)
|
|
51
|
+
touched.set(i, change = { set: [], remove: [] });
|
|
52
|
+
return change;
|
|
53
|
+
};
|
|
54
|
+
for (const [path, bytes] of changes.set)
|
|
55
|
+
(await at(path)).set.push([path, bytes]);
|
|
56
|
+
for (const path of changes.remove)
|
|
57
|
+
(await at(path)).remove.push(path);
|
|
58
|
+
const sources = [];
|
|
59
|
+
let next = 0;
|
|
60
|
+
for (const i of [...touched.keys()].sort((a, b) => a - b)) {
|
|
61
|
+
if (i > next)
|
|
62
|
+
sources.push({ stored: pieces.hash, from: next, to: i });
|
|
63
|
+
const entries = new Map(decodeFolder(await pieces.segment(i)));
|
|
64
|
+
const change = touched.get(i);
|
|
65
|
+
for (const path of change.remove)
|
|
66
|
+
entries.delete(path);
|
|
67
|
+
for (const [path, bytes] of change.set)
|
|
68
|
+
entries.set(path, bytes);
|
|
69
|
+
sources.push({ elements: ordered(entries) });
|
|
70
|
+
next = i + 1;
|
|
71
|
+
}
|
|
72
|
+
if (next < pieces.counts.length)
|
|
73
|
+
sources.push({ stored: pieces.hash, from: next, to: pieces.counts.length });
|
|
74
|
+
return sources;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* The sync's writer over an e3 store, for one workspace.
|
|
78
|
+
*
|
|
79
|
+
* @param storage - The store: a local repository's (`LocalStorage`) or the platform's
|
|
80
|
+
* @param repo - The repository: a local repository's path, or the platform's repository name
|
|
81
|
+
* @param ws - The workspace
|
|
82
|
+
* @returns The writer
|
|
83
|
+
*/
|
|
84
|
+
export function storageWriter(storage, repo, ws) {
|
|
85
|
+
const current = async (input) => (await workspaceGetDatasetHash(storage, repo, ws, inputPath(input))).hash;
|
|
86
|
+
const written = async (input) => {
|
|
87
|
+
const hash = await current(input);
|
|
88
|
+
if (hash === null)
|
|
89
|
+
throw new Error(`input ${input} is unassigned after it was written`);
|
|
90
|
+
return hash;
|
|
91
|
+
};
|
|
92
|
+
return {
|
|
93
|
+
current,
|
|
94
|
+
async setFile(input, bytes) {
|
|
95
|
+
await workspaceSetDataset(storage, repo, ws, inputPath(input), bytes, FilesFileType);
|
|
96
|
+
return written(input);
|
|
97
|
+
},
|
|
98
|
+
async setListing(input, rows) {
|
|
99
|
+
const value = new Map([...rows].sort(([a], [b]) => compareKeys(a, b)));
|
|
100
|
+
await workspaceSetDataset(storage, repo, ws, inputPath(input), value, FilesListingType);
|
|
101
|
+
return written(input);
|
|
102
|
+
},
|
|
103
|
+
async updateFolder(input, changes) {
|
|
104
|
+
const hash = changes.fresh ? null : await current(input);
|
|
105
|
+
const sources = hash === null ? [{ elements: ordered(changes.set) }] : await folderSources(storage, repo, hash, changes);
|
|
106
|
+
const stored = await storeCollection(storage, repo, FilesFolderType, sources);
|
|
107
|
+
// The store knows the new folder as the manifest it is, so e3's door
|
|
108
|
+
// takes it in as the input's value, checked and versioned, reading
|
|
109
|
+
// nothing. A memo of the manifest to itself, until e3-core has a door
|
|
110
|
+
// for a value the store holds (east-workspace#1314); gc drops it once
|
|
111
|
+
// the manifest is collected
|
|
112
|
+
await storage.refs.adoptionWrite(repo, stored, stored);
|
|
113
|
+
const adopted = await datasetAdoptKnown(storage, repo, ws, inputPath(input), stored);
|
|
114
|
+
if (adopted === null)
|
|
115
|
+
throw new Error(`input ${input} could not take the folder ${stored}`);
|
|
116
|
+
return adopted.hash;
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACjF,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,uBAAuB,EACvB,mBAAmB,GAGpB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,gBAAgB,EAAwB,MAAM,yBAAyB,CAAC;AAGjH,0CAA0C;AAC1C,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,WAAW,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;AAC3C,MAAM,YAAY,GAAG,eAAe,CAAC,eAAe,CAAC,CAAC;AAEtD,iCAAiC;AACjC,SAAS,OAAO,CAAC,OAAgD;IAC/D,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAwB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAC1G,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,aAAa,CAAC,OAAuB,EAAE,IAAY,EAAE,IAAY,EAAE,OAAsB;IACtG,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC/D,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6D,CAAC;IACrF,MAAM,EAAE,GAAG,KAAK,EAAE,IAAY,EAAE,EAAE;QAChC,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3E,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,GAAG;QAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;IAClF,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,MAAM;QAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEtE,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,GAAG,IAAI;YAAE,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;QACvE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAqB,YAAY,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACvD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG;YAAE,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;IACD,IAAI,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7G,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,OAAuB,EAAE,IAAY,EAAE,EAAU;IAC7E,MAAM,OAAO,GAAG,KAAK,EAAE,KAAa,EAA0B,EAAE,CAC9D,CAAC,MAAM,uBAAuB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5E,MAAM,OAAO,GAAG,KAAK,EAAE,KAAa,EAAmB,EAAE;QACvD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,IAAI,KAAK,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,qCAAqC,CAAC,CAAC;QACxF,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,OAAO;QACL,OAAO;QACP,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK;YACxB,MAAM,mBAAmB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;YACrF,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,IAA0C;YAChE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YACvE,MAAM,mBAAmB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,gBAAgB,CAAC,CAAC;YACxF,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO;YAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,OAAO,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YACzH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YAC9E,qEAAqE;YACrE,mEAAmE;YACnE,sEAAsE;YACtE,sEAAsE;YACtE,4BAA4B;YAC5B,MAAM,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;YACrF,IAAI,OAAO,KAAK,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,8BAA8B,MAAM,EAAE,CAAC,CAAC;YAC5F,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
import { type FilesBinding, type FilesLedgerValue, type FilesListingRow, type FilesManifest, type FilesMapping, type FilesProblem, type FilesSyncReport } from '@elaraai/e3-cloud-types';
|
|
6
|
+
/** A file of a source, as a listing gives it. */
|
|
7
|
+
export interface FilesObject {
|
|
8
|
+
/** Its key, relative to where the workspace binds the source */
|
|
9
|
+
readonly key: string;
|
|
10
|
+
/** Its size in bytes */
|
|
11
|
+
readonly size: number;
|
|
12
|
+
/** When it was last written */
|
|
13
|
+
readonly modified: Date;
|
|
14
|
+
/** Its ETag, which changes whenever its bytes do */
|
|
15
|
+
readonly etag: string;
|
|
16
|
+
}
|
|
17
|
+
/** How a reader lists a prefix. */
|
|
18
|
+
export interface FilesListOptions {
|
|
19
|
+
/** Only the files in the prefix's own folder, none in a folder under it */
|
|
20
|
+
readonly topFolderOnly?: boolean;
|
|
21
|
+
}
|
|
22
|
+
/** Reads one bound source: a bucket, under the prefix the binding names. */
|
|
23
|
+
export interface FilesReader {
|
|
24
|
+
/** Every file under a prefix (relative to the binding), in key order */
|
|
25
|
+
list(prefix: string, options?: FilesListOptions): AsyncIterable<FilesObject>;
|
|
26
|
+
/** One file's listing, or `null` when there is none */
|
|
27
|
+
stat(key: string): Promise<FilesObject | null>;
|
|
28
|
+
/**
|
|
29
|
+
* One file's bytes, as its listing gave it: a file that changed since, or
|
|
30
|
+
* that the reader cannot read, is refused with a {@link FilesUnreadableError}
|
|
31
|
+
*/
|
|
32
|
+
read(file: FilesObject): Promise<Uint8Array>;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* A file a reader cannot read, which a sync refuses and names in its report,
|
|
36
|
+
* going on with the rest; the next sync tries it again. A reader throws it
|
|
37
|
+
* for what is the file's own, never for its source as a whole: a file
|
|
38
|
+
* archived, changed or gone since it was listed, or closed to the reader's
|
|
39
|
+
* access. Any other error ends the round, which is tried again.
|
|
40
|
+
*/
|
|
41
|
+
export declare class FilesUnreadableError extends Error {
|
|
42
|
+
readonly key: string;
|
|
43
|
+
readonly reason: string;
|
|
44
|
+
/**
|
|
45
|
+
* @param key - The file's key, relative to the binding
|
|
46
|
+
* @param reason - Why, completing "'<key>' …"
|
|
47
|
+
*/
|
|
48
|
+
constructor(key: string, reason: string);
|
|
49
|
+
}
|
|
50
|
+
/** Writes one workspace's inputs. Each write answers the value it set: its hash. */
|
|
51
|
+
export interface FilesWriter {
|
|
52
|
+
/** The value an input holds now, its hash; `null` while it is unassigned */
|
|
53
|
+
current(input: string): Promise<string | null>;
|
|
54
|
+
/** Sets a `file` or `latest` input to a file's bytes */
|
|
55
|
+
setFile(input: string, bytes: Uint8Array): Promise<string>;
|
|
56
|
+
/**
|
|
57
|
+
* Updates a `folder` input: each file in `set` added or replaced, each path
|
|
58
|
+
* in `remove` taken out, every other entry as it was; or, `fresh`, the
|
|
59
|
+
* folder made of `set` alone
|
|
60
|
+
*/
|
|
61
|
+
updateFolder(input: string, changes: FolderChanges): Promise<string>;
|
|
62
|
+
/** Sets a `listing` input */
|
|
63
|
+
setListing(input: string, rows: ReadonlyMap<string, FilesListingRow>): Promise<string>;
|
|
64
|
+
}
|
|
65
|
+
/** What a sync changes in a `folder` input. */
|
|
66
|
+
export interface FolderChanges {
|
|
67
|
+
/** Each file added or replaced, by its path relative to the prefix */
|
|
68
|
+
readonly set: ReadonlyMap<string, Uint8Array>;
|
|
69
|
+
/** Each path taken out */
|
|
70
|
+
readonly remove: ReadonlySet<string>;
|
|
71
|
+
/** Whether the folder is `set` alone, whatever the input holds now */
|
|
72
|
+
readonly fresh: boolean;
|
|
73
|
+
}
|
|
74
|
+
/** What an input took of one file. */
|
|
75
|
+
export interface LedgerEntry {
|
|
76
|
+
readonly etag: string;
|
|
77
|
+
readonly size: number;
|
|
78
|
+
}
|
|
79
|
+
/** What an input took last: each file, the value the sync wrote, and what it was taken under. */
|
|
80
|
+
export interface LedgerState {
|
|
81
|
+
/** Each file it took, by key */
|
|
82
|
+
readonly entries: ReadonlyMap<string, LedgerEntry>;
|
|
83
|
+
/** The value the sync wrote, its hash; `null` before its first sync */
|
|
84
|
+
readonly value: string | null;
|
|
85
|
+
/**
|
|
86
|
+
* The mapping's shape and its source's binding it was taken under, as one
|
|
87
|
+
* text ({@link takenUnder}); `null` before its first sync
|
|
88
|
+
*/
|
|
89
|
+
readonly under: string | null;
|
|
90
|
+
}
|
|
91
|
+
/** What an input took before its first sync: nothing. */
|
|
92
|
+
export declare const NO_LEDGER: LedgerState;
|
|
93
|
+
/**
|
|
94
|
+
* What an input took, as a store keeps it (e3-cloud-types' `FilesLedgerType`).
|
|
95
|
+
*
|
|
96
|
+
* @param state - What it took
|
|
97
|
+
* @returns Its East value
|
|
98
|
+
*/
|
|
99
|
+
export declare function ledgerValueOf(state: LedgerState): FilesLedgerValue;
|
|
100
|
+
/**
|
|
101
|
+
* What an input took, from what a store keeps.
|
|
102
|
+
*
|
|
103
|
+
* @param value - Its East value
|
|
104
|
+
* @returns What it took
|
|
105
|
+
*/
|
|
106
|
+
export declare function ledgerStateOf(value: FilesLedgerValue): LedgerState;
|
|
107
|
+
/**
|
|
108
|
+
* What an input's files are taken under: its mapping's shape and where its
|
|
109
|
+
* source is bound, as one text. A ledger taken under another is not trusted:
|
|
110
|
+
* its keys may name other files, and its paths another folder's.
|
|
111
|
+
*
|
|
112
|
+
* @param mapping - The mapping
|
|
113
|
+
* @param binding - Its source's binding
|
|
114
|
+
* @returns The text
|
|
115
|
+
*/
|
|
116
|
+
export declare function takenUnder(mapping: FilesMapping, binding: FilesBinding): string;
|
|
117
|
+
/** Remembers, for each input, what it took last. */
|
|
118
|
+
export interface FilesLedger {
|
|
119
|
+
/** What an input took last; nothing before its first sync */
|
|
120
|
+
read(input: string): Promise<LedgerState>;
|
|
121
|
+
/** Remembers what an input took */
|
|
122
|
+
write(input: string, state: LedgerState): Promise<void>;
|
|
123
|
+
}
|
|
124
|
+
/** What a sync is given. */
|
|
125
|
+
export interface SyncOptions {
|
|
126
|
+
/** The declarations */
|
|
127
|
+
readonly manifest: FilesManifest;
|
|
128
|
+
/** Where each source's files are */
|
|
129
|
+
readonly bindings: readonly FilesBinding[];
|
|
130
|
+
/** Reads a bound source */
|
|
131
|
+
readonly reader: (binding: FilesBinding) => FilesReader;
|
|
132
|
+
/** Writes the workspace's inputs */
|
|
133
|
+
readonly writer: FilesWriter;
|
|
134
|
+
/** Remembers what each input took */
|
|
135
|
+
readonly ledger: FilesLedger;
|
|
136
|
+
/** List what a sync would take, and take nothing */
|
|
137
|
+
readonly dryRun?: boolean;
|
|
138
|
+
/** Only these inputs; every mapping when absent */
|
|
139
|
+
readonly inputs?: readonly string[];
|
|
140
|
+
/** The time, for the report */
|
|
141
|
+
readonly now?: () => Date;
|
|
142
|
+
/** The most bytes of a folder's files held before they are written: {@link FOLDER_BATCH_BYTES} when absent */
|
|
143
|
+
readonly batchBytes?: number;
|
|
144
|
+
}
|
|
145
|
+
/** What one round of a sync is given: a sync's options, and when to stop. */
|
|
146
|
+
export interface SyncRoundOptions extends SyncOptions {
|
|
147
|
+
/**
|
|
148
|
+
* Asked between inputs and between a folder's batches, once the round has
|
|
149
|
+
* written something: `true` stops the round there, every input as it was
|
|
150
|
+
* last written and remembered
|
|
151
|
+
*/
|
|
152
|
+
readonly shouldYield?: () => boolean;
|
|
153
|
+
}
|
|
154
|
+
/** What a round did, and whether it stopped before the end. */
|
|
155
|
+
export interface FilesSyncRound {
|
|
156
|
+
/** What it did: each input it reached, with what it took */
|
|
157
|
+
readonly report: FilesSyncReport;
|
|
158
|
+
/** Whether it stopped when asked, with inputs or a folder's files left for the next */
|
|
159
|
+
readonly yielded: boolean;
|
|
160
|
+
}
|
|
161
|
+
/** The most bytes of a folder's files a sync holds before it writes them, when not told: 256 MiB. */
|
|
162
|
+
export declare const FOLDER_BATCH_BYTES: number;
|
|
163
|
+
/** Thrown when a sync cannot start, naming every problem. */
|
|
164
|
+
export declare class FilesSyncError extends Error {
|
|
165
|
+
readonly problems: readonly FilesProblem[];
|
|
166
|
+
constructor(problems: readonly FilesProblem[]);
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Brings each mapping's input up to date with its source's files, stopping
|
|
170
|
+
* when asked between inputs or between a folder's batches: one round of a
|
|
171
|
+
* sync that hands over, as the platform's do. What it stopped before, the
|
|
172
|
+
* next round takes.
|
|
173
|
+
*
|
|
174
|
+
* @param options - The declarations, the bindings, what reads, writes and remembers, and when to stop
|
|
175
|
+
* @returns What it did, per input it reached, and whether it stopped before the end
|
|
176
|
+
* @throws {FilesSyncError} When a mapping's source is not bound, before
|
|
177
|
+
* anything is read
|
|
178
|
+
* @throws {Error} When an input named is not one a declaration fills
|
|
179
|
+
*/
|
|
180
|
+
export declare function syncFilesRound(options: SyncRoundOptions): Promise<FilesSyncRound>;
|
|
181
|
+
/**
|
|
182
|
+
* Brings each mapping's input up to date with its source's files, to the end.
|
|
183
|
+
*
|
|
184
|
+
* @param options - The declarations, the bindings, and what reads, writes and remembers
|
|
185
|
+
* @returns What it did, per mapping
|
|
186
|
+
* @throws {FilesSyncError} When a mapping's source is not bound, before
|
|
187
|
+
* anything is read
|
|
188
|
+
* @throws {Error} When an input named is not one a declaration fills
|
|
189
|
+
*/
|
|
190
|
+
export declare function syncFiles(options: SyncOptions): Promise<FilesSyncReport>;
|
|
191
|
+
//# sourceMappingURL=engine.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../../src/sync/engine.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA0CH,OAAO,EAEL,KAAK,YAAY,EAAE,KAAK,gBAAgB,EAAE,KAAK,eAAe,EAAE,KAAK,aAAa,EAAE,KAAK,YAAY,EACrG,KAAK,YAAY,EAAqB,KAAK,eAAe,EAC3D,MAAM,yBAAyB,CAAC;AAGjC,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,gEAAgE;IAChE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,wBAAwB;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,+BAA+B;IAC/B,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC;IACxB,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,mCAAmC;AACnC,MAAM,WAAW,gBAAgB;IAC/B,2EAA2E;IAC3E,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,4EAA4E;AAC5E,MAAM,WAAW,WAAW;IAC1B,wEAAwE;IACxE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC;IAC7E,uDAAuD;IACvD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;IAC/C;;;OAGG;IACH,IAAI,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CAC9C;AAED;;;;;;GAMG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;IAKjC,QAAQ,CAAC,GAAG,EAAE,MAAM;IAAE,QAAQ,CAAC,MAAM,EAAE,MAAM;IAJzD;;;OAGG;gBACkB,GAAG,EAAE,MAAM,EAAW,MAAM,EAAE,MAAM;CAI1D;AAED,oFAAoF;AACpF,MAAM,WAAW,WAAW;IAC1B,4EAA4E;IAC5E,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC/C,wDAAwD;IACxD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3D;;;;OAIG;IACH,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,6BAA6B;IAC7B,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACxF;AAED,+CAA+C;AAC/C,MAAM,WAAW,aAAa;IAC5B,sEAAsE;IACtE,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC9C,0BAA0B;IAC1B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACrC,sEAAsE;IACtE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;CACzB;AAED,sCAAsC;AACtC,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,iGAAiG;AACjG,MAAM,WAAW,WAAW;IAC1B,gCAAgC;IAChC,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,uEAAuE;IACvE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,yDAAyD;AACzD,eAAO,MAAM,SAAS,EAAE,WAA8D,CAAC;AAEvF;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,gBAAgB,CAMlE;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,gBAAgB,GAAG,WAAW,CAMlE;AAID;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,GAAG,MAAM,CAE/E;AAED,oDAAoD;AACpD,MAAM,WAAW,WAAW;IAC1B,6DAA6D;IAC7D,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAC1C,mCAAmC;IACnC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzD;AAED,4BAA4B;AAC5B,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,oCAAoC;IACpC,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C,2BAA2B;IAC3B,QAAQ,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,WAAW,CAAC;IACxD,oCAAoC;IACpC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,qCAAqC;IACrC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,oDAAoD;IACpD,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,mDAAmD;IACnD,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,+BAA+B;IAC/B,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B,8GAA8G;IAC9G,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,6EAA6E;AAC7E,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;;OAIG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,OAAO,CAAC;CACtC;AAED,+DAA+D;AAC/D,MAAM,WAAW,cAAc;IAC7B,4DAA4D;IAC5D,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;IACjC,uFAAuF;IACvF,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,qGAAqG;AACrG,eAAO,MAAM,kBAAkB,QAAoB,CAAC;AAKpD,6DAA6D;AAC7D,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;gBAE/B,QAAQ,EAAE,SAAS,YAAY,EAAE;CAK9C;AA+TD;;;;;;;;;;;GAWG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,cAAc,CAAC,CAqDvF;AAED;;;;;;;;GAQG;AACH,wBAAsB,SAAS,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,CAAC,CAE9E"}
|