@elaraai/e3-cloud-cli 1.0.97 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -7
- package/dist/src/cli.js +22 -4
- package/dist/src/cli.js.map +1 -1
- package/dist/src/commands/app.d.ts.map +1 -1
- package/dist/src/commands/app.js +3 -7
- package/dist/src/commands/app.js.map +1 -1
- package/dist/src/commands/feedback.d.ts.map +1 -1
- package/dist/src/commands/feedback.js +4 -8
- package/dist/src/commands/feedback.js.map +1 -1
- package/dist/src/commands/files.d.ts +8 -1
- package/dist/src/commands/files.d.ts.map +1 -1
- package/dist/src/commands/files.js +47 -35
- package/dist/src/commands/files.js.map +1 -1
- package/dist/src/commands/records.d.ts +9 -0
- package/dist/src/commands/records.d.ts.map +1 -1
- package/dist/src/commands/records.js +18 -6
- package/dist/src/commands/records.js.map +1 -1
- package/dist/src/commands/release.d.ts.map +1 -1
- package/dist/src/commands/release.js +6 -21
- package/dist/src/commands/release.js.map +1 -1
- package/dist/src/commands/schedule.js +2 -2
- package/dist/src/commands/schedule.js.map +1 -1
- package/dist/src/commands/secrets.js +2 -2
- package/dist/src/commands/secrets.js.map +1 -1
- package/dist/src/commands/user.js +7 -7
- package/dist/src/commands/user.js.map +1 -1
- package/dist/src/commands/whoami.d.ts +18 -0
- package/dist/src/commands/whoami.d.ts.map +1 -1
- package/dist/src/commands/whoami.js +29 -5
- package/dist/src/commands/whoami.js.map +1 -1
- package/dist/src/dataset-source.d.ts +102 -0
- package/dist/src/dataset-source.d.ts.map +1 -0
- package/dist/src/dataset-source.js +412 -0
- package/dist/src/dataset-source.js.map +1 -0
- package/dist/src/files-local.d.ts +12 -17
- package/dist/src/files-local.d.ts.map +1 -1
- package/dist/src/files-local.js +35 -11
- package/dist/src/files-local.js.map +1 -1
- package/dist/src/files-push.d.ts +129 -31
- package/dist/src/files-push.d.ts.map +1 -1
- package/dist/src/files-push.js +297 -139
- package/dist/src/files-push.js.map +1 -1
- package/dist/src/utils.d.ts +5 -0
- package/dist/src/utils.d.ts.map +1 -1
- package/dist/src/utils.js +6 -0
- package/dist/src/utils.js.map +1 -1
- package/dist/src/versions.d.ts +36 -0
- package/dist/src/versions.d.ts.map +1 -0
- package/dist/src/versions.js +83 -0
- package/dist/src/versions.js.map +1 -0
- package/package.json +9 -9
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* `dataset:<ws>.<name>`: a workspace's dataset offered to `e3-cloud files
|
|
7
|
+
* push` as the file of its value.
|
|
8
|
+
*
|
|
9
|
+
* The dataset is read from the repository `--from` names, a local
|
|
10
|
+
* repository's path or a repository's URL, defaulting as e3's `[repo]` does
|
|
11
|
+
* (`$E3_REPO`, then `.`). Its name resolves as e3's CLI resolves a flat path
|
|
12
|
+
* (e3-cli's `path-resolver`): the input, the task (its output) or the record
|
|
13
|
+
* of that name in the workspace. One that is unassigned (a task output not
|
|
14
|
+
* produced since its workspace's last deploy, an input never set, a record
|
|
15
|
+
* with no state yet), that the workspace does not hold, or that names two of
|
|
16
|
+
* them, is refused in its own row.
|
|
17
|
+
*
|
|
18
|
+
* Its bytes are its value as a deployed repository serves `e3 dataset get -f
|
|
19
|
+
* beast2`: a collection spliced from its stored segments a segment at a time
|
|
20
|
+
* (e3-core's `DatasetSegments.splice`, as the server splices them), any other
|
|
21
|
+
* value as it is stored, never held whole; so the same stored value gives the
|
|
22
|
+
* same bytes from a local repository and a deployed one, and a dataset pushed
|
|
23
|
+
* again unchanged is `unchanged`. They are read twice: once for their size,
|
|
24
|
+
* SHA-256 and header type, which the plan is given, then again as they are
|
|
25
|
+
* uploaded. Each read is pinned to the value first read (a local repository's
|
|
26
|
+
* by its hash, which names one value for ever; a deployed one's refused when
|
|
27
|
+
* the dataset names another) and its SHA-256 is checked as it ends, so a
|
|
28
|
+
* dataset a run writes again in between is refused ({@link DatasetMovedError}),
|
|
29
|
+
* never put as something else. A value of 8 MiB or less is kept from the first
|
|
30
|
+
* read and read once.
|
|
31
|
+
*/
|
|
32
|
+
import { createHash } from 'node:crypto';
|
|
33
|
+
import { encodeBeast2For, readBeast2Type, variant } from '@elaraai/east';
|
|
34
|
+
import { DatasetSegments, InvalidNameError, LocalStorage, WorkspaceNotDeployedError, WorkspaceNotFoundError, openDatasetObject, workspaceGetTree, } from '@elaraai/e3-core';
|
|
35
|
+
import { ApiError, DatasetHashMismatchError, datasetGetStatus, datasetGetStream, datasetListRecursive } from '@elaraai/e3-api-client';
|
|
36
|
+
import { parseDatasetPath } from '@elaraai/e3-types';
|
|
37
|
+
import { filesDatasetPathText } from '@elaraai/e3-cloud-types';
|
|
38
|
+
import { getValidToken } from './credentials.js';
|
|
39
|
+
import { openLocalRepository } from './files-local.js';
|
|
40
|
+
import { parseRepoUrl } from './utils.js';
|
|
41
|
+
/** What names a dataset among a push's paths. */
|
|
42
|
+
export const DATASET_ARG_PREFIX = 'dataset:';
|
|
43
|
+
/** A value this size or smaller is kept from its first read, and read once. */
|
|
44
|
+
const KEPT_BYTES = 8 * 1024 * 1024;
|
|
45
|
+
/** How much of a stored object one read takes, for a value that is no collection. */
|
|
46
|
+
const READ_BYTES = 8 * 1024 * 1024;
|
|
47
|
+
/** The segments a local read of a collection reads ahead of the upload. */
|
|
48
|
+
const READ_AHEAD = 2;
|
|
49
|
+
/** The most of a value's head read for its type: a type section wider is malformed, not large. */
|
|
50
|
+
const HEAD_TYPE_MAX_BYTES = 16 * 1024 * 1024;
|
|
51
|
+
/**
|
|
52
|
+
* The repository a push reads its datasets from: `--from`, else `$E3_REPO`,
|
|
53
|
+
* else the current directory, as e3's `[repo]` defaults.
|
|
54
|
+
*
|
|
55
|
+
* @param from - `--from`, as given
|
|
56
|
+
* @returns The repository
|
|
57
|
+
*/
|
|
58
|
+
export function datasetRepositoryArg(from) {
|
|
59
|
+
return from !== undefined && from.length > 0 ? from : (process.env.E3_REPO ?? '.');
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* The dataset a push's path names, or null for a path that names none.
|
|
63
|
+
*
|
|
64
|
+
* @param arg - The path, as given
|
|
65
|
+
* @returns The dataset, or null
|
|
66
|
+
* @throws {Error} For `dataset:` followed by anything but `<ws>.<name>`
|
|
67
|
+
*/
|
|
68
|
+
export function datasetArgOf(arg) {
|
|
69
|
+
if (!arg.startsWith(DATASET_ARG_PREFIX))
|
|
70
|
+
return null;
|
|
71
|
+
const spec = arg.slice(DATASET_ARG_PREFIX.length);
|
|
72
|
+
const shape = 'name a dataset as dataset:<workspace>.<name>, the name of its input, task or record (dataset:dev.customer_book)';
|
|
73
|
+
let parsed;
|
|
74
|
+
try {
|
|
75
|
+
parsed = parseDatasetPath(spec);
|
|
76
|
+
}
|
|
77
|
+
catch (err) {
|
|
78
|
+
throw new Error(`${arg}: ${err instanceof Error ? err.message : String(err)}: ${shape}`);
|
|
79
|
+
}
|
|
80
|
+
const [name, ...rest] = parsed.path;
|
|
81
|
+
if (name === undefined || rest.length > 0)
|
|
82
|
+
throw new Error(`${arg}: ${shape}`);
|
|
83
|
+
return { arg, workspace: parsed.ws, name: name.value };
|
|
84
|
+
}
|
|
85
|
+
/** A dataset that no longer holds the value a push read: a run wrote it again. */
|
|
86
|
+
export class DatasetMovedError extends Error {
|
|
87
|
+
constructor(arg) {
|
|
88
|
+
super(`${arg} changed while it was pushed (a run wrote it again): push it again`);
|
|
89
|
+
this.name = 'DatasetMovedError';
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
/** A tree path of fields. */
|
|
93
|
+
function fields(...names) {
|
|
94
|
+
return names.map((name) => variant('field', name));
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* A dataset's entry by where its workspace's tree holds it, as e3's CLI
|
|
98
|
+
* names it: `inputs/<name>`, `tasks/<name>` (its output: a tree names a task's
|
|
99
|
+
* output by the task, a listing that walks into the task by `output` under
|
|
100
|
+
* it) or `records/<name>`; null for anything else, which no flat name of an
|
|
101
|
+
* input, task or record reaches.
|
|
102
|
+
*/
|
|
103
|
+
function entryOf(prefix, name) {
|
|
104
|
+
if (prefix.length === 1 && prefix[0] === 'inputs')
|
|
105
|
+
return { name, kind: 'input', path: fields('inputs', name) };
|
|
106
|
+
if (prefix.length === 1 && prefix[0] === 'records')
|
|
107
|
+
return { name, kind: 'record', path: fields('records', name) };
|
|
108
|
+
if (prefix.length === 1 && prefix[0] === 'tasks')
|
|
109
|
+
return { name, kind: 'task output', path: fields('tasks', name, 'output') };
|
|
110
|
+
if (prefix.length === 2 && prefix[0] === 'tasks' && name === 'output') {
|
|
111
|
+
return { name: prefix[1], kind: 'task output', path: fields('tasks', prefix[1], 'output') };
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
/** The one entry a name names, or why there is none. */
|
|
116
|
+
function entryNamed(index, arg) {
|
|
117
|
+
const matches = index.filter((entry) => entry.name === arg.name);
|
|
118
|
+
const path = filesDatasetPathText(arg.workspace, arg.name);
|
|
119
|
+
if (matches.length === 1)
|
|
120
|
+
return matches[0];
|
|
121
|
+
if (matches.length > 1) {
|
|
122
|
+
return { refused: 'ambiguous', message: `${path} is ambiguous: ${arg.workspace} has ${matches.map((m) => `a ${m.kind}`).join(' and ')} named ${arg.name}` };
|
|
123
|
+
}
|
|
124
|
+
return { refused: 'not_found', message: `${arg.workspace} has no input, task or record named ${arg.name}` };
|
|
125
|
+
}
|
|
126
|
+
/** Why an unassigned dataset holds nothing, as its row says. */
|
|
127
|
+
function unassignedWords(arg, kind) {
|
|
128
|
+
const path = filesDatasetPathText(arg.workspace, arg.name);
|
|
129
|
+
switch (kind) {
|
|
130
|
+
case 'task output': return `${path} is unassigned: its task has not produced it since ${arg.workspace}'s last deploy (run its dataflow)`;
|
|
131
|
+
case 'input': return `${path} is unassigned: no value has been set`;
|
|
132
|
+
case 'record': return `${path} is unassigned: the record has no state yet`;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
/** A dataset that holds null, which e3 stores as no object: its type's encoding of null, and that encoding's hash. */
|
|
136
|
+
function nullValue(arg, kind, type) {
|
|
137
|
+
const bytes = encodeBeast2For(type)(null);
|
|
138
|
+
return { arg, kind, hash: createHash('sha256').update(bytes).digest('hex'), open: () => Promise.resolve(once(bytes)) };
|
|
139
|
+
}
|
|
140
|
+
/** Bytes as one chunk. */
|
|
141
|
+
// eslint-disable-next-line @typescript-eslint/require-await -- bytes in memory, as the async iterable a dataset's bytes are
|
|
142
|
+
async function* once(bytes) {
|
|
143
|
+
yield bytes;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Where a push reads its datasets: a deployed repository given its URL, a
|
|
147
|
+
* local one given its path.
|
|
148
|
+
*
|
|
149
|
+
* @param from - The repository ({@link datasetRepositoryArg})
|
|
150
|
+
* @returns The repository
|
|
151
|
+
* @throws {Error} When a local path holds no repository this e3 can open, or a URL names none
|
|
152
|
+
*/
|
|
153
|
+
export async function datasetRepository(from) {
|
|
154
|
+
return from.startsWith('https://') || from.startsWith('http://') ? remoteRepository(from) : localRepository(await openLocalRepository(from));
|
|
155
|
+
}
|
|
156
|
+
/** The entries of a local workspace's tree, each with its leaf. */
|
|
157
|
+
function localEntries(nodes, prefix = []) {
|
|
158
|
+
return nodes.flatMap((node) => {
|
|
159
|
+
if (node.kind === 'tree')
|
|
160
|
+
return localEntries(node.children, [...prefix, node.name]);
|
|
161
|
+
const entry = entryOf(prefix, node.name);
|
|
162
|
+
return entry === null ? [] : [{ ...entry, leaf: node }];
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
/** A local repository's datasets, read through e3-core. */
|
|
166
|
+
function localRepository(repo) {
|
|
167
|
+
const storage = new LocalStorage();
|
|
168
|
+
const trees = new Map();
|
|
169
|
+
const treeOf = (ws) => {
|
|
170
|
+
let tree = trees.get(ws);
|
|
171
|
+
if (tree === undefined)
|
|
172
|
+
trees.set(ws, tree = workspaceGetTree(storage, repo, ws, [], { includeTypes: true, includeStatus: true }));
|
|
173
|
+
return tree;
|
|
174
|
+
};
|
|
175
|
+
return {
|
|
176
|
+
async find(args) {
|
|
177
|
+
const lookups = [];
|
|
178
|
+
for (const arg of args) {
|
|
179
|
+
let tree;
|
|
180
|
+
try {
|
|
181
|
+
tree = await treeOf(arg.workspace);
|
|
182
|
+
}
|
|
183
|
+
catch (err) {
|
|
184
|
+
if (!(err instanceof WorkspaceNotFoundError || err instanceof WorkspaceNotDeployedError || err instanceof InvalidNameError))
|
|
185
|
+
throw err;
|
|
186
|
+
lookups.push({ refused: 'not_found', message: err.message });
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
const entry = entryNamed(localEntries(tree), arg);
|
|
190
|
+
if ('refused' in entry) {
|
|
191
|
+
lookups.push(entry);
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
const { leaf, kind } = entry;
|
|
195
|
+
if (leaf.refType === 'unassigned') {
|
|
196
|
+
lookups.push({ refused: 'unassigned', message: unassignedWords(arg, kind) });
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
if (leaf.refType === 'null' || leaf.hash === undefined) {
|
|
200
|
+
lookups.push({ found: nullValue(arg, kind, leaf.datasetType) });
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
const hash = leaf.hash;
|
|
204
|
+
// Pinned by its hash: what a repository holds under a hash never changes
|
|
205
|
+
lookups.push({ found: { arg, kind, hash, open: () => Promise.resolve(localBytes(storage, repo, hash)) } });
|
|
206
|
+
}
|
|
207
|
+
return lookups;
|
|
208
|
+
},
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* A stored value's bytes: a collection spliced from its segments (a record
|
|
213
|
+
* that declares an index read through its primary, as every reader reads
|
|
214
|
+
* it), any other value as stored, by ranges.
|
|
215
|
+
*/
|
|
216
|
+
async function* localBytes(storage, repo, hash) {
|
|
217
|
+
const opened = await openDatasetObject(storage, repo, hash);
|
|
218
|
+
if (opened.manifest !== null) {
|
|
219
|
+
yield* (await DatasetSegments.open(storage, repo, opened.hash)).splice({ readAhead: READ_AHEAD });
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
const { size } = await storage.objects.stat(repo, opened.hash);
|
|
223
|
+
for (let at = 0; at < size; at += READ_BYTES)
|
|
224
|
+
yield await storage.objects.readRange(repo, opened.hash, at, Math.min(READ_BYTES, size - at));
|
|
225
|
+
}
|
|
226
|
+
/** A deployed repository's datasets, read through its API with the login e3 and e3-cloud share. */
|
|
227
|
+
function remoteRepository(from) {
|
|
228
|
+
const { baseUrl, repo } = parseRepoUrl(from);
|
|
229
|
+
const options = async () => ({ token: await getValidToken(baseUrl) });
|
|
230
|
+
const indexes = new Map();
|
|
231
|
+
const indexOf = (ws) => {
|
|
232
|
+
let index = indexes.get(ws);
|
|
233
|
+
if (index === undefined) {
|
|
234
|
+
indexes.set(ws, index = (async () => {
|
|
235
|
+
const items = await datasetListRecursive(baseUrl, repo, ws, [], await options());
|
|
236
|
+
return items.flatMap((item) => {
|
|
237
|
+
if (item.type !== 'dataset')
|
|
238
|
+
return [];
|
|
239
|
+
// A listing's path is a keypath, `.inputs.customers`, as e3-cli's resolver splits it
|
|
240
|
+
const segments = item.value.path.split('.').filter((s) => s.length > 0);
|
|
241
|
+
const name = segments.at(-1);
|
|
242
|
+
const entry = name === undefined ? null : entryOf(segments.slice(0, -1), name);
|
|
243
|
+
return entry === null ? [] : [entry];
|
|
244
|
+
});
|
|
245
|
+
})());
|
|
246
|
+
}
|
|
247
|
+
return index;
|
|
248
|
+
};
|
|
249
|
+
return {
|
|
250
|
+
async find(args) {
|
|
251
|
+
const lookups = [];
|
|
252
|
+
for (const arg of args) {
|
|
253
|
+
let index;
|
|
254
|
+
try {
|
|
255
|
+
index = await indexOf(arg.workspace);
|
|
256
|
+
}
|
|
257
|
+
catch (err) {
|
|
258
|
+
if (!(err instanceof ApiError) || !['workspace_not_found', 'workspace_not_deployed', 'invalid_name'].includes(err.code))
|
|
259
|
+
throw err;
|
|
260
|
+
lookups.push({ refused: 'not_found', message: `${arg.workspace}: ${err.code.replace(/_/g, ' ')}` });
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
const entry = entryNamed(index, arg);
|
|
264
|
+
if ('refused' in entry) {
|
|
265
|
+
lookups.push(entry);
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
const { kind, path } = entry;
|
|
269
|
+
const status = await datasetGetStatus(baseUrl, repo, arg.workspace, path, await options());
|
|
270
|
+
if (status.refType === 'unassigned') {
|
|
271
|
+
lookups.push({ refused: 'unassigned', message: unassignedWords(arg, kind) });
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
274
|
+
if (status.refType === 'null' || status.hash.type === 'none') {
|
|
275
|
+
lookups.push({ found: nullValue(arg, kind, status.type) });
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
278
|
+
const hash = status.hash.value;
|
|
279
|
+
lookups.push({
|
|
280
|
+
found: {
|
|
281
|
+
arg,
|
|
282
|
+
kind,
|
|
283
|
+
hash,
|
|
284
|
+
open: async () => {
|
|
285
|
+
const read = await datasetGetStream(baseUrl, repo, arg.workspace, path, await options());
|
|
286
|
+
if (read.hash !== hash) {
|
|
287
|
+
// Moved since it was measured: its bytes are another value's
|
|
288
|
+
await read.chunks.return?.(undefined);
|
|
289
|
+
throw new DatasetMovedError(arg.arg);
|
|
290
|
+
}
|
|
291
|
+
return movesRaised(read.chunks, arg.arg);
|
|
292
|
+
},
|
|
293
|
+
},
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
return lookups;
|
|
297
|
+
},
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
/** A remote read's chunks, a move the client finds while they stream raised as the push's own. */
|
|
301
|
+
async function* movesRaised(chunks, arg) {
|
|
302
|
+
try {
|
|
303
|
+
yield* chunks;
|
|
304
|
+
}
|
|
305
|
+
catch (err) {
|
|
306
|
+
if (err instanceof DatasetHashMismatchError)
|
|
307
|
+
throw new DatasetMovedError(arg);
|
|
308
|
+
throw err;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
/** Bytes, one after another, in one new array. */
|
|
312
|
+
function joined(parts) {
|
|
313
|
+
const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
|
|
314
|
+
let at = 0;
|
|
315
|
+
for (const part of parts) {
|
|
316
|
+
out.set(part, at);
|
|
317
|
+
at += part.length;
|
|
318
|
+
}
|
|
319
|
+
return out;
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* A dataset's value read once for its size, its SHA-256 and the type its
|
|
323
|
+
* header declares, the bytes kept when they are few; and its bytes for the
|
|
324
|
+
* upload, read again pinned to the same value and checked as they end.
|
|
325
|
+
*
|
|
326
|
+
* @param found - The dataset
|
|
327
|
+
* @returns The measures and the bytes
|
|
328
|
+
* @throws {DatasetMovedError} When it moves while it is read
|
|
329
|
+
* @throws {Error} When its bytes are no beast2, which a stored value always is
|
|
330
|
+
*/
|
|
331
|
+
export async function measuredDataset(found) {
|
|
332
|
+
const digest = createHash('sha256');
|
|
333
|
+
let size = 0;
|
|
334
|
+
let kept = [];
|
|
335
|
+
// The head, until its type section is in hand: a short head fails to read as
|
|
336
|
+
// a malformed one does, so it is read again as more arrives
|
|
337
|
+
let head = [];
|
|
338
|
+
let headBytes = 0;
|
|
339
|
+
let type = null;
|
|
340
|
+
let headError = null;
|
|
341
|
+
for await (const chunk of await found.open()) {
|
|
342
|
+
digest.update(chunk);
|
|
343
|
+
size += chunk.length;
|
|
344
|
+
if (kept !== null) {
|
|
345
|
+
kept.push(chunk);
|
|
346
|
+
if (size > KEPT_BYTES)
|
|
347
|
+
kept = null;
|
|
348
|
+
}
|
|
349
|
+
if (type === null && headError === null) {
|
|
350
|
+
head.push(chunk);
|
|
351
|
+
headBytes += chunk.length;
|
|
352
|
+
try {
|
|
353
|
+
type = readBeast2Type(joined(head));
|
|
354
|
+
head = [];
|
|
355
|
+
}
|
|
356
|
+
catch (err) {
|
|
357
|
+
if (headBytes >= HEAD_TYPE_MAX_BYTES) {
|
|
358
|
+
headError = err;
|
|
359
|
+
head = [];
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
if (type === null) {
|
|
365
|
+
try {
|
|
366
|
+
if (headError !== null)
|
|
367
|
+
throw headError;
|
|
368
|
+
type = readBeast2Type(joined(head));
|
|
369
|
+
}
|
|
370
|
+
catch (err) {
|
|
371
|
+
throw new Error(`${found.arg.arg} holds no beast2 value: ${err instanceof Error ? err.message : String(err)}`);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
const sha256 = digest.digest('hex');
|
|
375
|
+
const whole = kept;
|
|
376
|
+
const bytes = {
|
|
377
|
+
size,
|
|
378
|
+
ranged: whole !== null,
|
|
379
|
+
read: whole !== null ? (start) => slicesFrom(whole, start) : (start) => checkedFrom(found, start, size, sha256),
|
|
380
|
+
};
|
|
381
|
+
return { size, sha256, type, origin: { path: filesDatasetPathText(found.arg.workspace, found.arg.name), hash: found.hash }, bytes };
|
|
382
|
+
}
|
|
383
|
+
/** Kept bytes from an offset on. */
|
|
384
|
+
// eslint-disable-next-line @typescript-eslint/require-await -- bytes in memory, as the async iterable a push reads
|
|
385
|
+
async function* slicesFrom(chunks, start) {
|
|
386
|
+
let at = 0;
|
|
387
|
+
for (const chunk of chunks) {
|
|
388
|
+
const end = at + chunk.length;
|
|
389
|
+
if (end > start)
|
|
390
|
+
yield at >= start ? chunk : chunk.subarray(start - at);
|
|
391
|
+
at = end;
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* A dataset's bytes from an offset on, read again from its start (a stream
|
|
396
|
+
* reads no other way), each byte hashed, and refused as they end when they
|
|
397
|
+
* are not the bytes measured.
|
|
398
|
+
*/
|
|
399
|
+
async function* checkedFrom(found, start, size, sha256) {
|
|
400
|
+
const digest = createHash('sha256');
|
|
401
|
+
let at = 0;
|
|
402
|
+
for await (const chunk of await found.open()) {
|
|
403
|
+
digest.update(chunk);
|
|
404
|
+
const end = at + chunk.length;
|
|
405
|
+
if (end > start)
|
|
406
|
+
yield at >= start ? chunk : chunk.subarray(start - at);
|
|
407
|
+
at = end;
|
|
408
|
+
}
|
|
409
|
+
if (at !== size || digest.digest('hex') !== sha256)
|
|
410
|
+
throw new DatasetMovedError(found.arg.arg);
|
|
411
|
+
}
|
|
412
|
+
//# sourceMappingURL=dataset-source.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dataset-source.js","sourceRoot":"","sources":["../../src/dataset-source.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,OAAO,EAAsB,MAAM,eAAe,CAAC;AAC7F,OAAO,EACL,eAAe,EAAE,gBAAgB,EAAE,YAAY,EAAE,yBAAyB,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,gBAAgB,GAExI,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,wBAAwB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AACtI,OAAO,EAAE,gBAAgB,EAAiB,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAA2B,MAAM,yBAAyB,CAAC;AAExF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,iDAAiD;AACjD,MAAM,CAAC,MAAM,kBAAkB,GAAG,UAAU,CAAC;AAE7C,+EAA+E;AAC/E,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAEnC,qFAAqF;AACrF,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAEnC,2EAA2E;AAC3E,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB,kGAAkG;AAClG,MAAM,mBAAmB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAU7C;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAwB;IAC3D,OAAO,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC;AACrF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,kBAAkB,CAAC;QAAE,OAAO,IAAI,CAAC;IACrD,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,iHAAiH,CAAC;IAChI,IAAI,MAA2C,CAAC;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;IAC3F,CAAC;IACD,MAAM,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC;IACpC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC;IAC/E,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;AACzD,CAAC;AA2BD,kFAAkF;AAClF,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,YAAY,GAAW;QACrB,KAAK,CAAC,GAAG,GAAG,oEAAoE,CAAC,CAAC;QAClF,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF;AAoBD,6BAA6B;AAC7B,SAAS,MAAM,CAAC,GAAG,KAAe;IAChC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,OAAO,CAAC,MAAyB,EAAE,IAAY;IACtD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC;IAChH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;IACnH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;IAC9H,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,QAAQ,CAAC,EAAE,CAAC;IAChG,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,wDAAwD;AACxD,SAAS,UAAU,CAAuB,KAAmB,EAAE,GAAe;IAC5E,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,oBAAoB,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,CAAE,CAAC;IAC7C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,IAAI,kBAAkB,GAAG,CAAC,SAAS,QAAQ,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;IAC9J,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,SAAS,uCAAuC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;AAC9G,CAAC;AAED,gEAAgE;AAChE,SAAS,eAAe,CAAC,GAAe,EAAE,IAAiB;IACzD,MAAM,IAAI,GAAG,oBAAoB,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3D,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,aAAa,CAAC,CAAC,OAAO,GAAG,IAAI,sDAAsD,GAAG,CAAC,SAAS,mCAAmC,CAAC;QACzI,KAAK,OAAO,CAAC,CAAC,OAAO,GAAG,IAAI,uCAAuC,CAAC;QACpE,KAAK,QAAQ,CAAC,CAAC,OAAO,GAAG,IAAI,6CAA6C,CAAC;IAC7E,CAAC;AACH,CAAC;AAED,sHAAsH;AACtH,SAAS,SAAS,CAAC,GAAe,EAAE,IAAiB,EAAE,IAAmB;IACxE,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1C,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;AACzH,CAAC;AAED,0BAA0B;AAC1B,4HAA4H;AAC5H,KAAK,SAAS,CAAC,CAAC,IAAI,CAAC,KAAiB;IACpC,MAAM,KAAK,CAAC;AACd,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAY;IAClD,OAAO,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;AAC/I,CAAC;AAOD,mEAAmE;AACnE,SAAS,YAAY,CAAC,KAA0B,EAAE,SAA4B,EAAE;IAC9E,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAgB,EAAE;QAC1C,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACrF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,2DAA2D;AAC3D,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,OAAO,GAAG,IAAI,YAAY,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,IAAI,GAAG,EAA+B,CAAC;IACrD,MAAM,MAAM,GAAG,CAAC,EAAU,EAAE,EAAE;QAC5B,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzB,IAAI,IAAI,KAAK,SAAS;YAAE,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACnI,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,OAAO;QACL,KAAK,CAAC,IAAI,CAAC,IAAI;YACb,MAAM,OAAO,GAAoB,EAAE,CAAC;YACpC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,IAAgB,CAAC;gBACrB,IAAI,CAAC;oBACH,IAAI,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACrC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,CAAC,CAAC,GAAG,YAAY,sBAAsB,IAAI,GAAG,YAAY,yBAAyB,IAAI,GAAG,YAAY,gBAAgB,CAAC;wBAAE,MAAM,GAAG,CAAC;oBACvI,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;oBAC7D,SAAS;gBACX,CAAC;gBACD,MAAM,KAAK,GAAG,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;gBAClD,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;oBACvB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACpB,SAAS;gBACX,CAAC;gBACD,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;gBAC7B,IAAI,IAAI,CAAC,OAAO,KAAK,YAAY,EAAE,CAAC;oBAClC,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;oBAC7E,SAAS;gBACX,CAAC;gBACD,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBACvD,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,WAAY,CAAC,EAAE,CAAC,CAAC;oBACjE,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;gBACvB,yEAAyE;gBACzE,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7G,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,OAAqB,EAAE,IAAY,EAAE,IAAY;IAC1E,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC5D,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QAC7B,KAAK,CAAC,CAAC,CAAC,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC;QAClG,OAAO;IACT,CAAC;IACD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IAC/D,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE,IAAI,UAAU;QAAE,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;AAC9I,CAAC;AAED,mGAAmG;AACnG,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;IACzD,MAAM,OAAO,GAAG,CAAC,EAAU,EAAE,EAAE;QAC7B,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC,KAAK,IAAI,EAAE;gBAClC,MAAM,KAAK,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,OAAO,EAAE,CAAC,CAAC;gBACjF,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC5B,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS;wBAAE,OAAO,EAAE,CAAC;oBACvC,qFAAqF;oBACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC7B,MAAM,KAAK,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;oBAC/E,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACvC,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,EAAE,CAAC,CAAC;QACR,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IACF,OAAO;QACL,KAAK,CAAC,IAAI,CAAC,IAAI;YACb,MAAM,OAAO,GAAoB,EAAE,CAAC;YACpC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,KAAmB,CAAC;gBACxB,IAAI,CAAC;oBACH,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACvC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,CAAC,CAAC,GAAG,YAAY,QAAQ,CAAC,IAAI,CAAC,CAAC,qBAAqB,EAAE,wBAAwB,EAAE,cAAc,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;wBAAE,MAAM,GAAG,CAAC;oBACnI,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;oBACpG,SAAS;gBACX,CAAC;gBACD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBACrC,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;oBACvB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACpB,SAAS;gBACX,CAAC;gBACD,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;gBAC7B,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,MAAM,OAAO,EAAE,CAAC,CAAC;gBAC3F,IAAI,MAAM,CAAC,OAAO,KAAK,YAAY,EAAE,CAAC;oBACpC,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;oBAC7E,SAAS;gBACX,CAAC;gBACD,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC7D,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBAC3D,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC;oBACX,KAAK,EAAE;wBACL,GAAG;wBACH,IAAI;wBACJ,IAAI;wBACJ,IAAI,EAAE,KAAK,IAAI,EAAE;4BACf,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,MAAM,OAAO,EAAE,CAAC,CAAC;4BACzF,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;gCACvB,6DAA6D;gCAC7D,MAAO,IAAI,CAAC,MAAqC,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC;gCACtE,MAAM,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;4BACvC,CAAC;4BACD,OAAO,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;wBAC3C,CAAC;qBACF;iBACF,CAAC,CAAC;YACL,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,kGAAkG;AAClG,KAAK,SAAS,CAAC,CAAC,WAAW,CAAC,MAAiC,EAAE,GAAW;IACxE,IAAI,CAAC;QACH,KAAK,CAAC,CAAC,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,wBAAwB;YAAE,MAAM,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAC9E,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAcD,kDAAkD;AAClD,SAAS,MAAM,CAAC,KAA4B;IAC1C,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IACpE,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAClB,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC;IACpB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAmB;IACvD,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,IAAI,GAAwB,EAAE,CAAC;IACnC,6EAA6E;IAC7E,4DAA4D;IAC5D,IAAI,IAAI,GAAiB,EAAE,CAAC;IAC5B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,IAAI,GAAyB,IAAI,CAAC;IACtC,IAAI,SAAS,GAAY,IAAI,CAAC;IAC9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrB,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACrB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,IAAI,IAAI,GAAG,UAAU;gBAAE,IAAI,GAAG,IAAI,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACxC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;YAC1B,IAAI,CAAC;gBACH,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBACpC,IAAI,GAAG,EAAE,CAAC;YACZ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,SAAS,IAAI,mBAAmB,EAAE,CAAC;oBACrC,SAAS,GAAG,GAAG,CAAC;oBAChB,IAAI,GAAG,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,SAAS,KAAK,IAAI;gBAAE,MAAM,SAAS,CAAC;YACxC,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,2BAA2B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjH,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,IAAI,CAAC;IACnB,MAAM,KAAK,GAAmB;QAC5B,IAAI;QACJ,MAAM,EAAE,KAAK,KAAK,IAAI;QACtB,IAAI,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;KAChH,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,oBAAoB,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC;AACtI,CAAC;AAED,oCAAoC;AACpC,mHAAmH;AACnH,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,MAA6B,EAAE,KAAa;IACrE,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC;QAC9B,IAAI,GAAG,GAAG,KAAK;YAAE,MAAM,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;QACxE,EAAE,GAAG,GAAG,CAAC;IACX,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,KAAK,SAAS,CAAC,CAAC,WAAW,CAAC,KAAmB,EAAE,KAAa,EAAE,IAAY,EAAE,MAAc;IAC1F,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,GAAG,GAAG,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC;QAC9B,IAAI,GAAG,GAAG,KAAK;YAAE,MAAM,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;QACxE,EAAE,GAAG,GAAG,CAAC;IACX,CAAC;IACD,IAAI,EAAE,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,MAAM;QAAE,MAAM,IAAI,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AACjG,CAAC"}
|
|
@@ -8,7 +8,7 @@ import { type FilesBinding, type FilesManifest, type FilesProblem, type FilesPus
|
|
|
8
8
|
import { type FilesLedger, type FilesReader } from '@elaraai/e3-cloud-files/sync';
|
|
9
9
|
import type { OfferedFile } from './files-push.js';
|
|
10
10
|
/** The version of the state file this CLI writes. */
|
|
11
|
-
export declare const LOCAL_FILES_VERSION =
|
|
11
|
+
export declare const LOCAL_FILES_VERSION = 3n;
|
|
12
12
|
/** What a local repository keeps of one workspace's Files. */
|
|
13
13
|
export declare const LocalFilesStateType: StructType<{
|
|
14
14
|
/** The state file's version: {@link LOCAL_FILES_VERSION} */
|
|
@@ -155,6 +155,7 @@ export declare const LocalFilesStateType: StructType<{
|
|
|
155
155
|
readonly sha256: OptionType<StringType>;
|
|
156
156
|
readonly staging: OptionType<StringType>;
|
|
157
157
|
}>>;
|
|
158
|
+
readonly seeds: OptionType<IntegerType>;
|
|
158
159
|
}>>;
|
|
159
160
|
/** The last sync's report; a dry run's is never kept */
|
|
160
161
|
readonly lastSync: OptionType<StructType<{
|
|
@@ -334,6 +335,10 @@ export declare const LocalFilesStateType: StructType<{
|
|
|
334
335
|
}>;
|
|
335
336
|
}>>;
|
|
336
337
|
readonly commit: OptionType<StringType>;
|
|
338
|
+
readonly seed: OptionType<StructType<{
|
|
339
|
+
readonly commit: StringType;
|
|
340
|
+
readonly number: OptionType<IntegerType>;
|
|
341
|
+
}>>;
|
|
337
342
|
readonly kept: OptionType<StringType>;
|
|
338
343
|
readonly waiting: OptionType<StringType>;
|
|
339
344
|
}>>;
|
|
@@ -355,7 +360,7 @@ export declare function openLocalRepository(repoArg: string): Promise<string>;
|
|
|
355
360
|
* @throws {Error} When the name is no workspace's
|
|
356
361
|
*/
|
|
357
362
|
export declare function statePath(repoPath: string, ws: string): string;
|
|
358
|
-
/** What a repository keeps of a workspace's Files: nothing yet, before its first deploy;
|
|
363
|
+
/** What a repository keeps of a workspace's Files: nothing yet, before its first deploy; an earlier version's file lifted. */
|
|
359
364
|
export declare function readState(repoPath: string, ws: string): LocalFilesState;
|
|
360
365
|
/** Keeps a workspace's Files, replacing the file whole so a reader never sees half of it. */
|
|
361
366
|
export declare function writeState(repoPath: string, ws: string, state: LocalFilesState): void;
|
|
@@ -448,22 +453,12 @@ export declare function syncLocal(repoPath: string, ws: string, options?: LocalS
|
|
|
448
453
|
* @returns Its status
|
|
449
454
|
*/
|
|
450
455
|
export declare function statusLocal(repoPath: string, ws: string): Promise<FilesStatus>;
|
|
451
|
-
/** What a local push is given. */
|
|
452
|
-
export interface
|
|
453
|
-
/** The target every file is for;
|
|
456
|
+
/** What a local push's plan is given. */
|
|
457
|
+
export interface LocalPlanOptions {
|
|
458
|
+
/** The target every file is for; absent to match each by its name */
|
|
454
459
|
readonly as?: string;
|
|
455
|
-
/** Records whose file is taken as their state again */
|
|
456
|
-
readonly reset?: readonly string[];
|
|
457
|
-
/** Plan, and put nothing */
|
|
458
|
-
readonly dryRun?: boolean;
|
|
459
|
-
/** The profile to put with, over each binding's */
|
|
460
|
-
readonly profile?: string;
|
|
461
460
|
/** Reads a bound source, when not its bucket with the developer's credentials ({@link s3Readers}) */
|
|
462
461
|
readonly reader?: (binding: FilesBinding) => FilesReader;
|
|
463
|
-
/** Puts a file to a binding, when not through the developer's credentials (a test's) */
|
|
464
|
-
readonly put?: (binding: FilesBinding, key: string, file: OfferedFile) => Promise<void>;
|
|
465
|
-
/** The runner that takes a typed collection in and runs a fed mutation */
|
|
466
|
-
readonly runner?: TaskRunner;
|
|
467
462
|
}
|
|
468
463
|
/**
|
|
469
464
|
* Plans a push on a local repository: the same plan a deployed workspace's
|
|
@@ -473,10 +468,10 @@ export interface LocalPushOptions {
|
|
|
473
468
|
*
|
|
474
469
|
* @param repoPath - The repository
|
|
475
470
|
* @param ws - The workspace
|
|
476
|
-
* @param files - The files offered
|
|
471
|
+
* @param files - The files offered, from disk or a repository's datasets
|
|
477
472
|
* @param options - The target named, and the reader
|
|
478
473
|
* @returns The plan, as the route answers one
|
|
479
474
|
* @throws {Error} When the workspace has no declarations deployed
|
|
480
475
|
*/
|
|
481
|
-
export declare function planLocal(repoPath: string, ws: string, files: readonly OfferedFile[], options?:
|
|
476
|
+
export declare function planLocal(repoPath: string, ws: string, files: readonly OfferedFile[], options?: LocalPlanOptions): Promise<FilesPushPlan>;
|
|
482
477
|
//# sourceMappingURL=files-local.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"files-local.d.ts","sourceRoot":"","sources":["../../src/files-local.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"files-local.d.ts","sourceRoot":"","sources":["../../src/files-local.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA8BH,OAAO,EACL,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EACP,KAAK,WAAW,EAC9E,MAAM,eAAe,CAAC;AACvB,OAAO,EAAwG,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEzJ,OAAO,EAEL,KAAK,YAAY,EAAE,KAAK,aAAa,EAA2B,KAAK,YAAY,EAAE,KAAK,aAAa,EAAE,KAAK,WAAW,EAAE,KAAK,eAAe,EAAE,KAAK,WAAW,EAChK,MAAM,yBAAyB,CAAC;AAMjC,OAAO,EAEL,KAAK,WAAW,EAAoB,KAAK,WAAW,EACrD,MAAM,8BAA8B,CAAC;AAEtC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAInD,qDAAqD;AACrD,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,8DAA8D;AAC9D,eAAO,MAAM,mBAAmB;IAC9B,4DAA4D;;IAE5D,iDAAiD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAEjD,iCAAiC;;;;;;;;IAEjC,oDAAoD;;;;;;;;;;;;;;;;;;;;IAEpD,wDAAwD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAExD,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC,OAAO,mBAAmB,CAAC,CAAC;AA4BtE;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAI1E;AAED;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,CAI9D;AAeD,8HAA8H;AAC9H,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,eAAe,CA4BvE;AAED,6FAA6F;AAC7F,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,GAAG,IAAI,CAMrF;AAED,2GAA2G;AAC3G,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,WAAW,CAcrE;AA8BD;;;;;;;GAOG;AACH,wBAAgB,SAAS,IAAI,CAAC,OAAO,EAAE,YAAY,KAAK,WAAW,CAYlE;AAcD,sDAAsD;AACtD,MAAM,WAAW,iBAAiB;IAChC,kFAAkF;IAClF,QAAQ,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IACtE,uEAAuE;IACvE,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C,mFAAmF;IACnF,QAAQ,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,CAAC;CAC1C;AAED;;;;;;;;;GASG;AACH,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAmBvJ;AAED,uBAAuB;AACvB,MAAM,WAAW,eAAe;IAC9B,2EAA2E;IAC3E,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C,iGAAiG;IACjG,QAAQ,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,CAAC;CAC1C;AAED;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,eAAe,CAc9F;AAED,kCAAkC;AAClC,MAAM,WAAW,gBAAgB;IAC/B,4DAA4D;IAC5D,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;IAC1C,mGAAmG;IACnG,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,oDAAoD;IACpD,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,0GAA0G;IAC1G,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,WAAW,CAAC;IACzD,uGAAuG;IACvG,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC;CAC9B;AAWD;;;;;;;;;;;GAWG;AACH,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAoBtH;AAED;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAmBpF;AAED,yCAAyC;AACzC,MAAM,WAAW,gBAAgB;IAC/B,qEAAqE;IACrE,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC;IACrB,qGAAqG;IACrG,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,WAAW,CAAC;CAC1D;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,WAAW,EAAE,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,aAAa,CAAC,CAgCnJ"}
|
package/dist/src/files-local.js
CHANGED
|
@@ -11,16 +11,19 @@
|
|
|
11
11
|
* deployed to the workspace, where each source is bound, what each target
|
|
12
12
|
* took last, and the last sync's report. It belongs to the repository, never
|
|
13
13
|
* to the project, so removing the repository removes it, with the bytes the
|
|
14
|
-
* syncs took in. A state file
|
|
15
|
-
* input, no hashes
|
|
16
|
-
*
|
|
14
|
+
* syncs took in. A state file an earlier release wrote (version 1, before
|
|
15
|
+
* typed files: mappings by input, no hashes; version 2, before a record's
|
|
16
|
+
* seeds were counted) is read through `@elaraai/e3-cloud-files/compat`
|
|
17
|
+
* and written at version 3 on its next write.
|
|
17
18
|
*
|
|
18
19
|
* A sync reads each source with the developer's own cloud credentials (the
|
|
19
20
|
* binding's profile, else the SDK's usual chain) and writes the inputs
|
|
20
21
|
* through e3's own dataset writes, a typed file through e3's door for a
|
|
21
22
|
* file (streamed to the repository's own scratch first), and a fed mutation
|
|
22
23
|
* through e3's own `recordMutate` on a local runner, so the repository holds
|
|
23
|
-
* what a deployed workspace's sync would.
|
|
24
|
+
* what a deployed workspace's sync would. A file a push put from a dataset
|
|
25
|
+
* names the dataset in its object's metadata, and its seed's or feed's commit
|
|
26
|
+
* records it.
|
|
24
27
|
*/
|
|
25
28
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
26
29
|
import { userInfo } from 'node:os';
|
|
@@ -31,13 +34,13 @@ import { LocalStorage, LocalTaskRunner, repoGet, repositoryOpen, workspaceGetDat
|
|
|
31
34
|
import { nameProblem } from '@elaraai/e3-types';
|
|
32
35
|
import { FilesBindingType, FilesLedgerType, FilesManifestType, FilesMappingType, FilesSyncReportType, filesTargetText, } from '@elaraai/e3-cloud-types';
|
|
33
36
|
import { checkBindings, checkFiles, parseBucketUrl, readPackageSurface } from '@elaraai/e3-cloud-files/check';
|
|
34
|
-
import { FilesLedgerTypeV1, FilesManifestTypeV1, FilesSyncReportTypeV1, liftFilesLedgerV1, liftFilesManifestV1, liftFilesSyncReportV1, } from '@elaraai/e3-cloud-files/compat';
|
|
35
|
-
import { FilesSyncError, FilesUnreadableError, NO_LEDGER, ledgerKeyOf, ledgerStateOf, ledgerValueOf, planPush, syncFiles } from '@elaraai/e3-cloud-files/sync';
|
|
37
|
+
import { FilesLedgerTypeV1, FilesLedgerTypeV2, FilesManifestTypeV1, FilesSyncReportTypeV1, FilesSyncReportTypeV2, liftFilesLedgerV1, liftFilesLedgerV2, liftFilesManifestV1, liftFilesSyncReportV1, liftFilesSyncReportV2, } from '@elaraai/e3-cloud-files/compat';
|
|
38
|
+
import { FilesSyncError, FilesUnreadableError, NO_LEDGER, ledgerKeyOf, ledgerStateOf, ledgerValueOf, planPush, syncFiles, } from '@elaraai/e3-cloud-files/sync';
|
|
36
39
|
import { globMatcher } from '@elaraai/e3-cloud-files/check';
|
|
37
40
|
import { storageWriter } from '@elaraai/e3-cloud-files/storage';
|
|
38
41
|
import { s3FilesReader } from '@elaraai/e3-cloud-files/s3';
|
|
39
42
|
/** The version of the state file this CLI writes. */
|
|
40
|
-
export const LOCAL_FILES_VERSION =
|
|
43
|
+
export const LOCAL_FILES_VERSION = 3n;
|
|
41
44
|
/** What a local repository keeps of one workspace's Files. */
|
|
42
45
|
export const LocalFilesStateType = StructType({
|
|
43
46
|
/** The state file's version: {@link LOCAL_FILES_VERSION} */
|
|
@@ -58,9 +61,18 @@ const LocalFilesStateTypeV1 = StructType({
|
|
|
58
61
|
ledgers: DictType(StringType, FilesLedgerTypeV1),
|
|
59
62
|
lastSync: OptionType(FilesSyncReportTypeV1),
|
|
60
63
|
});
|
|
64
|
+
/** The state file as the second version wrote it: a record's seeds not counted. */
|
|
65
|
+
const LocalFilesStateTypeV2 = StructType({
|
|
66
|
+
version: IntegerType,
|
|
67
|
+
manifest: OptionType(FilesManifestType),
|
|
68
|
+
bindings: ArrayType(FilesBindingType),
|
|
69
|
+
ledgers: DictType(StringType, FilesLedgerTypeV2),
|
|
70
|
+
lastSync: OptionType(FilesSyncReportTypeV2),
|
|
71
|
+
});
|
|
61
72
|
const encodeState = encodeJSONFor(LocalFilesStateType);
|
|
62
73
|
const decodeState = decodeJSONFor(LocalFilesStateType);
|
|
63
74
|
const decodeStateV1 = decodeJSONFor(LocalFilesStateTypeV1);
|
|
75
|
+
const decodeStateV2 = decodeJSONFor(LocalFilesStateTypeV2);
|
|
64
76
|
const sameMapping = equalFor(FilesMappingType);
|
|
65
77
|
/** A workspace's Files before its first deploy: nothing. */
|
|
66
78
|
const NO_STATE = { version: LOCAL_FILES_VERSION, manifest: none, bindings: [], ledgers: new Map(), lastSync: none };
|
|
@@ -100,15 +112,26 @@ function versionOf(text) {
|
|
|
100
112
|
const version = parsed.version;
|
|
101
113
|
return typeof version === 'number' ? version : typeof version === 'string' ? Number(version) : 1;
|
|
102
114
|
}
|
|
103
|
-
/** What a repository keeps of a workspace's Files: nothing yet, before its first deploy;
|
|
115
|
+
/** What a repository keeps of a workspace's Files: nothing yet, before its first deploy; an earlier version's file lifted. */
|
|
104
116
|
export function readState(repoPath, ws) {
|
|
105
117
|
const path = statePath(repoPath, ws);
|
|
106
118
|
if (!existsSync(path))
|
|
107
119
|
return NO_STATE;
|
|
108
120
|
try {
|
|
109
121
|
const bytes = readFileSync(path);
|
|
110
|
-
|
|
122
|
+
const version = versionOf(new TextDecoder().decode(bytes));
|
|
123
|
+
if (version >= 3)
|
|
111
124
|
return decodeState(bytes);
|
|
125
|
+
if (version === 2) {
|
|
126
|
+
const v2 = decodeStateV2(bytes);
|
|
127
|
+
return {
|
|
128
|
+
version: LOCAL_FILES_VERSION,
|
|
129
|
+
manifest: v2.manifest,
|
|
130
|
+
bindings: v2.bindings,
|
|
131
|
+
ledgers: new Map([...v2.ledgers].map(([key, ledger]) => [key, liftFilesLedgerV2(ledger)])),
|
|
132
|
+
lastSync: v2.lastSync.type === 'some' ? some(liftFilesSyncReportV2(v2.lastSync.value)) : none,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
112
135
|
const v1 = decodeStateV1(bytes);
|
|
113
136
|
return {
|
|
114
137
|
version: LOCAL_FILES_VERSION,
|
|
@@ -149,7 +172,7 @@ export function stateLedger(repoPath, ws) {
|
|
|
149
172
|
/**
|
|
150
173
|
* A reader whose failures name the source, where it is bound and the profile
|
|
151
174
|
* it reads with: an expired login, a denied read or a missing bucket says
|
|
152
|
-
* which binding it was.
|
|
175
|
+
* which binding it was. Where a file came from is the reader's own to say.
|
|
153
176
|
*/
|
|
154
177
|
function naming(reader, binding) {
|
|
155
178
|
const where = `source ${binding.source} (${binding.url}${binding.profile.type === 'some' ? `, profile ${binding.profile.value}` : ''})`;
|
|
@@ -173,6 +196,7 @@ function naming(reader, binding) {
|
|
|
173
196
|
read: (object) => reader.read(object).catch(fail),
|
|
174
197
|
readRange: (object, start, end) => reader.readRange(object, start, end).catch(fail),
|
|
175
198
|
open: (object, options) => reader.open(object, options).catch(fail),
|
|
199
|
+
...(reader.origin !== undefined && { origin: (object) => reader.origin(object).catch(fail) }),
|
|
176
200
|
};
|
|
177
201
|
}
|
|
178
202
|
/**
|
|
@@ -346,7 +370,7 @@ export async function statusLocal(repoPath, ws) {
|
|
|
346
370
|
*
|
|
347
371
|
* @param repoPath - The repository
|
|
348
372
|
* @param ws - The workspace
|
|
349
|
-
* @param files - The files offered
|
|
373
|
+
* @param files - The files offered, from disk or a repository's datasets
|
|
350
374
|
* @param options - The target named, and the reader
|
|
351
375
|
* @returns The plan, as the route answers one
|
|
352
376
|
* @throws {Error} When the workspace has no declarations deployed
|