@elaraai/e3-cloud-files 1.0.94 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +221 -28
- package/dist/src/authoring.d.ts +90 -22
- package/dist/src/authoring.d.ts.map +1 -1
- package/dist/src/authoring.js +122 -46
- package/dist/src/authoring.js.map +1 -1
- package/dist/src/check/index.d.ts +3 -4
- package/dist/src/check/index.d.ts.map +1 -1
- package/dist/src/check/index.js +3 -4
- package/dist/src/check/index.js.map +1 -1
- package/dist/src/check.d.ts +44 -9
- package/dist/src/check.d.ts.map +1 -1
- package/dist/src/check.js +130 -53
- package/dist/src/check.js.map +1 -1
- package/dist/src/compat.d.ts +384 -0
- package/dist/src/compat.d.ts.map +1 -0
- package/dist/src/compat.js +258 -0
- package/dist/src/compat.js.map +1 -0
- package/dist/src/index.d.ts +14 -9
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +14 -9
- package/dist/src/index.js.map +1 -1
- package/dist/src/s3/index.d.ts.map +1 -1
- package/dist/src/s3/index.js +34 -12
- package/dist/src/s3/index.js.map +1 -1
- package/dist/src/storage/index.d.ts +84 -3
- package/dist/src/storage/index.d.ts.map +1 -1
- package/dist/src/storage/index.js +387 -13
- package/dist/src/storage/index.js.map +1 -1
- package/dist/src/sync/engine.d.ts +288 -49
- package/dist/src/sync/engine.d.ts.map +1 -1
- package/dist/src/sync/engine.js +605 -110
- package/dist/src/sync/engine.js.map +1 -1
- package/dist/src/sync/index.d.ts +3 -2
- package/dist/src/sync/index.d.ts.map +1 -1
- package/dist/src/sync/index.js +3 -2
- package/dist/src/sync/index.js.map +1 -1
- package/dist/src/sync/memory.d.ts +67 -12
- package/dist/src/sync/memory.d.ts.map +1 -1
- package/dist/src/sync/memory.js +248 -18
- package/dist/src/sync/memory.js.map +1 -1
- package/dist/src/sync/push.d.ts +56 -0
- package/dist/src/sync/push.d.ts.map +1 -0
- package/dist/src/sync/push.js +205 -0
- package/dist/src/sync/push.js.map +1 -0
- package/dist/test/decode.examples.d.ts +9 -4
- package/dist/test/decode.examples.d.ts.map +1 -1
- package/dist/test/decode.examples.js +15 -3
- package/dist/test/decode.examples.js.map +1 -1
- package/dist/test/typed.examples.d.ts +142 -0
- package/dist/test/typed.examples.d.ts.map +1 -0
- package/dist/test/typed.examples.js +125 -0
- package/dist/test/typed.examples.js.map +1 -0
- package/package.json +10 -6
package/README.md
CHANGED
|
@@ -9,22 +9,35 @@ changed, the same way on a local repository and on a deployed workspace.
|
|
|
9
9
|
- **Your data stays in the bucket.** Locally, a sync reads the bucket with
|
|
10
10
|
your own cloud credentials. The bytes land only in the local e3 repository,
|
|
11
11
|
never in your project.
|
|
12
|
-
- **A file arrives as it is**, a `Blob
|
|
13
|
-
JSON, as CSV, or as text.
|
|
12
|
+
- **A file arrives as it is**, a `Blob`, and a task decodes it in East: as
|
|
13
|
+
East's JSON, as CSV, or as text. Or, named with a **type**, a beast2 file
|
|
14
|
+
arrives as the value it holds, an input of that type with no task in
|
|
15
|
+
between, a collection of any size.
|
|
16
|
+
- **Records are seeded and fed from files.** A record may be seeded from a
|
|
17
|
+
file (`upload.record`): pending until its file arrives, which is its
|
|
18
|
+
first state, and in its history the seed names the file. A record's
|
|
19
|
+
mutation may be fed by files: each new file is the mutation's one
|
|
20
|
+
argument, applied once.
|
|
21
|
+
- **Push from disk.** `e3-cloud files push` puts files from your machine
|
|
22
|
+
into a source and takes them in, with your e3 login alone where the source
|
|
23
|
+
is the deployment's own bucket.
|
|
14
24
|
- **Nothing polls.** A sync runs when someone asks for it, and reads only the
|
|
15
25
|
files that changed since the last one.
|
|
16
26
|
|
|
17
|
-
Its version is
|
|
18
|
-
|
|
27
|
+
Its version is e3 cloud's own, shared by every `@elaraai/e3-cloud-*` package.
|
|
28
|
+
Each release is built with one release of east and e3, which its peers name
|
|
29
|
+
exactly.
|
|
19
30
|
|
|
20
31
|
## Install
|
|
21
32
|
|
|
22
33
|
```bash
|
|
23
|
-
npm install @elaraai/e3-cloud-files@<
|
|
34
|
+
npm install @elaraai/e3-cloud-files@<release>
|
|
24
35
|
```
|
|
25
36
|
|
|
26
|
-
|
|
27
|
-
`
|
|
37
|
+
Install the release built with the e3 your deployment runs, beside its peers
|
|
38
|
+
at the versions it names (`npm view @elaraai/e3-cloud-files@<release>
|
|
39
|
+
peerDependencies`): `@elaraai/east` and `@elaraai/e3`. The `e3-cloud` CLI
|
|
40
|
+
(`@elaraai/e3-cloud-cli`) of the same release deploys and syncs.
|
|
28
41
|
|
|
29
42
|
## Declaring the files
|
|
30
43
|
|
|
@@ -54,9 +67,13 @@ export default Files.manifest([upload]);
|
|
|
54
67
|
| Shape | Selects | The input's type | Largest file |
|
|
55
68
|
|---|---|---|---|
|
|
56
69
|
| `file(input, { key })` | One key | `Blob` | 1 GiB less 64 KiB |
|
|
70
|
+
| `file(input, { key, type })` | One key, a beast2 file of `type` | `type` | 1 GiB for a value that is no collection; a collection any size (a segment at most 64 MiB) |
|
|
57
71
|
| `latest(input, { prefix, include? })` | The greatest key under the prefix that matches | `Blob` | 1 GiB less 64 KiB |
|
|
72
|
+
| `latest(input, { prefix, include?, type })` | The greatest key, a beast2 file of `type` | `type` | as a typed `file` |
|
|
58
73
|
| `folder(input, { prefix, include? })` | Every key under the prefix that matches, by its path under the prefix | `Dict<String, Blob>` | 64 MiB less 64 KiB each |
|
|
59
74
|
| `listing(input, { prefix, include? })` | The same keys' size, time and ETag, without their bytes | `Dict<String, Struct{size, modified, etag}>` | none |
|
|
75
|
+
| `record(name, type, { key } \| { prefix, include? }, parts?)` | One key, or the greatest under a prefix: a beast2 file of `type`, the record's first state and, while unmutated, its state again | the record's (`e3.record(name, type, parts)`: its indexes, mutations and migrations, no initial value) | as a typed `file` |
|
|
76
|
+
| `mutation(def, { key } \| { prefix, include? })` | One key, or every file under a prefix its glob matches, each new one in key order: a beast2 file of the mutation's one argument's type | (the mutation's, as declared) | 1 GiB |
|
|
60
77
|
|
|
61
78
|
- **Each shape returns an ordinary `e3.input`.** It has no default, so it
|
|
62
79
|
stays unassigned until a sync sets it. A task takes it like any input, and
|
|
@@ -72,8 +89,154 @@ export default Files.manifest([upload]);
|
|
|
72
89
|
subfolders' too, their paths naming the subfolder (`2026/10/a.json`).
|
|
73
90
|
- **Mistakes are refused as they are declared** (`FilesDeclarationError`, one
|
|
74
91
|
line per problem): a name e3 refuses, a key that isn't relative, a glob
|
|
75
|
-
that can't match,
|
|
76
|
-
shapes filling one
|
|
92
|
+
that can't match, a `patch` mutation or one of another arity than one, and
|
|
93
|
+
in `Files.manifest` two sources of one name or two shapes filling one
|
|
94
|
+
target.
|
|
95
|
+
|
|
96
|
+
## A typed file
|
|
97
|
+
|
|
98
|
+
A `file` or `latest` that names a `type` is an input of that type, and the
|
|
99
|
+
sync takes the file in as the value it holds, with no task in between:
|
|
100
|
+
|
|
101
|
+
```ts
|
|
102
|
+
// src/package.ts
|
|
103
|
+
export const Price = StructType({ sku: StringType, price: FloatType });
|
|
104
|
+
export const prices = upload.file('prices', { key: 'prices.beast2', type: ArrayType(Price) }); // an input of Array<Price>
|
|
105
|
+
export const stock = upload.latest('stock', { prefix: 'stock/', type: ArrayType(Stock) }); // the greatest key, typed
|
|
106
|
+
export const summary = e3.task('price_summary', [prices], …); // used as any dataset
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
- **The header first.** The sync reads a typed file's beast2 header before
|
|
110
|
+
any other byte of it and refuses a file of another type (`type_mismatch`,
|
|
111
|
+
naming both types) or no beast2 (`not_beast2`); the input keeps its value
|
|
112
|
+
and the sync goes on. A deploy of the declarations checks the type against
|
|
113
|
+
the package's input, record or mutation too.
|
|
114
|
+
- **Any size.** A collection (an `Array`, a `Set`, a `Dict`) is taken in by
|
|
115
|
+
pieces, as e3 takes in any delivery, however large; in the cloud never
|
|
116
|
+
through a disk. A value that is no collection is held whole: at most 1 GiB.
|
|
117
|
+
A file whose segments are larger than beast2's 64 MiB is refused naming
|
|
118
|
+
the fix (write it again with a current writer).
|
|
119
|
+
- **The same bytes again change nothing.** A file re-uploaded unchanged (a
|
|
120
|
+
new ETag, the same SHA-256) gives the same value: nothing downstream runs.
|
|
121
|
+
The report counts it `changed`, not `updated`.
|
|
122
|
+
|
|
123
|
+
## A record seeded from a file
|
|
124
|
+
|
|
125
|
+
`upload.record(name, type, { key }, parts)` (or `{ prefix, include }`, the
|
|
126
|
+
greatest key) declares `e3.record(name, type, parts)` with no initial value,
|
|
127
|
+
and returns the record, used in tasks as any record. A record is declared
|
|
128
|
+
whole, so its indexes, mutations and migrations are its `parts`, as
|
|
129
|
+
`e3.record` takes them; an initial value is refused, since its file is its
|
|
130
|
+
first state:
|
|
131
|
+
|
|
132
|
+
```ts
|
|
133
|
+
// src/package.ts
|
|
134
|
+
export const Customer = StructType({ name: StringType, tier: IntegerType });
|
|
135
|
+
export const CustomerBook = DictType(StringType, Customer);
|
|
136
|
+
export const customers = upload.record('customers', CustomerBook, { key: 'customers.beast2' }, {
|
|
137
|
+
mutations: (mutation) => ({
|
|
138
|
+
set_tier: mutation.reduce(East.function([CustomerBook, StringType, IntegerType], CustomerBook, …)),
|
|
139
|
+
}),
|
|
140
|
+
});
|
|
141
|
+
export const customerCount = e3.task('customer_count', [customers], …);
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
- **Pending until its file.** The record has no state until its file
|
|
145
|
+
arrives, which is its first state (`$init`): nothing runs on an empty
|
|
146
|
+
state first, and the tasks over it wait as they wait on an unset input,
|
|
147
|
+
the run completing without them. The workspace page shows it **Pending**,
|
|
148
|
+
its own page says what seeds it, and `files status` prints `pending`.
|
|
149
|
+
- **Seeded again while unmutated.** A changed file while nothing has mutated
|
|
150
|
+
the record since its seed is its state again (`$seed`). Once mutated the
|
|
151
|
+
record is kept (the report says how many mutations since the seed), until
|
|
152
|
+
`files sync --reset <record>` (or `files push --reset <record>`) takes the
|
|
153
|
+
file as its state over whatever was done.
|
|
154
|
+
- **In its history** the seed reads as any commit, through the `files`
|
|
155
|
+
channel: "Initialised from customers.beast2 (12,004 rows)", "Seeded again
|
|
156
|
+
from customers.beast2 (12,100 rows)", on the record's page and in
|
|
157
|
+
`e3-cloud records history`. A rollback to an earlier seed brings its state
|
|
158
|
+
back, as to any commit.
|
|
159
|
+
- **A deploy that cannot keep the record** (its type changed) under
|
|
160
|
+
`--schema reset` unassigns it: pending again, seeded by the next version's
|
|
161
|
+
file once the new declarations are deployed.
|
|
162
|
+
- The file is taken in as a typed input is (by pieces, resumable), then
|
|
163
|
+
committed as a system commit with the file's key, ETag and SHA-256 as its
|
|
164
|
+
arguments, under one hold of the repository's running work.
|
|
165
|
+
|
|
166
|
+
## A record fed through a mutation
|
|
167
|
+
|
|
168
|
+
`upload.mutation(def, { key })` or `{ prefix, include }` feeds a record's
|
|
169
|
+
mutation: `def` is one the record declares, `record.mutations.<name>`, a
|
|
170
|
+
`reduce` or an `edit` taking exactly one argument (a `patch` cannot be fed),
|
|
171
|
+
and each file is that argument:
|
|
172
|
+
|
|
173
|
+
```ts
|
|
174
|
+
// src/package.ts
|
|
175
|
+
export const PriceBook = DictType(StringType, FloatType);
|
|
176
|
+
export const priceBook = e3.record('price_book', PriceBook, {
|
|
177
|
+
initial: new Map(),
|
|
178
|
+
mutations: (mutation) => ({
|
|
179
|
+
load_prices: mutation.reduce(East.function([PriceBook, ArrayType(Price)], PriceBook, …)),
|
|
180
|
+
}),
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
// src/files.ts
|
|
184
|
+
upload.mutation(priceBook.mutations.load_prices, { prefix: 'deltas/', include: '*.beast2' }); // each new delta file calls load_prices with its value
|
|
185
|
+
export default Files.manifest([upload]);
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
- **Once per file.** Every new file the mapping selects is applied, in key
|
|
189
|
+
order, once by its bytes' hash: a sync run again applies nothing twice,
|
|
190
|
+
and the same bytes under another name are applied no more.
|
|
191
|
+
- **The record's rules decide.** A file the mutation refuses (`invalid`,
|
|
192
|
+
`failed`, `timed_out`) is reported with what it said and tried again only
|
|
193
|
+
once the file changes, and the files after it wait; a feed of a record
|
|
194
|
+
with no state yet, or one busy, waits for the next sync. Its commit reads
|
|
195
|
+
in the record's history as any mutation's, through the `files` channel,
|
|
196
|
+
naming the file it was fed.
|
|
197
|
+
|
|
198
|
+
## Pushing files from disk
|
|
199
|
+
|
|
200
|
+
`e3-cloud files push` puts files from your machine into a source and takes
|
|
201
|
+
them in: put, then one sync of what was put, idempotent and safe to repeat.
|
|
202
|
+
|
|
203
|
+
```
|
|
204
|
+
$ e3-cloud files push https://e3.example.com/repos/acme dev ./data/prices.beast2 ./data/deltas/2026-10-19.beast2
|
|
205
|
+
upload s3://acme-files/acme/dev/
|
|
206
|
+
prices.beast2 → prices Array<Struct{sku, price}> 38 MiB put taken in 412,310 rows
|
|
207
|
+
2026-10-19.beast2 → mutation:price_book.load_prices load_prices 1 MiB put committed 9f2c1a0b7e3d
|
|
208
|
+
Next: e3 dataflow run https://e3.example.com/repos/acme dev
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
- **Each file is matched to the mapping that takes it**: a `file` by its
|
|
212
|
+
name, a `latest` under its prefix (a greater key there would shadow it:
|
|
213
|
+
refused `not_latest`), a directory to the `folder`; or `--as <target>`
|
|
214
|
+
names one for every file (`orders`, `record:customers`,
|
|
215
|
+
`mutation:price_book.load_prices`). A file no mapping takes, or several
|
|
216
|
+
would, is refused naming them.
|
|
217
|
+
- **Checked before it is put**: against the mapping's type (the file's
|
|
218
|
+
header is read) and its limit.
|
|
219
|
+
- **The deployment's own bucket needs no credentials of yours**: the
|
|
220
|
+
platform signs the upload, bound to the file's SHA-256, one PUT up to
|
|
221
|
+
5 GiB or a multipart upload in 64 MiB parts. A bucket the deployment lists
|
|
222
|
+
as a source is written with your own credentials (`--profile`); a local
|
|
223
|
+
repository's sources, with the binding's profile.
|
|
224
|
+
- **Unchanged once it is there**: a file the source holds already, by its
|
|
225
|
+
hash and ETag, is `unchanged`, and running the command again puts only
|
|
226
|
+
what is not there. `--dry-run` plans and puts nothing.
|
|
227
|
+
- The output is a row per file: the file, its target and kind, its size,
|
|
228
|
+
`put`, `unchanged` or `refused`, and what the sync did (`taken in` with
|
|
229
|
+
the rows, `$init`/`$seed` or the mutation with its commit, `kept`, or the
|
|
230
|
+
refusal). The command exits 1 when any file was refused or failed.
|
|
231
|
+
|
|
232
|
+
## Making a beast2 file
|
|
233
|
+
|
|
234
|
+
A beast2 file already is an East value. The East SDK writes one from a value
|
|
235
|
+
(`encodeBeast2For(T)(value)`), or a collection of any size by elements
|
|
236
|
+
(`Beast2ElementWriter`, which the platform's own tests use for files of a
|
|
237
|
+
gibibyte); east-py writes one from Python; `e3 convert` converts a file of
|
|
238
|
+
another format; `e3 dataset get -f beast2` writes any dataset of a workspace
|
|
239
|
+
as one. A typed mapping takes it as it stands, its header saying its type.
|
|
77
240
|
|
|
78
241
|
## Decoding in a task
|
|
79
242
|
|
|
@@ -138,13 +301,22 @@ e3 dataflow run ./repo dev
|
|
|
138
301
|
only each folder's n greatest keys, which caps what comes down to your
|
|
139
302
|
machine.
|
|
140
303
|
- **`files sync`** lists each source and compares every file with what its
|
|
141
|
-
|
|
142
|
-
writes only the
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
304
|
+
target took last, by ETag and size. It reads only what is new or changed,
|
|
305
|
+
writes only the targets whose files changed, and says what it did per
|
|
306
|
+
target: the rows taken in, the commit a feed made, what was refused and
|
|
307
|
+
why. `--target` syncs only those (an input's name, `record:<name>`,
|
|
308
|
+
`mutation:<record>.<name>`, split at its last dot, so a mutation whose
|
|
309
|
+
name holds a dot is named only through the API's structured targets);
|
|
310
|
+
`--reset <record>` takes a seeded record's
|
|
311
|
+
file as its state again whatever has mutated it since; `--dry-run` lists
|
|
312
|
+
what a sync would take and takes nothing. A sync doesn't run the dataflow:
|
|
313
|
+
`e3 dataflow run` does, and the sync prints the command when anything
|
|
314
|
+
changed.
|
|
315
|
+
- **`files push`** puts files from disk into a source and takes them in
|
|
316
|
+
(Pushing files from disk, above).
|
|
317
|
+
- **`files status`** shows each mapping with its shape, its type, the file
|
|
318
|
+
it holds and the commit it made (or `pending` for a record not yet
|
|
319
|
+
seeded), where each source is bound, and the last sync.
|
|
148
320
|
|
|
149
321
|
A sync is safe to repeat and to stop. Each input is remembered only once it
|
|
150
322
|
is written, so a sync that stops part way takes the rest next time.
|
|
@@ -229,8 +401,11 @@ is read. So a sync holds one batch of files and the pieces it changes,
|
|
|
229
401
|
however large the folder; the first sync of 6 GB of files holds 256 MiB at a
|
|
230
402
|
time, and a sync stopped part way takes the rest next time.
|
|
231
403
|
|
|
232
|
-
A `file` or `latest` input is held whole while it is written: a
|
|
233
|
-
needs some 3 GiB of memory.
|
|
404
|
+
A `file` or `latest` input with no type is held whole while it is written: a
|
|
405
|
+
1 GiB file needs some 3 GiB of memory. A typed collection is never held
|
|
406
|
+
whole: in the cloud it is staged in the data bucket as it streams and taken
|
|
407
|
+
in by intake units on the sandbox functions; locally it is streamed to the
|
|
408
|
+
repository's own scratch and taken in by e3's door for a file.
|
|
234
409
|
|
|
235
410
|
### On a local repository
|
|
236
411
|
|
|
@@ -248,8 +423,11 @@ and profile it was: `aws sso login --profile <name>` renews it.
|
|
|
248
423
|
|
|
249
424
|
`@elaraai/e3-cloud-files/check` holds the checks without the authoring:
|
|
250
425
|
|
|
251
|
-
- `checkFiles(manifest,
|
|
252
|
-
(`
|
|
426
|
+
- `checkFiles(manifest, surface)`: the declarations against a package's
|
|
427
|
+
inputs, records and mutations (`readPackageSurface(read, packageHash)`
|
|
428
|
+
reads them from its objects): an input writable and of the mapping's type,
|
|
429
|
+
a record of the file's type, a mutation of the record taking one argument
|
|
430
|
+
of the file's type;
|
|
253
431
|
- `checkBindings(manifest, bindings, policy)`: bindings against the
|
|
254
432
|
declarations, and in the cloud against the sources the deployment grants.
|
|
255
433
|
|
|
@@ -260,13 +438,27 @@ rendered as one line by `filesProblemText`.
|
|
|
260
438
|
|
|
261
439
|
In `@elaraai/e3-cloud-types`, re-exported here:
|
|
262
440
|
|
|
263
|
-
- `FilesManifestType`: the sources' names, and each mapping's
|
|
264
|
-
|
|
441
|
+
- `FilesManifestType`: the sources' names, and each mapping's target
|
|
442
|
+
(`FilesTargetType`: `input`, `record` or `mutation`), source and shape
|
|
443
|
+
(`FilesShapeType`: `file` and `latest`, each with an optional `type`;
|
|
444
|
+
`folder`, `listing`).
|
|
265
445
|
- `FilesBindingType`: a source's URL, region, and, locally, profile and
|
|
266
446
|
`latest`.
|
|
267
|
-
- `FilesSyncReportType`: per
|
|
268
|
-
unchanged and refused
|
|
269
|
-
|
|
447
|
+
- `FilesSyncReportType`: per target, the files new, changed, removed,
|
|
448
|
+
unchanged and refused (`FilesRefusalReasonType`: `too_large`,
|
|
449
|
+
`segment_too_large`, `unreadable`, `not_beast2`, `type_mismatch`,
|
|
450
|
+
`mutation`, `missing`), the bytes taken in, whether it was written, the
|
|
451
|
+
typed value taken (`FilesTakenType`: its rows), the commit a seed or feed
|
|
452
|
+
made, and why a seed was kept or a feed waits.
|
|
453
|
+
- `FilesStatusType` (each mapping with what it holds and whether its record
|
|
454
|
+
is pending), `FilesProblemType`, the push's `FilesPushRequestType` and
|
|
455
|
+
`FilesPushPlanType`.
|
|
456
|
+
- Values stored by the first version (mappings by input, a refusal as one
|
|
457
|
+
text) are read by their header's type and lifted
|
|
458
|
+
(`@elaraai/e3-cloud-files/compat`): declarations, ledgers and reports a
|
|
459
|
+
workspace stored before this version read as they were, and are written
|
|
460
|
+
again in the current form on their next write. A local repository's state
|
|
461
|
+
file says its version (`version: 2`).
|
|
270
462
|
- The inputs' types: `FilesFileType` (`Blob`), `FilesFolderType`
|
|
271
463
|
(`Dict<String, Blob>`), `FilesListingType`.
|
|
272
464
|
|
|
@@ -276,10 +468,11 @@ In `@elaraai/e3-cloud-types`, re-exported here:
|
|
|
276
468
|
|---|---|
|
|
277
469
|
| `@elaraai/e3-cloud-files` | Authoring: `Files.source`, the shapes, `Files.manifest`, and the types |
|
|
278
470
|
| `@elaraai/e3-cloud-files/check` | The checks, without the authoring |
|
|
279
|
-
| `@elaraai/e3-cloud-files/sync` | The sync (`syncFiles`) over what reads, writes and remembers, and
|
|
280
|
-
| `@elaraai/e3-cloud-files/storage` | The sync's writer over an e3 store (needs `@elaraai/e3-core`) |
|
|
471
|
+
| `@elaraai/e3-cloud-files/sync` | The sync (`syncFiles`) over what reads, writes and remembers, the push's plan (`planPush`), and the ports in memory for tests |
|
|
472
|
+
| `@elaraai/e3-cloud-files/storage` | The sync's writer over an e3 store, and the cloud's over a staging (needs `@elaraai/e3-core`) |
|
|
281
473
|
| `@elaraai/e3-cloud-files/s3` | The sync's reader of a bucket (needs `@aws-sdk/client-s3`) |
|
|
282
|
-
| `@elaraai/e3-cloud-files/
|
|
474
|
+
| `@elaraai/e3-cloud-files/compat` | The first version's stored types, and their lift into the current ones |
|
|
475
|
+
| `@elaraai/e3-cloud-files/examples/*` | The worked examples (`orders`, `typed`) and the decoding examples (`decode`) |
|
|
283
476
|
|
|
284
477
|
## License
|
|
285
478
|
|
package/dist/src/authoring.d.ts
CHANGED
|
@@ -3,29 +3,55 @@
|
|
|
3
3
|
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
4
|
*/
|
|
5
5
|
/**
|
|
6
|
-
* Declaring which inputs come from files
|
|
6
|
+
* Declaring which inputs come from files, and which records are seeded or
|
|
7
|
+
* fed from them.
|
|
7
8
|
*
|
|
8
9
|
* ```ts
|
|
9
10
|
* export const upload = Files.source('upload');
|
|
10
|
-
* export const rates = upload.file('rates', { key: 'rates.json' });
|
|
11
|
-
* export const
|
|
12
|
-
* export const
|
|
11
|
+
* export const rates = upload.file('rates', { key: 'rates.json' }); // a Blob
|
|
12
|
+
* export const orders = upload.file('orders', { key: 'orders.beast2', type: ArrayType(Order) }); // an Array<Order>
|
|
13
|
+
* export const stock = upload.latest('stock', { prefix: 'stock/', type: ArrayType(Stock) });
|
|
14
|
+
* export const files = upload.folder('invoices', { prefix: 'invoices/', include: '*.pdf' });
|
|
15
|
+
* upload.mutation(priceBook.mutations.load_prices, { key: 'prices.beast2' }); // each new file calls it
|
|
13
16
|
* export default Files.manifest([upload]);
|
|
14
17
|
* ```
|
|
15
18
|
*
|
|
16
|
-
* Each declaration is an ordinary `e3.input` with no default, so the
|
|
17
|
-
* is plain e3 and tasks take these inputs as any other. A sync fills
|
|
18
|
-
* until the first, they are unassigned.
|
|
19
|
+
* Each input declaration is an ordinary `e3.input` with no default, so the
|
|
20
|
+
* package is plain e3 and tasks take these inputs as any other. A sync fills
|
|
21
|
+
* them; until the first, they are unassigned. A `file` or `latest` that
|
|
22
|
+
* names a `type` is an input of that type, filled with the beast2 value the
|
|
23
|
+
* file holds; one that names none is a `Blob`, filled with the file's
|
|
24
|
+
* bytes. A record seeded here is an `e3.record` with no initial value,
|
|
25
|
+
* declared whole with its indexes, mutations and migrations. A mutation fed
|
|
26
|
+
* here is one a record of the package declares, a `reduce` or an `edit`
|
|
27
|
+
* taking one argument (`record.mutations.<name>`): a sync calls it once with
|
|
28
|
+
* each new file's value. What each source's files fill is the
|
|
19
29
|
* {@link Files.manifest}, which `e3-cloud files deploy` deploys beside the
|
|
20
30
|
* package.
|
|
21
31
|
*
|
|
22
32
|
* @packageDocumentation
|
|
23
33
|
*/
|
|
24
|
-
import e3 from '@elaraai/e3';
|
|
34
|
+
import e3, { type IndexDraft, type MutationDef, type MutationDraft, type RecordConfig } from '@elaraai/e3';
|
|
25
35
|
import { type EastType } from '@elaraai/east';
|
|
26
36
|
import { FilesFileType, FilesFolderType, FilesListingType, type FilesManifest, type FilesMapping, type FilesProblem } from '@elaraai/e3-cloud-types';
|
|
27
|
-
/** The input a declaration makes: an `e3.input` of its
|
|
37
|
+
/** The input a declaration makes: an `e3.input` of its type, with no default. */
|
|
28
38
|
export type FilesInput<Name extends string, T extends EastType> = ReturnType<typeof e3.input<Name, T>>;
|
|
39
|
+
/** The mutation a feed declaration hands back: its record's definition, as it was given. */
|
|
40
|
+
export type FilesMutation<M extends MutationDef> = M;
|
|
41
|
+
/**
|
|
42
|
+
* What a record seeded from a file is declared with beside its file: its
|
|
43
|
+
* indexes, mutations and migrations, as `e3.record` takes them, and never an
|
|
44
|
+
* initial value, since its file is its first state.
|
|
45
|
+
*
|
|
46
|
+
* @typeParam T - The record's state type
|
|
47
|
+
* @typeParam I - Its indexes, as `indexes` returns them
|
|
48
|
+
* @typeParam M - Its mutations, as `mutations` returns them
|
|
49
|
+
*/
|
|
50
|
+
export type FilesRecordParts<T extends EastType, I extends object, M extends object> = Omit<RecordConfig<T, I, M>, 'initial'>;
|
|
51
|
+
/** One file, by its key relative to where the workspace binds the source. */
|
|
52
|
+
export interface FilesKeySpec {
|
|
53
|
+
readonly key: string;
|
|
54
|
+
}
|
|
29
55
|
/** The selection of `latest`, `folder` and `listing`: a prefix, and a glob over the paths under it. */
|
|
30
56
|
export interface FilesSelection {
|
|
31
57
|
/** The folder, relative to where the workspace binds the source: `''` for all of it, else ending with `/` */
|
|
@@ -33,6 +59,14 @@ export interface FilesSelection {
|
|
|
33
59
|
/** A glob over paths relative to the prefix: `*` within a segment, `**` across segments, `?` one character, `[a-z]` one of a set, `{csv,json}` any of its alternatives */
|
|
34
60
|
readonly include?: string;
|
|
35
61
|
}
|
|
62
|
+
/** One file, typed: the beast2 value of `type` it holds. */
|
|
63
|
+
export interface FilesTypedKeySpec<T extends EastType> extends FilesKeySpec {
|
|
64
|
+
readonly type: T;
|
|
65
|
+
}
|
|
66
|
+
/** The newest file of a selection, typed. */
|
|
67
|
+
export interface FilesTypedSelection<T extends EastType> extends FilesSelection {
|
|
68
|
+
readonly type: T;
|
|
69
|
+
}
|
|
36
70
|
/** Thrown when declarations are not well-formed, naming every problem. */
|
|
37
71
|
export declare class FilesDeclarationError extends Error {
|
|
38
72
|
/** Every problem */
|
|
@@ -41,7 +75,7 @@ export declare class FilesDeclarationError extends Error {
|
|
|
41
75
|
}
|
|
42
76
|
/**
|
|
43
77
|
* A source of files: a name each workspace binds to a bucket and a prefix,
|
|
44
|
-
* and the inputs its files fill.
|
|
78
|
+
* and the inputs, records and mutations its files fill.
|
|
45
79
|
*/
|
|
46
80
|
export declare class FilesSource {
|
|
47
81
|
#private;
|
|
@@ -49,27 +83,30 @@ export declare class FilesSource {
|
|
|
49
83
|
readonly name: string;
|
|
50
84
|
/** @internal Use {@link Files.source}. */
|
|
51
85
|
constructor(name: string);
|
|
52
|
-
/** The
|
|
86
|
+
/** The targets this source fills, in the order they were declared. */
|
|
53
87
|
get mappings(): readonly FilesMapping[];
|
|
54
88
|
/**
|
|
55
|
-
* One file
|
|
89
|
+
* One file: the input `.inputs.<input>`. As a `Blob`, its bytes as they
|
|
90
|
+
* are; or, with `type`, as the beast2 value of that type the file holds
|
|
91
|
+
* (a collection of any size, taken in by pieces).
|
|
56
92
|
*
|
|
57
93
|
* @param input - The input's name
|
|
58
|
-
* @param spec - `key`: the file, relative to where the workspace binds the source
|
|
94
|
+
* @param spec - `key`: the file, relative to where the workspace binds the source; `type`: the value it holds
|
|
59
95
|
* @returns The input, for tasks to take
|
|
60
96
|
*/
|
|
61
|
-
file<Name extends string>(input: Name, spec:
|
|
62
|
-
|
|
63
|
-
}): FilesInput<Name, typeof FilesFileType>;
|
|
97
|
+
file<Name extends string>(input: Name, spec: FilesKeySpec): FilesInput<Name, typeof FilesFileType>;
|
|
98
|
+
file<Name extends string, T extends EastType>(input: Name, spec: FilesTypedKeySpec<T>): FilesInput<Name, T>;
|
|
64
99
|
/**
|
|
65
|
-
* The newest file under a prefix
|
|
66
|
-
* `include`, so files named by their time pick the latest.
|
|
100
|
+
* The newest file under a prefix: the greatest key that matches
|
|
101
|
+
* `include`, so files named by their time pick the latest. As a `Blob`,
|
|
102
|
+
* or, with `type`, as the value it holds.
|
|
67
103
|
*
|
|
68
104
|
* @param input - The input's name
|
|
69
|
-
* @param spec - The prefix,
|
|
105
|
+
* @param spec - The prefix, a glob over the paths under it, and the value's type
|
|
70
106
|
* @returns The input, for tasks to take
|
|
71
107
|
*/
|
|
72
108
|
latest<Name extends string>(input: Name, spec: FilesSelection): FilesInput<Name, typeof FilesFileType>;
|
|
109
|
+
latest<Name extends string, T extends EastType>(input: Name, spec: FilesTypedSelection<T>): FilesInput<Name, T>;
|
|
73
110
|
/**
|
|
74
111
|
* Every file under a prefix that matches `include`, as a
|
|
75
112
|
* `Dict<String, Blob>` keyed by each file's path relative to the prefix.
|
|
@@ -89,6 +126,37 @@ export declare class FilesSource {
|
|
|
89
126
|
* @returns The input, for tasks to take
|
|
90
127
|
*/
|
|
91
128
|
listing<Name extends string>(input: Name, spec: FilesSelection): FilesInput<Name, typeof FilesListingType>;
|
|
129
|
+
/**
|
|
130
|
+
* A record seeded from a file: declared with no initial value, so it is
|
|
131
|
+
* pending until its file arrives, which is its first state (`$init`;
|
|
132
|
+
* nothing runs on an empty state first, and its dependents wait as they
|
|
133
|
+
* wait on an unset input); seeded again (`$seed`) while nothing has
|
|
134
|
+
* mutated it since, or when a sync is asked to reset it; kept otherwise.
|
|
135
|
+
* It is declared whole, as any record is: its indexes, mutations and
|
|
136
|
+
* migrations are given here, as `e3.record` takes them.
|
|
137
|
+
*
|
|
138
|
+
* @param name - The record's name
|
|
139
|
+
* @param type - Its state's type, which the file must hold
|
|
140
|
+
* @param spec - One file (`key`), or the newest under a prefix (`prefix`, `include`)
|
|
141
|
+
* @param parts - Its indexes, mutations and migrations, as `e3.record` takes them; none when omitted
|
|
142
|
+
* @returns The record, for tasks to take and its mutations to be fed: `e3.record(name, type, parts)`
|
|
143
|
+
* @throws {FilesDeclarationError} When the name is one e3 refuses, or the spec is
|
|
144
|
+
* @throws {Error} When `parts` holds an initial value, or holds what `e3.record` refuses, naming it
|
|
145
|
+
*/
|
|
146
|
+
record<Name extends string, T extends EastType, I extends Readonly<Record<string, IndexDraft>> = Record<never, never>, M extends Readonly<Record<string, MutationDraft>> = Record<never, never>>(name: Name, type: T, spec: FilesKeySpec | FilesSelection, parts?: FilesRecordParts<T, I, M>): ReturnType<typeof e3.record<Name, T, I, M>>;
|
|
147
|
+
/**
|
|
148
|
+
* A record's mutation, fed: a sync calls it once with each new file's
|
|
149
|
+
* value, which must be of the mutation's one argument's type. The
|
|
150
|
+
* mutation is one a record of the package declares, a `reduce` or an
|
|
151
|
+
* `edit` taking exactly one argument; a `patch` mutation takes a patch,
|
|
152
|
+
* not a value, so it cannot be fed.
|
|
153
|
+
*
|
|
154
|
+
* @param def - The mutation, as its record holds it: `record.mutations.<name>`, a `mutation.reduce(...)` or `mutation.edit(...)` of its declaration
|
|
155
|
+
* @param spec - One file (`key`), or the newest under a prefix (`prefix`, `include`)
|
|
156
|
+
* @returns The mutation, as given
|
|
157
|
+
* @throws {FilesDeclarationError} When the mutation takes other than one argument, or is a `patch`
|
|
158
|
+
*/
|
|
159
|
+
mutation<M extends MutationDef>(def: M, spec: FilesKeySpec | FilesSelection): FilesMutation<M>;
|
|
92
160
|
}
|
|
93
161
|
/** Declaring files. */
|
|
94
162
|
export declare const Files: {
|
|
@@ -96,18 +164,18 @@ export declare const Files: {
|
|
|
96
164
|
* A source of files, which each workspace binds to a bucket and a prefix.
|
|
97
165
|
*
|
|
98
166
|
* @param name - Its name, unique among a solution's sources
|
|
99
|
-
* @returns The source, whose methods declare
|
|
167
|
+
* @returns The source, whose methods declare what its files fill
|
|
100
168
|
* @throws {FilesDeclarationError} When the name is one e3 refuses
|
|
101
169
|
*/
|
|
102
170
|
source(name: string): FilesSource;
|
|
103
171
|
/**
|
|
104
172
|
* The declarations `e3-cloud files deploy` deploys: the sources, and the
|
|
105
|
-
* inputs their files fill.
|
|
173
|
+
* inputs, records and mutations their files fill.
|
|
106
174
|
*
|
|
107
175
|
* @param sources - Every source
|
|
108
176
|
* @returns The manifest, an East value of `FilesManifestType`
|
|
109
177
|
* @throws {FilesDeclarationError} When two sources share a name, or two
|
|
110
|
-
* mappings fill one
|
|
178
|
+
* mappings fill one target
|
|
111
179
|
*/
|
|
112
180
|
manifest(sources: readonly FilesSource[]): FilesManifest;
|
|
113
181
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"authoring.d.ts","sourceRoot":"","sources":["../../src/authoring.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH
|
|
1
|
+
{"version":3,"file":"authoring.d.ts","sourceRoot":"","sources":["../../src/authoring.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,EAAE,EAAE,KAAK,UAAU,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3G,OAAO,EAAwC,KAAK,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEpF,OAAO,EACL,aAAa,EAAE,eAAe,EAAE,gBAAgB,EAEhD,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,KAAK,YAAY,EACzD,MAAM,yBAAyB,CAAC;AAGjC,iFAAiF;AACjF,MAAM,MAAM,UAAU,CAAC,IAAI,SAAS,MAAM,EAAE,CAAC,SAAS,QAAQ,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAEvG,4FAA4F;AAC5F,MAAM,MAAM,aAAa,CAAC,CAAC,SAAS,WAAW,IAAI,CAAC,CAAC;AAErD;;;;;;;;GAQG;AACH,MAAM,MAAM,gBAAgB,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,MAAM,EAAE,CAAC,SAAS,MAAM,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;AAE9H,6EAA6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED,uGAAuG;AACvG,MAAM,WAAW,cAAc;IAC7B,6GAA6G;IAC7G,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,0KAA0K;IAC1K,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,4DAA4D;AAC5D,MAAM,WAAW,iBAAiB,CAAC,CAAC,SAAS,QAAQ,CAAE,SAAQ,YAAY;IACzE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;CAClB;AAED,6CAA6C;AAC7C,MAAM,WAAW,mBAAmB,CAAC,CAAC,SAAS,QAAQ,CAAE,SAAQ,cAAc;IAC7E,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;CAClB;AAED,0EAA0E;AAC1E,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,oBAAoB;IACpB,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;gBAE/B,QAAQ,EAAE,SAAS,YAAY,EAAE;CAK9C;AAmDD;;;GAGG;AACH,qBAAa,WAAW;;IACtB,wBAAwB;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAGtB,0CAA0C;gBAC9B,IAAI,EAAE,MAAM;IAKxB,sEAAsE;IACtE,IAAI,QAAQ,IAAI,SAAS,YAAY,EAAE,CAEtC;IAcD;;;;;;;;OAQG;IACH,IAAI,CAAC,IAAI,SAAS,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,GAAG,UAAU,CAAC,IAAI,EAAE,OAAO,aAAa,CAAC;IAClG,IAAI,CAAC,IAAI,SAAS,MAAM,EAAE,CAAC,SAAS,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IAQ3G;;;;;;;;OAQG;IACH,MAAM,CAAC,IAAI,SAAS,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,GAAG,UAAU,CAAC,IAAI,EAAE,OAAO,aAAa,CAAC;IACtG,MAAM,CAAC,IAAI,SAAS,MAAM,EAAE,CAAC,SAAS,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IAQ/G;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,SAAS,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,GAAG,UAAU,CAAC,IAAI,EAAE,OAAO,eAAe,CAAC;IAIxG;;;;;;;;OAQG;IACH,OAAO,CAAC,IAAI,SAAS,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,GAAG,UAAU,CAAC,IAAI,EAAE,OAAO,gBAAgB,CAAC;IAI1G;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CACJ,IAAI,SAAS,MAAM,EACnB,CAAC,SAAS,QAAQ,EAClB,CAAC,SAAS,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,EACrE,CAAC,SAAS,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,EACxE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,GAAG,cAAc,EAAE,KAAK,CAAC,EAAE,gBAAgB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAY3I;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,CAAC,SAAS,WAAW,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,GAAG,cAAc,GAAG,aAAa,CAAC,CAAC,CAAC;CAW/F;AAED,uBAAuB;AACvB,eAAO,MAAM,KAAK;IAChB;;;;;;OAMG;iBACU,MAAM,GAAG,WAAW;IAIjC;;;;;;;;OAQG;sBACe,SAAS,WAAW,EAAE,GAAG,aAAa;CAmBzD,CAAC"}
|