@tinyshed/tinystore 0.1.0-rc.4 → 0.1.0-rc.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -37
- package/bin/tinystore.js +12 -1
- package/dist/backup.d.ts +33 -0
- package/dist/backup.js +49 -0
- package/dist/blobs.d.ts +160 -79
- package/dist/blobs.js +510 -217
- package/dist/channels.d.ts +90 -0
- package/dist/channels.js +291 -0
- package/dist/clock.js +3 -3
- package/dist/config.d.ts +209 -86
- package/dist/config.js +581 -538
- package/dist/connection.d.ts +64 -2
- package/dist/connection.js +201 -16
- package/dist/console.d.ts +27 -3
- package/dist/console.js +165 -12
- package/dist/dotenv.d.ts +7 -0
- package/dist/dotenv.js +79 -0
- package/dist/errors.d.ts +61 -1
- package/dist/errors.js +81 -1
- package/dist/handles.d.ts +44 -2
- package/dist/handles.js +81 -2
- package/dist/index.d.ts +17 -11
- package/dist/index.js +5 -3
- package/dist/instruments.d.ts +177 -0
- package/dist/instruments.js +819 -0
- package/dist/jobs.d.ts +335 -239
- package/dist/jobs.js +716 -536
- package/dist/json.d.ts +2 -0
- package/dist/json.js +122 -0
- package/dist/kv.d.ts +130 -243
- package/dist/kv.js +273 -640
- package/dist/limits.d.ts +80 -0
- package/dist/limits.js +139 -0
- package/dist/locks.d.ts +62 -0
- package/dist/locks.js +237 -0
- package/dist/logger.d.ts +45 -9
- package/dist/logger.js +150 -38
- package/dist/metrics.d.ts +210 -209
- package/dist/metrics.js +319 -556
- package/dist/once.d.ts +19 -18
- package/dist/once.js +74 -63
- package/dist/query.d.ts +181 -0
- package/dist/query.js +788 -0
- package/dist/records.d.ts +241 -150
- package/dist/records.js +507 -291
- package/dist/runtime/addon.d.ts +14 -0
- package/dist/runtime/addon.js +69 -0
- package/dist/runtime/bun.js +5 -7
- package/dist/runtime/node.js +5 -7
- package/dist/runtime/pipe.d.ts +17 -0
- package/dist/runtime/pipe.js +133 -0
- package/dist/runtime/shared.d.ts +6 -0
- package/dist/runtime/shared.js +19 -0
- package/dist/runtime.d.ts +5 -2
- package/dist/search.d.ts +189 -0
- package/dist/search.js +281 -0
- package/dist/session.d.ts +31 -3
- package/dist/session.js +153 -41
- package/dist/sql.d.ts +228 -78
- package/dist/sql.js +745 -240
- package/dist/store.d.ts +244 -24
- package/dist/store.js +270 -67
- package/dist/time.d.ts +5 -1
- package/dist/time.js +22 -2
- package/dist/tx.d.ts +61 -0
- package/dist/tx.js +314 -0
- package/dist/wire/codec.d.ts +23 -7
- package/dist/wire/codec.js +56 -35
- package/dist/wire/frame.d.ts +2 -0
- package/dist/wire/frame.js +31 -14
- package/dist/wire/messages.d.ts +259 -0
- package/dist/wire/messages.js +22 -0
- package/dist/wire/msgpack.d.ts +49 -1
- package/dist/wire/msgpack.js +293 -56
- package/dist/wire/protocol.d.ts +2683 -0
- package/dist/wire/protocol.js +10846 -0
- package/package.json +39 -38
- package/src/backup.ts +81 -0
- package/src/blobs.ts +678 -301
- package/src/channels.ts +377 -0
- package/src/clock.ts +4 -4
- package/src/config.ts +902 -612
- package/src/connection.ts +279 -21
- package/src/console.ts +180 -12
- package/src/dotenv.ts +82 -0
- package/src/errors.ts +105 -0
- package/src/handles.ts +157 -8
- package/src/index.ts +158 -52
- package/src/instruments.ts +1042 -0
- package/src/jobs.ts +1017 -733
- package/src/json.ts +133 -0
- package/src/kv.ts +424 -847
- package/src/limits.ts +202 -0
- package/src/locks.ts +320 -0
- package/src/logger.ts +209 -40
- package/src/metrics.ts +557 -752
- package/src/once.ts +86 -83
- package/src/query.ts +1005 -0
- package/src/records.ts +795 -418
- package/src/runtime/addon.ts +99 -0
- package/src/runtime/bun.ts +5 -7
- package/src/runtime/node.ts +5 -7
- package/src/runtime/pipe.ts +178 -0
- package/src/runtime/shared.ts +20 -0
- package/src/runtime.ts +5 -2
- package/src/search.ts +480 -0
- package/src/session.ts +187 -41
- package/src/sql.ts +993 -306
- package/src/store.ts +473 -78
- package/src/time.ts +48 -3
- package/src/tx.ts +426 -0
- package/src/wire/codec.ts +78 -37
- package/src/wire/frame.ts +44 -14
- package/src/wire/messages.ts +25 -0
- package/src/wire/msgpack.ts +327 -56
- package/src/wire/protocol.ts +11709 -0
- package/dist/limiter.d.ts +0 -45
- package/dist/limiter.js +0 -68
- package/dist/quota.d.ts +0 -44
- package/dist/quota.js +0 -98
- package/src/limiter.ts +0 -99
- package/src/quota.ts +0 -137
package/dist/blobs.js
CHANGED
|
@@ -1,299 +1,592 @@
|
|
|
1
|
-
// The application's files
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
|
|
1
|
+
// The application's files, as docs/guides/blobs/README.md has them: files by path in
|
|
2
|
+
// named sets, written whole and replaced whole, over the protocol. A file's
|
|
3
|
+
// bytes go in pieces within the credit of the side that takes them, and a
|
|
4
|
+
// whole read ends CorruptError before its last piece when its bytes changed.
|
|
5
|
+
import { pieceOf } from './connection.js';
|
|
5
6
|
import { ClosedError, InvalidError, OutcomeUnknownError } from './errors.js';
|
|
6
|
-
import { checkName, handleOn } from './handles.js';
|
|
7
|
-
import {
|
|
7
|
+
import { checkName, handleOn, viaLink } from './handles.js';
|
|
8
|
+
import { ConnectionLostError, watch } from './session.js';
|
|
8
9
|
import { dateOf, ms, unixMs } from './time.js';
|
|
9
|
-
import {
|
|
10
|
-
|
|
11
|
-
const
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
checkName(name, 'bucket');
|
|
19
|
-
const open = BlobsBucket.encode({
|
|
20
|
-
name,
|
|
21
|
-
defaultTtl: options?.defaultTtl === undefined ? undefined : ms(options.defaultTtl),
|
|
22
|
-
maxSize: options?.maxSize,
|
|
23
|
-
});
|
|
24
|
-
return new BlobBucket(this.#link, name, open, []);
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
function ownerText(owner) {
|
|
28
|
-
if (typeof owner === 'number' || typeof owner === 'bigint') {
|
|
29
|
-
if (typeof owner === 'number' && !Number.isSafeInteger(owner)) {
|
|
30
|
-
throw new InvalidError(`the owner ${owner} is not an integer; an owner is text or an integer`);
|
|
31
|
-
}
|
|
32
|
-
return String(owner);
|
|
33
|
-
}
|
|
34
|
-
return owner;
|
|
35
|
-
}
|
|
36
|
-
const lossy = new TextDecoder();
|
|
37
|
-
function objectOf(o) {
|
|
38
|
-
const meta = {};
|
|
39
|
-
for (const [name, value] of Object.entries(o.meta ?? {})) {
|
|
40
|
-
meta[name] = typeof value === 'string' ? value : lossy.decode(value);
|
|
41
|
-
}
|
|
42
|
-
return {
|
|
43
|
-
key: o.key ?? '',
|
|
44
|
-
size: o.size ?? 0,
|
|
45
|
-
etag: o.etag ?? '',
|
|
46
|
-
contentType: o.contentType,
|
|
47
|
-
modified: new Date(o.modified ?? 0),
|
|
48
|
-
expires: dateOf(o.expires),
|
|
49
|
-
meta,
|
|
50
|
-
};
|
|
51
|
-
}
|
|
52
|
-
function writeFields(options) {
|
|
53
|
-
return {
|
|
54
|
-
contentType: options?.contentType,
|
|
55
|
-
meta: options?.meta,
|
|
10
|
+
import { BlobsAt, BlobsDelete, BlobsExpire, BlobsFolder, BlobsFound, BlobsGot, BlobsHead, BlobsInfo, BlobsList, BlobsMove, BlobsOpen, BlobsPage, BlobsPiece, BlobsRead, BlobsUsage, BlobsWrite, BlobsWritten, methods, } from './wire/protocol.js';
|
|
11
|
+
const encoder = new TextEncoder();
|
|
12
|
+
const decoder = new TextDecoder();
|
|
13
|
+
const nothing = new Uint8Array(0);
|
|
14
|
+
/** Files as `store.files` opens them. */
|
|
15
|
+
export function openFiles(link, name, options) {
|
|
16
|
+
checkName(name, 'files');
|
|
17
|
+
const open = BlobsOpen.encode({
|
|
18
|
+
name,
|
|
56
19
|
ttl: options?.ttl === undefined ? undefined : ms(options.ttl),
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
};
|
|
20
|
+
maxFileSize: options?.maxFileSize === undefined ? undefined : bytesOf(options.maxFileSize),
|
|
21
|
+
});
|
|
22
|
+
return new Files(link, name, open, []);
|
|
61
23
|
}
|
|
62
|
-
/**
|
|
63
|
-
|
|
24
|
+
/**
|
|
25
|
+
* A named set of files by path, or a folder of it, whose calls are the
|
|
26
|
+
* folder's: a path in it is the folder's segments and then its own.
|
|
27
|
+
*/
|
|
28
|
+
export class Files {
|
|
64
29
|
name;
|
|
65
30
|
#link;
|
|
31
|
+
#via;
|
|
66
32
|
#open;
|
|
67
|
-
#
|
|
68
|
-
|
|
69
|
-
|
|
33
|
+
#folder;
|
|
34
|
+
/** Files come from `store.files`. */
|
|
35
|
+
constructor(link, name, open, folder) {
|
|
70
36
|
this.name = name;
|
|
37
|
+
this.#link = link;
|
|
38
|
+
this.#via = viaLink(link);
|
|
71
39
|
this.#open = open;
|
|
72
|
-
this.#
|
|
40
|
+
this.#folder = folder;
|
|
73
41
|
}
|
|
74
|
-
/** The folder
|
|
75
|
-
|
|
76
|
-
return new
|
|
77
|
-
...this.#
|
|
78
|
-
...
|
|
42
|
+
/** The folder of these segments, one each: `folder('users', 42)` is users/42, never users/420. */
|
|
43
|
+
folder(...segments) {
|
|
44
|
+
return new Files(this.#link, this.name, this.#open, [
|
|
45
|
+
...this.#folder,
|
|
46
|
+
...segments.map(segmentOf),
|
|
79
47
|
]);
|
|
80
48
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
49
|
+
/**
|
|
50
|
+
* Writes the file whole and replaces whatever the path held, returning
|
|
51
|
+
* once it is on disk as the store's durability says. Nobody sees half of
|
|
52
|
+
* it, while it is written or after a crash.
|
|
53
|
+
*/
|
|
54
|
+
async put(path, body, options) {
|
|
55
|
+
// a put that returned wrote: only a create writes nothing
|
|
56
|
+
return (await this.#write(path, body, options, false));
|
|
88
57
|
}
|
|
89
|
-
|
|
90
|
-
|
|
58
|
+
/** Writes the file only where the path holds none, and says whether it did. */
|
|
59
|
+
async create(path, body, options) {
|
|
60
|
+
return (await this.#write(path, body, options, true)) !== undefined;
|
|
91
61
|
}
|
|
92
|
-
/**
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
62
|
+
/**
|
|
63
|
+
* What the file carries, and its bytes: a file up to about a megabyte comes
|
|
64
|
+
* with them, and a larger one's are read when they are read, on its ETag,
|
|
65
|
+
* as S3's GET with If-Match reads them. Undefined when the path holds none.
|
|
66
|
+
*/
|
|
67
|
+
async get(path) {
|
|
68
|
+
const answer = await this.#call('blobs.get', handle => BlobsAt.encode({ handle, folder: this.#folder, path }), 'read');
|
|
69
|
+
const got = BlobsGot.decode(answer);
|
|
70
|
+
if (got.info === undefined) {
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
const info = infoOf(got.info);
|
|
74
|
+
const bytes = got.bytes ?? nothing;
|
|
75
|
+
const whole = bytes.length === info.size ? bytes : undefined;
|
|
76
|
+
return new StoredFile(info, whole, (offset, length) => this.#read(path, info.etag, offset, length));
|
|
96
77
|
}
|
|
97
|
-
|
|
98
|
-
|
|
78
|
+
/** What the file carries and none of its bytes; undefined when the path holds none. */
|
|
79
|
+
async head(path) {
|
|
80
|
+
const answer = await this.#call('blobs.head', handle => BlobsAt.encode({ handle, folder: this.#folder, path }), 'read');
|
|
81
|
+
const head = BlobsHead.decode(answer);
|
|
82
|
+
return head.info === undefined ? undefined : infoOf(head.info);
|
|
99
83
|
}
|
|
100
|
-
/**
|
|
101
|
-
async
|
|
102
|
-
|
|
84
|
+
/** Deletes the file, or with ifMatch only that version of it; no file there is not an error. */
|
|
85
|
+
async delete(path, options) {
|
|
86
|
+
await this.#call('blobs.delete', handle => BlobsDelete.encode({ handle, folder: this.#folder, path, ifMatch: options?.ifMatch }), 'write');
|
|
103
87
|
}
|
|
104
|
-
/**
|
|
105
|
-
|
|
106
|
-
|
|
88
|
+
/**
|
|
89
|
+
* A copy at `to` that shares the bytes, which are not copied. A file at
|
|
90
|
+
* `to` is ConflictError unless ifMatch names the version to replace.
|
|
91
|
+
*/
|
|
92
|
+
copy(from, to, options) {
|
|
93
|
+
return this.#move('blobs.copy', from, to, options?.ifMatch);
|
|
107
94
|
}
|
|
108
|
-
/**
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
95
|
+
/**
|
|
96
|
+
* The file moved to `to` in one write: a file there is ConflictError
|
|
97
|
+
* unless ifMatch names the version to replace, and none at `from` NotFoundError.
|
|
98
|
+
*/
|
|
99
|
+
rename(from, to, options) {
|
|
100
|
+
return this.#move('blobs.rename', from, to, options?.ifMatch);
|
|
112
101
|
}
|
|
113
|
-
/**
|
|
114
|
-
async
|
|
115
|
-
|
|
102
|
+
/** Sets when the file expires, a span from now or a moment; says whether there was a file. */
|
|
103
|
+
async expire(path, after) {
|
|
104
|
+
const span = after instanceof Date ? Math.max(0, unixMs(after) - Date.now()) : ms(after);
|
|
105
|
+
const answer = await this.#call('blobs.expire', handle => BlobsExpire.encode({ handle, folder: this.#folder, path, after: span }), 'write');
|
|
106
|
+
return BlobsFound.decode(answer).found === true;
|
|
116
107
|
}
|
|
117
|
-
/** A page of the
|
|
118
|
-
async
|
|
119
|
-
const
|
|
120
|
-
|
|
108
|
+
/** A page of the files in the folder and in the folders in it, in the byte order of their paths. */
|
|
109
|
+
async list(options) {
|
|
110
|
+
const answer = await this.#call('blobs.list', handle => BlobsList.encode({
|
|
111
|
+
handle,
|
|
112
|
+
folder: this.#folder,
|
|
113
|
+
prefix: options?.prefix ?? '',
|
|
121
114
|
after: options?.after,
|
|
122
115
|
limit: options?.limit,
|
|
123
|
-
}))
|
|
124
|
-
const page = BlobsPage.decode(
|
|
125
|
-
return {
|
|
126
|
-
items: got.items.map(item => objectOf(BlobsObject.decode(item))),
|
|
127
|
-
next: page.more === true ? (page.after ?? '') : undefined,
|
|
128
|
-
};
|
|
116
|
+
}), 'read');
|
|
117
|
+
const page = BlobsPage.decode(answer);
|
|
118
|
+
return { files: (page.files ?? []).map(infoOf), next: page.next };
|
|
129
119
|
}
|
|
120
|
+
/** Every file of the folder, a page at a time, holding nothing between pages. */
|
|
130
121
|
async *all(options) {
|
|
131
122
|
let after;
|
|
132
123
|
for (;;) {
|
|
133
|
-
const page = await this.
|
|
134
|
-
yield* page.
|
|
124
|
+
const page = await this.list({ prefix: options?.prefix, after });
|
|
125
|
+
yield* page.files;
|
|
135
126
|
if (page.next === undefined) {
|
|
136
127
|
return;
|
|
137
128
|
}
|
|
138
129
|
after = page.next;
|
|
139
130
|
}
|
|
140
131
|
}
|
|
132
|
+
/** The files in the folder and their bytes, as it is now: it counts, it does not limit. */
|
|
133
|
+
async usage() {
|
|
134
|
+
const answer = await this.#call('blobs.usage', handle => BlobsFolder.encode({ handle, folder: this.#folder }), 'read');
|
|
135
|
+
const usage = BlobsUsage.decode(answer);
|
|
136
|
+
return { count: usage.count ?? 0, size: usage.size ?? 0 };
|
|
137
|
+
}
|
|
138
|
+
/** Removes the folder's every file and every folder in it, at once for every reader. */
|
|
139
|
+
async clear() {
|
|
140
|
+
await this.#call('blobs.clear', handle => BlobsFolder.encode({ handle, folder: this.#folder }), 'write');
|
|
141
|
+
}
|
|
141
142
|
/**
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
143
|
+
* Begins a file written a piece at a time, which appears at `commit` and
|
|
144
|
+
* never before: `await using upload = await files.upload(path)`, and
|
|
145
|
+
* leaving the block without commit aborts it.
|
|
145
146
|
*/
|
|
146
|
-
async
|
|
147
|
-
const
|
|
147
|
+
async upload(path, options) {
|
|
148
|
+
const connection = await this.#link.connection();
|
|
149
|
+
const write = BlobsWrite.encode({
|
|
150
|
+
...this.#fields(path, options),
|
|
151
|
+
handle: await this.#handle(connection),
|
|
152
|
+
});
|
|
153
|
+
const stream = await connection.session.open(methods['blobs.upload'], write, false);
|
|
154
|
+
return new Upload(stream, pieceOf(connection), options?.signal);
|
|
155
|
+
}
|
|
156
|
+
#handle(connection) {
|
|
157
|
+
return handleOn(connection, methods['blobs.open'], this.#open);
|
|
158
|
+
}
|
|
159
|
+
#call(method, body, idempotence) {
|
|
160
|
+
return this.#via.call(methods['blobs.open'], this.#open, methods[method], body, idempotence);
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* A file's bytes as S3's GET reads them with a range and If-Match: on the
|
|
164
|
+
* ETag a get gave, ConflictError once the path holds another file or none;
|
|
165
|
+
* the whole file checked, a range not. Nothing is asked before the first read.
|
|
166
|
+
*/
|
|
167
|
+
#read(path, ifMatch, offset, length) {
|
|
168
|
+
let stream;
|
|
169
|
+
const open = () => this.#link.run('read', async (connection) => {
|
|
170
|
+
const read = BlobsRead.encode({
|
|
171
|
+
handle: await this.#handle(connection),
|
|
172
|
+
folder: this.#folder,
|
|
173
|
+
path,
|
|
174
|
+
ifMatch,
|
|
175
|
+
offset,
|
|
176
|
+
length,
|
|
177
|
+
});
|
|
178
|
+
const opened = await connection.session.open(methods['blobs.read'], read, true);
|
|
179
|
+
return { opened, first: await opened.next() };
|
|
180
|
+
});
|
|
181
|
+
return new ReadableStream({
|
|
182
|
+
async pull(controller) {
|
|
183
|
+
try {
|
|
184
|
+
if (stream === undefined) {
|
|
185
|
+
// the RESPONSE's piece is no DATA, so it takes none of the credit
|
|
186
|
+
const { opened, first } = await open();
|
|
187
|
+
stream = opened;
|
|
188
|
+
enqueue(controller, first.body);
|
|
189
|
+
if (first.end) {
|
|
190
|
+
controller.close();
|
|
191
|
+
}
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
const event = await stream.next();
|
|
195
|
+
if (event.end) {
|
|
196
|
+
controller.close();
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
stream.consumed(event.body.length);
|
|
200
|
+
enqueue(controller, event.body);
|
|
201
|
+
}
|
|
202
|
+
catch (err) {
|
|
203
|
+
controller.error(err);
|
|
204
|
+
}
|
|
205
|
+
},
|
|
206
|
+
cancel() {
|
|
207
|
+
stream?.cancel();
|
|
208
|
+
},
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
async #move(method, from, to, ifMatch) {
|
|
212
|
+
const answer = await this.#call(method, handle => BlobsMove.encode({ handle, folder: this.#folder, from, to, ifMatch }), 'write');
|
|
213
|
+
return infoOf(BlobsInfo.decode(answer));
|
|
214
|
+
}
|
|
215
|
+
#fields(path, options) {
|
|
216
|
+
return {
|
|
217
|
+
folder: this.#folder,
|
|
218
|
+
path,
|
|
219
|
+
contentType: options?.contentType,
|
|
220
|
+
meta: options?.meta,
|
|
221
|
+
ttl: options?.ttl === undefined ? undefined : ms(options.ttl),
|
|
222
|
+
ifMatch: options?.ifMatch,
|
|
223
|
+
size: options?.size,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Writes a body: in one call when it fits a piece, else as an upload over
|
|
228
|
+
* a stream. What it wrote, nothing when a create found a file.
|
|
229
|
+
*/
|
|
230
|
+
async #write(path, body, options, create) {
|
|
231
|
+
const source = sourceOf(body);
|
|
148
232
|
const fields = {
|
|
149
|
-
...
|
|
233
|
+
...this.#fields(path, options),
|
|
150
234
|
contentType: options?.contentType ?? source.contentType,
|
|
235
|
+
size: options?.size ?? source.size,
|
|
236
|
+
create,
|
|
151
237
|
};
|
|
152
238
|
return this.#link.run('write', async (connection) => {
|
|
153
|
-
const
|
|
154
|
-
|
|
239
|
+
const handle = await this.#handle(connection);
|
|
240
|
+
const piece = pieceOf(connection);
|
|
241
|
+
const whole = await source.whole(piece);
|
|
242
|
+
if (whole !== undefined) {
|
|
243
|
+
const write = BlobsWrite.encode({ ...fields, handle, bytes: whole });
|
|
244
|
+
return writtenOf(await connection.session.call(methods['blobs.put'], write));
|
|
245
|
+
}
|
|
246
|
+
const write = BlobsWrite.encode({ ...fields, handle });
|
|
247
|
+
const stream = await connection.session.open(methods['blobs.upload'], write, false);
|
|
248
|
+
return uploaded(stream, source, piece, options?.signal);
|
|
155
249
|
}, options?.signal);
|
|
156
250
|
}
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* A file written a piece at a time, as `files.upload` begins it: bytes go in
|
|
254
|
+
* with `write`, the file appears at `commit` and never before, and an upload
|
|
255
|
+
* that ends otherwise leaves nothing.
|
|
256
|
+
*/
|
|
257
|
+
export class Upload {
|
|
258
|
+
#stream;
|
|
259
|
+
#piece;
|
|
260
|
+
#unwatch;
|
|
261
|
+
#filled = 0;
|
|
262
|
+
/** the writes so far, one after another, so that two writes never interleave */
|
|
263
|
+
#writing = Promise.resolve();
|
|
264
|
+
#done = false;
|
|
265
|
+
/** Uploads come from `files.upload`. */
|
|
266
|
+
constructor(stream, piece, signal) {
|
|
267
|
+
this.#stream = stream;
|
|
268
|
+
this.#piece = new Uint8Array(piece);
|
|
269
|
+
this.#unwatch = watch(signal, stream);
|
|
270
|
+
}
|
|
271
|
+
/** Takes bytes, text as UTF-8, waiting while the server catches up. */
|
|
272
|
+
write(bytes) {
|
|
273
|
+
if (this.#done) {
|
|
274
|
+
return Promise.reject(new InvalidError('the upload has ended'));
|
|
275
|
+
}
|
|
276
|
+
const written = this.#writing.then(() => this.#take(bytes));
|
|
277
|
+
this.#writing = written.catch(() => { });
|
|
278
|
+
return written;
|
|
279
|
+
}
|
|
157
280
|
/**
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
* long, is not checked against the object's hash.
|
|
281
|
+
* Publishes the file whole, replacing what the path held as `put` does,
|
|
282
|
+
* once it is on disk as the store's durability says.
|
|
161
283
|
*/
|
|
162
|
-
async
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
284
|
+
async commit() {
|
|
285
|
+
if (this.#done) {
|
|
286
|
+
throw new InvalidError('the upload has ended');
|
|
287
|
+
}
|
|
288
|
+
this.#done = true;
|
|
289
|
+
let ending = false;
|
|
290
|
+
try {
|
|
291
|
+
await this.#writing;
|
|
292
|
+
ending = true;
|
|
293
|
+
await this.#send(true);
|
|
294
|
+
// the RESPONSE that began it, and then the file the commit wrote
|
|
295
|
+
await this.#stream.next();
|
|
296
|
+
return writtenOf((await this.#stream.next()).body);
|
|
297
|
+
}
|
|
298
|
+
catch (err) {
|
|
299
|
+
throw lostAs(err, ending);
|
|
300
|
+
}
|
|
301
|
+
finally {
|
|
302
|
+
this.#unwatch();
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
/** Ends the upload, which leaves nothing; after `commit` it does nothing. */
|
|
306
|
+
async abort() {
|
|
307
|
+
if (this.#done) {
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
this.#done = true;
|
|
311
|
+
this.#stream.cancel();
|
|
312
|
+
this.#unwatch();
|
|
313
|
+
}
|
|
314
|
+
async [Symbol.asyncDispose]() {
|
|
315
|
+
await this.abort();
|
|
316
|
+
}
|
|
317
|
+
async #take(bytes) {
|
|
318
|
+
const chunk = typeof bytes === 'string' ? encoder.encode(bytes) : bytes;
|
|
319
|
+
try {
|
|
320
|
+
for (let at = 0; at < chunk.length;) {
|
|
321
|
+
const taken = Math.min(this.#piece.length - this.#filled, chunk.length - at);
|
|
322
|
+
this.#piece.set(chunk.subarray(at, at + taken), this.#filled);
|
|
323
|
+
this.#filled += taken;
|
|
324
|
+
at += taken;
|
|
325
|
+
if (this.#filled === this.#piece.length) {
|
|
326
|
+
await this.#send(false);
|
|
327
|
+
}
|
|
179
328
|
}
|
|
180
|
-
|
|
181
|
-
|
|
329
|
+
}
|
|
330
|
+
catch (err) {
|
|
331
|
+
throw lostAs(err, false);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
/** Sends what the piece holds, the message a copy of it, so the piece fills again at once. */
|
|
335
|
+
async #send(end) {
|
|
336
|
+
const body = BlobsPiece.encode({ bytes: this.#piece.subarray(0, this.#filled) });
|
|
337
|
+
this.#filled = 0;
|
|
338
|
+
await this.#stream.send(body, end);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Bytes of a file, read when they are read, as a Blob's are: `stream()`, or
|
|
343
|
+
* all of them with `bytes()`, `text()`, `json()` or `arrayBuffer()`, which
|
|
344
|
+
* hold them in memory. Each read is a read of its own.
|
|
345
|
+
*/
|
|
346
|
+
export class FileBytes {
|
|
347
|
+
size;
|
|
348
|
+
#stream;
|
|
349
|
+
/** File bytes come from `files.get` and `slice`. */
|
|
350
|
+
constructor(size, stream) {
|
|
351
|
+
this.size = size;
|
|
352
|
+
this.#stream = stream;
|
|
353
|
+
}
|
|
354
|
+
/** Its bytes as they come, the credit of each given back once it is taken. */
|
|
355
|
+
stream() {
|
|
356
|
+
return this.#stream();
|
|
357
|
+
}
|
|
358
|
+
async bytes() {
|
|
359
|
+
const whole = new Uint8Array(this.size);
|
|
360
|
+
let at = 0;
|
|
361
|
+
for await (const piece of this.stream()) {
|
|
362
|
+
whole.set(piece, at);
|
|
363
|
+
at += piece.length;
|
|
364
|
+
}
|
|
365
|
+
return at === whole.length ? whole : whole.slice(0, at);
|
|
366
|
+
}
|
|
367
|
+
async arrayBuffer() {
|
|
368
|
+
return (await this.bytes()).buffer;
|
|
369
|
+
}
|
|
370
|
+
async text() {
|
|
371
|
+
return decoder.decode(await this.bytes());
|
|
372
|
+
}
|
|
373
|
+
async json() {
|
|
374
|
+
return JSON.parse(await this.text());
|
|
182
375
|
}
|
|
183
376
|
}
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
377
|
+
/**
|
|
378
|
+
* A file as `get` found it: what it carries, and its bytes. A whole read is
|
|
379
|
+
* checked against its SHA-256 and fails CorruptError before its last bytes
|
|
380
|
+
* when they changed; `slice` reads a range, which is not checked.
|
|
381
|
+
*/
|
|
382
|
+
export class StoredFile extends FileBytes {
|
|
383
|
+
path;
|
|
384
|
+
etag;
|
|
385
|
+
contentType;
|
|
386
|
+
lastModified;
|
|
387
|
+
expires;
|
|
388
|
+
meta;
|
|
389
|
+
/** its bytes when they came with the get */
|
|
390
|
+
#whole;
|
|
391
|
+
#read;
|
|
392
|
+
/** Stored files come from `files.get`. */
|
|
393
|
+
constructor(info, whole, read) {
|
|
394
|
+
super(info.size, () => (whole === undefined ? read() : streamOf(whole)));
|
|
395
|
+
this.path = info.path;
|
|
396
|
+
this.etag = info.etag;
|
|
397
|
+
this.contentType = info.contentType;
|
|
398
|
+
this.lastModified = info.lastModified;
|
|
399
|
+
this.expires = info.expires;
|
|
400
|
+
this.meta = info.meta;
|
|
401
|
+
this.#whole = whole;
|
|
402
|
+
this.#read = read;
|
|
187
403
|
}
|
|
188
|
-
|
|
189
|
-
|
|
404
|
+
/**
|
|
405
|
+
* A range of its bytes, as Blob.slice takes one: an end left out is the
|
|
406
|
+
* file's, and a negative place counts back from it, so `slice(-500)` is the
|
|
407
|
+
* last 500 bytes.
|
|
408
|
+
*/
|
|
409
|
+
slice(start, end) {
|
|
410
|
+
const from = placeOf(start, this.size, 0);
|
|
411
|
+
const to = Math.max(from, placeOf(end, this.size, this.size));
|
|
412
|
+
const whole = this.#whole;
|
|
413
|
+
if (whole !== undefined) {
|
|
414
|
+
const bytes = whole.subarray(from, to);
|
|
415
|
+
return new FileBytes(bytes.length, () => streamOf(bytes));
|
|
416
|
+
}
|
|
417
|
+
return new FileBytes(to - from, () => this.#read(from, to - from));
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
/** A place in bytes as Blob.slice reads one: negative from the end, and within the size. */
|
|
421
|
+
function placeOf(place, size, absent) {
|
|
422
|
+
if (place === undefined) {
|
|
423
|
+
return absent;
|
|
190
424
|
}
|
|
191
|
-
|
|
192
|
-
|
|
425
|
+
const at = Math.trunc(place);
|
|
426
|
+
return at < 0 ? Math.max(size + at, 0) : Math.min(at, size);
|
|
427
|
+
}
|
|
428
|
+
/** Bytes in hand as a stream, a copy, so that a reader's changes stay its own. */
|
|
429
|
+
function streamOf(bytes) {
|
|
430
|
+
return new ReadableStream({
|
|
431
|
+
start(controller) {
|
|
432
|
+
if (bytes.length > 0) {
|
|
433
|
+
controller.enqueue(bytes.slice());
|
|
434
|
+
}
|
|
435
|
+
controller.close();
|
|
436
|
+
},
|
|
437
|
+
});
|
|
438
|
+
}
|
|
439
|
+
/** Enqueues a piece's bytes, none when it carries none. */
|
|
440
|
+
function enqueue(controller, body) {
|
|
441
|
+
const bytes = BlobsPiece.decode(body).bytes ?? nothing;
|
|
442
|
+
if (bytes.length > 0) {
|
|
443
|
+
controller.enqueue(bytes);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
function sourceOf(body) {
|
|
447
|
+
const bytes = typeof body === 'string'
|
|
448
|
+
? encoder.encode(body)
|
|
449
|
+
: body instanceof ArrayBuffer
|
|
450
|
+
? new Uint8Array(body)
|
|
451
|
+
: body;
|
|
452
|
+
if (bytes instanceof Uint8Array) {
|
|
193
453
|
return {
|
|
194
|
-
size:
|
|
454
|
+
size: bytes.length,
|
|
195
455
|
contentType: undefined,
|
|
196
456
|
replayable: true,
|
|
197
|
-
chunks: () =>
|
|
457
|
+
chunks: () => [bytes],
|
|
458
|
+
whole: async (piece) => (bytes.length <= piece ? bytes : undefined),
|
|
198
459
|
};
|
|
199
460
|
}
|
|
200
|
-
if (
|
|
201
|
-
const blob =
|
|
202
|
-
const contentType = blob.type === '' ? undefined : blob.type;
|
|
461
|
+
if (bytes instanceof Blob) {
|
|
462
|
+
const blob = bytes;
|
|
203
463
|
return {
|
|
204
|
-
size:
|
|
205
|
-
contentType,
|
|
464
|
+
size: blob.size,
|
|
465
|
+
contentType: blob.type === '' ? undefined : blob.type,
|
|
206
466
|
replayable: true,
|
|
207
|
-
chunks: () =>
|
|
467
|
+
chunks: () => blob.stream(),
|
|
468
|
+
whole: async (piece) => blob.size <= piece ? new Uint8Array(await blob.arrayBuffer()) : undefined,
|
|
208
469
|
};
|
|
209
470
|
}
|
|
210
|
-
const stream =
|
|
211
|
-
return {
|
|
471
|
+
const stream = bytes;
|
|
472
|
+
return {
|
|
473
|
+
size: undefined,
|
|
474
|
+
contentType: undefined,
|
|
475
|
+
replayable: false,
|
|
476
|
+
chunks: () => stream,
|
|
477
|
+
whole: async () => undefined,
|
|
478
|
+
};
|
|
212
479
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
480
|
+
/**
|
|
481
|
+
* The bytes of chunks again in pieces of `size` bytes, the last shorter. A
|
|
482
|
+
* chunk that holds a whole piece goes as it is, without a copy.
|
|
483
|
+
*/
|
|
484
|
+
async function* pieces(chunks, size) {
|
|
485
|
+
let piece = new Uint8Array(size);
|
|
486
|
+
let filled = 0;
|
|
487
|
+
for await (const chunk of chunks) {
|
|
488
|
+
let at = 0;
|
|
489
|
+
while (at < chunk.length) {
|
|
490
|
+
if (filled === 0 && chunk.length - at >= size) {
|
|
491
|
+
yield chunk.subarray(at, at + size);
|
|
492
|
+
at += size;
|
|
493
|
+
continue;
|
|
494
|
+
}
|
|
495
|
+
const taken = Math.min(size - filled, chunk.length - at);
|
|
496
|
+
piece.set(chunk.subarray(at, at + taken), filled);
|
|
497
|
+
filled += taken;
|
|
498
|
+
at += taken;
|
|
499
|
+
if (filled === size) {
|
|
500
|
+
yield piece;
|
|
501
|
+
piece = new Uint8Array(size);
|
|
502
|
+
filled = 0;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
216
505
|
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
for await (const piece of stream) {
|
|
220
|
-
yield* sliced(piece);
|
|
506
|
+
if (filled > 0) {
|
|
507
|
+
yield piece.subarray(0, filled);
|
|
221
508
|
}
|
|
222
509
|
}
|
|
223
510
|
/**
|
|
224
|
-
* Sends
|
|
225
|
-
* waits for
|
|
226
|
-
* DATA
|
|
511
|
+
* Sends an upload's pieces as DATA within the stream's credit, the last
|
|
512
|
+
* ending them, and waits for what the commit wrote. A connection lost before
|
|
513
|
+
* that last DATA left aborted the upload, which left nothing, and a body that
|
|
514
|
+
* reads again goes again on the next connection; lost after it, the
|
|
227
515
|
* commit's outcome is unknown.
|
|
228
516
|
*/
|
|
229
|
-
async function
|
|
230
|
-
const stream = await connection.session.open(methods['blobs.put'], request, false);
|
|
517
|
+
async function uploaded(stream, source, piece, signal) {
|
|
231
518
|
const unwatch = watch(signal, stream);
|
|
232
|
-
let
|
|
519
|
+
let ending = false;
|
|
233
520
|
try {
|
|
234
521
|
let last;
|
|
235
|
-
for await (const
|
|
236
|
-
if (piece.length === 0) {
|
|
237
|
-
continue;
|
|
238
|
-
}
|
|
522
|
+
for await (const bytes of pieces(source.chunks(), piece)) {
|
|
239
523
|
if (last !== undefined) {
|
|
240
|
-
await stream.send(last, false);
|
|
524
|
+
await stream.send(BlobsPiece.encode({ bytes: last }), false);
|
|
241
525
|
}
|
|
242
|
-
last =
|
|
526
|
+
last = bytes;
|
|
243
527
|
}
|
|
244
|
-
|
|
245
|
-
await stream.send(last ??
|
|
246
|
-
|
|
247
|
-
|
|
528
|
+
ending = true;
|
|
529
|
+
await stream.send(BlobsPiece.encode({ bytes: last ?? nothing }), true);
|
|
530
|
+
// the RESPONSE that began it, and then what the commit wrote
|
|
531
|
+
await stream.next();
|
|
532
|
+
return writtenOf((await stream.next()).body);
|
|
248
533
|
}
|
|
249
534
|
catch (err) {
|
|
250
|
-
if (err instanceof
|
|
251
|
-
if (ended) {
|
|
252
|
-
throw new OutcomeUnknownError(`the connection was lost as the upload committed: ${err.message}`);
|
|
253
|
-
}
|
|
254
|
-
if (!source.replayable) {
|
|
255
|
-
throw new ClosedError(`the connection was lost during the upload, which left nothing: ${err.message}`);
|
|
256
|
-
}
|
|
535
|
+
if (err instanceof ConnectionLostError && !ending && source.replayable) {
|
|
257
536
|
// the server aborted what it had of it, so the whole body goes again
|
|
258
|
-
throw new
|
|
537
|
+
throw new ConnectionLostError(err.message, false);
|
|
259
538
|
}
|
|
260
539
|
stream.cancel();
|
|
261
|
-
throw err;
|
|
540
|
+
throw lostAs(err, ending);
|
|
262
541
|
}
|
|
263
542
|
finally {
|
|
264
543
|
unwatch();
|
|
265
544
|
}
|
|
266
545
|
}
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
unwatch();
|
|
283
|
-
controller.error(err);
|
|
284
|
-
}
|
|
285
|
-
},
|
|
286
|
-
cancel() {
|
|
287
|
-
unwatch();
|
|
288
|
-
stream.cancel();
|
|
289
|
-
},
|
|
290
|
-
});
|
|
291
|
-
const bytes = async () => new Uint8Array(await new Response(body).arrayBuffer());
|
|
546
|
+
/** What a lost connection means for an upload: nothing left before its last DATA, unknown after. */
|
|
547
|
+
function lostAs(err, ending) {
|
|
548
|
+
if (!(err instanceof ConnectionLostError)) {
|
|
549
|
+
return err;
|
|
550
|
+
}
|
|
551
|
+
if (ending) {
|
|
552
|
+
return new OutcomeUnknownError(`the connection was lost as the upload committed: ${err.message}`);
|
|
553
|
+
}
|
|
554
|
+
return new ClosedError(`the connection was lost during the upload, which left nothing: ${err.message}`);
|
|
555
|
+
}
|
|
556
|
+
function writtenOf(body) {
|
|
557
|
+
const written = BlobsWritten.decode(body);
|
|
558
|
+
return written.info === undefined ? undefined : infoOf(written.info);
|
|
559
|
+
}
|
|
560
|
+
function infoOf(info) {
|
|
292
561
|
return {
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
562
|
+
path: info.path ?? '',
|
|
563
|
+
size: info.size ?? 0,
|
|
564
|
+
etag: info.etag ?? '',
|
|
565
|
+
contentType: info.contentType ?? '',
|
|
566
|
+
lastModified: new Date(info.lastModified ?? 0),
|
|
567
|
+
expires: dateOf(info.expires),
|
|
568
|
+
meta: info.meta ?? {},
|
|
298
569
|
};
|
|
299
570
|
}
|
|
571
|
+
/** A folder's segment as text, an integer its decimal spelling; the server refuses one that is not a segment. */
|
|
572
|
+
function segmentOf(segment) {
|
|
573
|
+
if (typeof segment === 'number' && !Number.isSafeInteger(segment)) {
|
|
574
|
+
throw new InvalidError(`the folder segment ${segment}: a segment is text or an integer`);
|
|
575
|
+
}
|
|
576
|
+
return String(segment);
|
|
577
|
+
}
|
|
578
|
+
const binary = { KiB: 2 ** 10, MiB: 2 ** 20, GiB: 2 ** 30, TiB: 2 ** 40 };
|
|
579
|
+
/** A size's bytes. '2GB' is refused: it is 10⁹ bytes to some and 2³⁰ to others. */
|
|
580
|
+
function bytesOf(size) {
|
|
581
|
+
if (typeof size === 'number') {
|
|
582
|
+
if (!Number.isSafeInteger(size) || size < 0) {
|
|
583
|
+
throw new InvalidError(`a size of ${size} bytes`);
|
|
584
|
+
}
|
|
585
|
+
return size;
|
|
586
|
+
}
|
|
587
|
+
const parts = /^(\d+(?:\.\d+)?)(KiB|MiB|GiB|TiB)$/.exec(size.trim());
|
|
588
|
+
if (parts === null) {
|
|
589
|
+
throw new InvalidError(`the size ${JSON.stringify(size)}; write it in bytes, or as 64KiB, 10MiB, 2GiB or 1TiB`);
|
|
590
|
+
}
|
|
591
|
+
return Math.floor(Number(parts[1]) * binary[parts[2]]);
|
|
592
|
+
}
|